aboutsummaryrefslogtreecommitdiffstats
path: root/arch
diff options
authorMarc Zyngier <maz@kernel.org>2026-05-28 10:23:29 +0100
committerMarc Zyngier <maz@kernel.org>2026-05-28 10:23:29 +0100
commit6c3bda3efca7ea321d73c2d4df822e736a00c5a3 (patch)
treeb989f2a2b992152a5e7b7f78c4c77d8b342e7121 /arch
parentb4d1896c717a48fc4b9c9fdaee04d7d603fd1da2 (diff)
parent8853566dfbab1a255ae72676ab5ec43e1631ddb7 (diff)
downloadlinux-next-history-6c3bda3efca7ea321d73c2d4df822e736a00c5a3.tar.gz
Merge branch kvm-arm64/nv-granule-sizes into kvmarm-master/next
* kvm-arm64/nv-granule-sizes: : . : Tidying up of the behaviour when the selected page size in not : implemented, courtesy of Wei-Lin Chang. From the initial cover : letter: : : "This small series fixes the granule size selection for software stage-1 : and stage-2 walks. Previously we treat the guest's TCR/VTCR.TGx as-is : and use the encoded granule size for the walks. However this is : incorrect if the granule sizes are not advertised in the guest's : ID_AA64MMFR0_EL1.TGRAN*. The architecture specifies that when an : unsupported size is programed in TGx, it must be treated as an : implemented size. Fix this by choosing an available one while : prioritizing PAGE_SIZE." : . KVM: arm64: Fallback to a supported value for unsupported guest TGx KVM: arm64: nv: Use literal granule size in TLBI range calculation KVM: arm64: Factor out TG0/1 decoding of VTCR and TCR KVM: arm64: nv: Rename vtcr_to_walk_info() to setup_s2_walk() Signed-off-by: Marc Zyngier <maz@kernel.org>
Diffstat (limited to 'arch')
-rw-r--r--arch/arm64/kvm/at.c125
-rw-r--r--arch/arm64/kvm/nested.c144
2 files changed, 196 insertions, 73 deletions
diff --git a/arch/arm64/kvm/at.c b/arch/arm64/kvm/at.c
index 9f8f0ae8e86e8..60d51e98ccb00 100644
--- a/arch/arm64/kvm/at.c
+++ b/arch/arm64/kvm/at.c
@@ -136,14 +136,106 @@ static void compute_s1poe(struct kvm_vcpu *vcpu, struct s1_walk_info *wi)
wi->e0poe = (wi->regime != TR_EL2) && (val & TCR2_EL1_E0POE);
}
+#define _has_tgran(__r, __sz) \
+ ({ \
+ u64 _s1, _mmfr0 = __r; \
+ \
+ _s1 = SYS_FIELD_GET(ID_AA64MMFR0_EL1, \
+ TGRAN##__sz, _mmfr0); \
+ \
+ _s1 != ID_AA64MMFR0_EL1_TGRAN##__sz##_NI; \
+ })
+
+static bool has_tgran(u64 mmfr0, unsigned int shift)
+{
+ switch (shift) {
+ case 12:
+ return _has_tgran(mmfr0, 4);
+ case 14:
+ return _has_tgran(mmfr0, 16);
+ case 16:
+ return _has_tgran(mmfr0, 64);
+ default:
+ BUG();
+ }
+}
+
+static unsigned int tcr_to_tg0_pgshift(u64 tcr)
+{
+ u64 tg0 = tcr & TCR_TG0_MASK;
+
+ switch (tg0) {
+ case TCR_TG0_4K:
+ return 12;
+ case TCR_TG0_16K:
+ return 14;
+ case TCR_TG0_64K:
+ default: /* IMPDEF: treat any other value as 64k */
+ return 16;
+ }
+}
+
+static unsigned int tcr_to_tg1_pgshift(u64 tcr)
+{
+ u64 tg1 = tcr & TCR_TG1_MASK;
+
+ switch (tg1) {
+ case TCR_TG1_4K:
+ return 12;
+ case TCR_TG1_16K:
+ return 14;
+ case TCR_TG1_64K:
+ default: /* IMPDEF: treat any other value as 64k */
+ return 16;
+ }
+}
+
+static unsigned int fallback_tgran_shift(u64 mmfr0)
+{
+ if (has_tgran(mmfr0, PAGE_SHIFT))
+ return PAGE_SHIFT;
+ else if (has_tgran(mmfr0, 12))
+ return 12;
+ else if (has_tgran(mmfr0, 14))
+ return 14;
+ else if (has_tgran(mmfr0, 16))
+ return 16;
+ else /* Should be unreacheable */
+ return PAGE_SHIFT;
+}
+
+static unsigned int tcr_tg_pgshift(struct kvm *kvm, u64 tcr, bool upper_range)
+{
+ u64 mmfr0 = kvm_read_vm_id_reg(kvm, SYS_ID_AA64MMFR0_EL1);
+ unsigned int shift;
+
+ /* Someone was silly enough to encode TG0/TG1 differently */
+ if (upper_range)
+ shift = tcr_to_tg1_pgshift(tcr);
+ else
+ shift = tcr_to_tg0_pgshift(tcr);
+
+ /*
+ * If TGx is programmed to an unimplemented value (not advertised in
+ * ID_AA64MMFR0_EL1), we should treat it as if an implemented value is
+ * written, as per the architecture. Choose an available one while
+ * prioritizing PAGE_SIZE.
+ */
+ if (!has_tgran(mmfr0, shift))
+ return fallback_tgran_shift(mmfr0);
+
+ return shift;
+}
+
static int setup_s1_walk(struct kvm_vcpu *vcpu, struct s1_walk_info *wi,
struct s1_walk_result *wr, u64 va)
{
- u64 hcr, sctlr, tcr, tg, ps, ia_bits, ttbr;
+ u64 hcr, sctlr, tcr, ps, ia_bits, ttbr;
unsigned int stride, x;
- bool va55, tbi, lva;
+ bool va55, tbi, lva, upper_range;
va55 = va & BIT(55);
+ upper_range = va55 && wi->regime != TR_EL2;
if (vcpu_has_nv(vcpu)) {
hcr = __vcpu_sys_reg(vcpu, HCR_EL2);
@@ -174,35 +266,12 @@ static int setup_s1_walk(struct kvm_vcpu *vcpu, struct s1_walk_info *wi,
BUG();
}
- /* Someone was silly enough to encode TG0/TG1 differently */
- if (va55 && wi->regime != TR_EL2) {
+ if (upper_range)
wi->txsz = FIELD_GET(TCR_T1SZ_MASK, tcr);
- tg = FIELD_GET(TCR_TG1_MASK, tcr);
-
- switch (tg << TCR_TG1_SHIFT) {
- case TCR_TG1_4K:
- wi->pgshift = 12; break;
- case TCR_TG1_16K:
- wi->pgshift = 14; break;
- case TCR_TG1_64K:
- default: /* IMPDEF: treat any other value as 64k */
- wi->pgshift = 16; break;
- }
- } else {
+ else
wi->txsz = FIELD_GET(TCR_T0SZ_MASK, tcr);
- tg = FIELD_GET(TCR_TG0_MASK, tcr);
-
- switch (tg << TCR_TG0_SHIFT) {
- case TCR_TG0_4K:
- wi->pgshift = 12; break;
- case TCR_TG0_16K:
- wi->pgshift = 14; break;
- case TCR_TG0_64K:
- default: /* IMPDEF: treat any other value as 64k */
- wi->pgshift = 16; break;
- }
- }
+ wi->pgshift = tcr_tg_pgshift(vcpu->kvm, tcr, upper_range);
wi->pa52bit = has_52bit_pa(vcpu, wi, tcr);
ia_bits = get_ia_size(wi);
diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c
index 883b6c1008fbb..3204b3ef60ddd 100644
--- a/arch/arm64/kvm/nested.c
+++ b/arch/arm64/kvm/nested.c
@@ -378,32 +378,104 @@ static int walk_nested_s2_pgd(struct kvm_vcpu *vcpu, phys_addr_t ipa,
return 0;
}
-static void vtcr_to_walk_info(u64 vtcr, struct s2_walk_info *wi)
+#define _has_tgran_2(__r, __sz) \
+ ({ \
+ u64 _s1, _s2, _mmfr0 = __r; \
+ \
+ _s2 = SYS_FIELD_GET(ID_AA64MMFR0_EL1, \
+ TGRAN##__sz##_2, _mmfr0); \
+ \
+ _s1 = SYS_FIELD_GET(ID_AA64MMFR0_EL1, \
+ TGRAN##__sz, _mmfr0); \
+ \
+ ((_s2 != ID_AA64MMFR0_EL1_TGRAN##__sz##_2_NI && \
+ _s2 != ID_AA64MMFR0_EL1_TGRAN##__sz##_2_TGRAN##__sz) || \
+ (_s2 == ID_AA64MMFR0_EL1_TGRAN##__sz##_2_TGRAN##__sz && \
+ _s1 != ID_AA64MMFR0_EL1_TGRAN##__sz##_NI)); \
+ })
+
+static bool has_tgran_2(u64 mmfr0, unsigned int shift)
+{
+ switch (shift) {
+ case 12:
+ return _has_tgran_2(mmfr0, 4);
+ case 14:
+ return _has_tgran_2(mmfr0, 16);
+ case 16:
+ return _has_tgran_2(mmfr0, 64);
+ default:
+ BUG();
+ }
+}
+
+static unsigned int fallback_tgran2_shift(u64 mmfr0)
{
- wi->t0sz = vtcr & TCR_EL2_T0SZ_MASK;
+ if (has_tgran_2(mmfr0, PAGE_SHIFT))
+ return PAGE_SHIFT;
+ else if (has_tgran_2(mmfr0, 12))
+ return 12;
+ else if (has_tgran_2(mmfr0, 14))
+ return 14;
+ else if (has_tgran_2(mmfr0, 16))
+ return 16;
+ else
+ return PAGE_SHIFT;
+}
- switch (FIELD_GET(VTCR_EL2_TG0_MASK, vtcr)) {
+static unsigned int vtcr_to_tg0_pgshift(struct kvm *kvm, u64 vtcr)
+{
+ u64 tg0 = FIELD_GET(VTCR_EL2_TG0_MASK, vtcr);
+ u64 mmfr0 = kvm_read_vm_id_reg(kvm, SYS_ID_AA64MMFR0_EL1);
+ unsigned int shift;
+
+ switch (tg0) {
case VTCR_EL2_TG0_4K:
- wi->pgshift = 12; break;
+ shift = 12;
+ break;
case VTCR_EL2_TG0_16K:
- wi->pgshift = 14; break;
+ shift = 14;
+ break;
case VTCR_EL2_TG0_64K:
- default: /* IMPDEF: treat any other value as 64k */
- wi->pgshift = 16; break;
+ /* IMPDEF: treat any other value as 64k, subject to fallback */
+ default:
+ shift = 16;
}
+ /*
+ * If TGx is programmed to an unimplemented value (not advertised in
+ * ID_AA64MMFR0_EL1), we should treat it as if an implemented value is
+ * written, as per the architecture. Choose an available one while
+ * prioritizing PAGE_SIZE.
+ */
+ if (!has_tgran_2(mmfr0, shift))
+ return fallback_tgran2_shift(mmfr0);
+
+ return shift;
+}
+
+static size_t vtcr_to_tg0_pgsize(struct kvm *kvm, u64 vtcr)
+{
+ return BIT(vtcr_to_tg0_pgshift(kvm, vtcr));
+}
+
+static void setup_s2_walk(struct kvm_vcpu *vcpu, struct s2_walk_info *wi)
+{
+ u64 vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2);
+
+ wi->baddr = vcpu_read_sys_reg(vcpu, VTTBR_EL2);
+ wi->t0sz = vtcr & VTCR_EL2_T0SZ_MASK;
+ wi->pgshift = vtcr_to_tg0_pgshift(vcpu->kvm, vtcr);
wi->sl = FIELD_GET(VTCR_EL2_SL0_MASK, vtcr);
/* Global limit for now, should eventually be per-VM */
wi->max_oa_bits = min(get_kvm_ipa_limit(),
ps_to_output_size(FIELD_GET(VTCR_EL2_PS_MASK, vtcr), false));
-
wi->ha = vtcr & VTCR_EL2_HA;
+ wi->be = vcpu_read_sys_reg(vcpu, SCTLR_EL2) & SCTLR_ELx_EE;
}
int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa,
struct kvm_s2_trans *result)
{
- u64 vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2);
struct s2_walk_info wi;
int ret;
@@ -412,11 +484,7 @@ int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa,
if (!vcpu_has_nv(vcpu))
return 0;
- wi.baddr = vcpu_read_sys_reg(vcpu, VTTBR_EL2);
-
- vtcr_to_walk_info(vtcr, &wi);
-
- wi.be = vcpu_read_sys_reg(vcpu, SCTLR_EL2) & SCTLR_ELx_EE;
+ setup_s2_walk(vcpu, &wi);
ret = walk_nested_s2_pgd(vcpu, gipa, &wi, result);
if (ret)
@@ -512,20 +580,21 @@ static u8 pgshift_level_to_ttl(u16 shift, u8 level)
*/
static u8 get_guest_mapping_ttl(struct kvm_s2_mmu *mmu, u64 addr)
{
- u64 tmp, sz = 0, vtcr = mmu->tlb_vtcr;
+ size_t tg0_size = vtcr_to_tg0_pgsize(kvm_s2_mmu_to_kvm(mmu), mmu->tlb_vtcr);
+ u64 tmp, sz = 0;
kvm_pte_t pte;
u8 ttl, level;
lockdep_assert_held_write(&kvm_s2_mmu_to_kvm(mmu)->mmu_lock);
- switch (FIELD_GET(VTCR_EL2_TG0_MASK, vtcr)) {
- case VTCR_EL2_TG0_4K:
+ switch (tg0_size) {
+ case SZ_4K:
ttl = (TLBI_TTL_TG_4K << 2);
break;
- case VTCR_EL2_TG0_16K:
+ case SZ_16K:
ttl = (TLBI_TTL_TG_16K << 2);
break;
- case VTCR_EL2_TG0_64K:
+ case SZ_64K:
default: /* IMPDEF: treat any other value as 64k */
ttl = (TLBI_TTL_TG_64K << 2);
break;
@@ -535,19 +604,19 @@ static u8 get_guest_mapping_ttl(struct kvm_s2_mmu *mmu, u64 addr)
again:
/* Iteratively compute the block sizes for a particular granule size */
- switch (FIELD_GET(VTCR_EL2_TG0_MASK, vtcr)) {
- case VTCR_EL2_TG0_4K:
+ switch (tg0_size) {
+ case SZ_4K:
if (sz < SZ_4K) sz = SZ_4K;
else if (sz < SZ_2M) sz = SZ_2M;
else if (sz < SZ_1G) sz = SZ_1G;
else sz = 0;
break;
- case VTCR_EL2_TG0_16K:
+ case SZ_16K:
if (sz < SZ_16K) sz = SZ_16K;
else if (sz < SZ_32M) sz = SZ_32M;
else sz = 0;
break;
- case VTCR_EL2_TG0_64K:
+ case SZ_64K:
default: /* IMPDEF: treat any other value as 64k */
if (sz < SZ_64K) sz = SZ_64K;
else if (sz < SZ_512M) sz = SZ_512M;
@@ -598,14 +667,14 @@ unsigned long compute_tlb_inval_range(struct kvm_s2_mmu *mmu, u64 val)
if (!max_size) {
/* Compute the maximum extent of the invalidation */
- switch (FIELD_GET(VTCR_EL2_TG0_MASK, mmu->tlb_vtcr)) {
- case VTCR_EL2_TG0_4K:
+ switch (vtcr_to_tg0_pgsize(kvm, mmu->tlb_vtcr)) {
+ case SZ_4K:
max_size = SZ_1G;
break;
- case VTCR_EL2_TG0_16K:
+ case SZ_16K:
max_size = SZ_32M;
break;
- case VTCR_EL2_TG0_64K:
+ case SZ_64K:
default: /* IMPDEF: treat any other value as 64k */
/*
* No, we do not support 52bit IPA in nested yet. Once
@@ -1498,21 +1567,6 @@ static void kvm_map_l1_vncr(struct kvm_vcpu *vcpu)
}
}
-#define has_tgran_2(__r, __sz) \
- ({ \
- u64 _s1, _s2, _mmfr0 = __r; \
- \
- _s2 = SYS_FIELD_GET(ID_AA64MMFR0_EL1, \
- TGRAN##__sz##_2, _mmfr0); \
- \
- _s1 = SYS_FIELD_GET(ID_AA64MMFR0_EL1, \
- TGRAN##__sz, _mmfr0); \
- \
- ((_s2 != ID_AA64MMFR0_EL1_TGRAN##__sz##_2_NI && \
- _s2 != ID_AA64MMFR0_EL1_TGRAN##__sz##_2_TGRAN##__sz) || \
- (_s2 == ID_AA64MMFR0_EL1_TGRAN##__sz##_2_TGRAN##__sz && \
- _s1 != ID_AA64MMFR0_EL1_TGRAN##__sz##_NI)); \
- })
/*
* Our emulated CPU doesn't support all the possible features. For the
* sake of simplicity (and probably mental sanity), wipe out a number
@@ -1599,15 +1653,15 @@ u64 limit_nv_id_reg(struct kvm *kvm, u32 reg, u64 val)
*/
switch (PAGE_SIZE) {
case SZ_4K:
- if (has_tgran_2(orig_val, 4))
+ if (_has_tgran_2(orig_val, 4))
val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR0_EL1, TGRAN4_2, IMP);
fallthrough;
case SZ_16K:
- if (has_tgran_2(orig_val, 16))
+ if (_has_tgran_2(orig_val, 16))
val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR0_EL1, TGRAN16_2, IMP);
fallthrough;
case SZ_64K:
- if (has_tgran_2(orig_val, 64))
+ if (_has_tgran_2(orig_val, 64))
val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR0_EL1, TGRAN64_2, IMP);
break;
}
generated by cgit 1.3-korg (git 2.53.0) at 2026-06-01 04:05:07 +0000