hyp/nvhe/sys_regs.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Copyright (C) 2021 Google LLC
0004  * Author: Fuad Tabba <tabba@google.com>
0005  */
0006
0007 #include <linux/irqchip/arm-gic-v3.h>
0008
0009 #include <asm/kvm_asm.h>
0010 #include <asm/kvm_mmu.h>
0011
0012 #include <hyp/adjust_pc.h>
0013
0014 #include <nvhe/fixed_config.h>
0015
0016 #include "../../sys_regs.h"
0017
0018 /*
0019  * Copies of the host's CPU features registers holding sanitized values at hyp.
0020  */
0021 u64 id_aa64pfr0_el1_sys_val;
0022 u64 id_aa64pfr1_el1_sys_val;
0023 u64 id_aa64isar0_el1_sys_val;
0024 u64 id_aa64isar1_el1_sys_val;
0025 u64 id_aa64isar2_el1_sys_val;
0026 u64 id_aa64mmfr0_el1_sys_val;
0027 u64 id_aa64mmfr1_el1_sys_val;
0028 u64 id_aa64mmfr2_el1_sys_val;
0029
0030 /*
0031  * Inject an unknown/undefined exception to an AArch64 guest while most of its
0032  * sysregs are live.
0033  */
0034 static void inject_undef64(struct kvm_vcpu *vcpu)
0035 {
0036     u64 esr = (ESR_ELx_EC_UNKNOWN << ESR_ELx_EC_SHIFT);
0037
0038     *vcpu_pc(vcpu) = read_sysreg_el2(SYS_ELR);
0039     *vcpu_cpsr(vcpu) = read_sysreg_el2(SYS_SPSR);
0040
0041     kvm_pend_exception(vcpu, EXCEPT_AA64_EL1_SYNC);
0042
0043     __kvm_adjust_pc(vcpu);
0044
0045     write_sysreg_el1(esr, SYS_ESR);
0046     write_sysreg_el1(read_sysreg_el2(SYS_ELR), SYS_ELR);
0047     write_sysreg_el2(*vcpu_pc(vcpu), SYS_ELR);
0048     write_sysreg_el2(*vcpu_cpsr(vcpu), SYS_SPSR);
0049 }
0050
0051 /*
0052  * Returns the restricted features values of the feature register based on the
0053  * limitations in restrict_fields.
0054  * A feature id field value of 0b0000 does not impose any restrictions.
0055  * Note: Use only for unsigned feature field values.
0056  */
0057 static u64 get_restricted_features_unsigned(u64 sys_reg_val,
0058                         u64 restrict_fields)
0059 {
0060     u64 value = 0UL;
0061     u64 mask = GENMASK_ULL(ARM64_FEATURE_FIELD_BITS - 1, 0);
0062
0063     /*
0064      * According to the Arm Architecture Reference Manual, feature fields
0065      * use increasing values to indicate increases in functionality.
0066      * Iterate over the restricted feature fields and calculate the minimum
0067      * unsigned value between the one supported by the system, and what the
0068      * value is being restricted to.
0069      */
0070     while (sys_reg_val && restrict_fields) {
0071         value |= min(sys_reg_val & mask, restrict_fields & mask);
0072         sys_reg_val &= ~mask;
0073         restrict_fields &= ~mask;
0074         mask <<= ARM64_FEATURE_FIELD_BITS;
0075     }
0076
0077     return value;
0078 }
0079
0080 /*
0081  * Functions that return the value of feature id registers for protected VMs
0082  * based on allowed features, system features, and KVM support.
0083  */
0084
0085 static u64 get_pvm_id_aa64pfr0(const struct kvm_vcpu *vcpu)
0086 {
0087     const struct kvm *kvm = (const struct kvm *)kern_hyp_va(vcpu->kvm);
0088     u64 set_mask = 0;
0089     u64 allow_mask = PVM_ID_AA64PFR0_ALLOW;
0090
0091     set_mask |= get_restricted_features_unsigned(id_aa64pfr0_el1_sys_val,
0092         PVM_ID_AA64PFR0_RESTRICT_UNSIGNED);
0093
0094     /* Spectre and Meltdown mitigation in KVM */
0095     set_mask |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_CSV2),
0096                    (u64)kvm->arch.pfr0_csv2);
0097     set_mask |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_CSV3),
0098                    (u64)kvm->arch.pfr0_csv3);
0099
0100     return (id_aa64pfr0_el1_sys_val & allow_mask) | set_mask;
0101 }
0102
0103 static u64 get_pvm_id_aa64pfr1(const struct kvm_vcpu *vcpu)
0104 {
0105     const struct kvm *kvm = (const struct kvm *)kern_hyp_va(vcpu->kvm);
0106     u64 allow_mask = PVM_ID_AA64PFR1_ALLOW;
0107
0108     if (!kvm_has_mte(kvm))
0109         allow_mask &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_MTE);
0110
0111     return id_aa64pfr1_el1_sys_val & allow_mask;
0112 }
0113
0114 static u64 get_pvm_id_aa64zfr0(const struct kvm_vcpu *vcpu)
0115 {
0116     /*
0117      * No support for Scalable Vectors, therefore, hyp has no sanitized
0118      * copy of the feature id register.
0119      */
0120     BUILD_BUG_ON(PVM_ID_AA64ZFR0_ALLOW != 0ULL);
0121     return 0;
0122 }
0123
0124 static u64 get_pvm_id_aa64dfr0(const struct kvm_vcpu *vcpu)
0125 {
0126     /*
0127      * No support for debug, including breakpoints, and watchpoints,
0128      * therefore, pKVM has no sanitized copy of the feature id register.
0129      */
0130     BUILD_BUG_ON(PVM_ID_AA64DFR0_ALLOW != 0ULL);
0131     return 0;
0132 }
0133
0134 static u64 get_pvm_id_aa64dfr1(const struct kvm_vcpu *vcpu)
0135 {
0136     /*
0137      * No support for debug, therefore, hyp has no sanitized copy of the
0138      * feature id register.
0139      */
0140     BUILD_BUG_ON(PVM_ID_AA64DFR1_ALLOW != 0ULL);
0141     return 0;
0142 }
0143
0144 static u64 get_pvm_id_aa64afr0(const struct kvm_vcpu *vcpu)
0145 {
0146     /*
0147      * No support for implementation defined features, therefore, hyp has no
0148      * sanitized copy of the feature id register.
0149      */
0150     BUILD_BUG_ON(PVM_ID_AA64AFR0_ALLOW != 0ULL);
0151     return 0;
0152 }
0153
0154 static u64 get_pvm_id_aa64afr1(const struct kvm_vcpu *vcpu)
0155 {
0156     /*
0157      * No support for implementation defined features, therefore, hyp has no
0158      * sanitized copy of the feature id register.
0159      */
0160     BUILD_BUG_ON(PVM_ID_AA64AFR1_ALLOW != 0ULL);
0161     return 0;
0162 }
0163
0164 static u64 get_pvm_id_aa64isar0(const struct kvm_vcpu *vcpu)
0165 {
0166     return id_aa64isar0_el1_sys_val & PVM_ID_AA64ISAR0_ALLOW;
0167 }
0168
0169 static u64 get_pvm_id_aa64isar1(const struct kvm_vcpu *vcpu)
0170 {
0171     u64 allow_mask = PVM_ID_AA64ISAR1_ALLOW;
0172
0173     if (!vcpu_has_ptrauth(vcpu))
0174         allow_mask &= ~(ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_APA) |
0175                 ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_API) |
0176                 ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_GPA) |
0177                 ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_GPI));
0178
0179     return id_aa64isar1_el1_sys_val & allow_mask;
0180 }
0181
0182 static u64 get_pvm_id_aa64isar2(const struct kvm_vcpu *vcpu)
0183 {
0184     u64 allow_mask = PVM_ID_AA64ISAR2_ALLOW;
0185
0186     if (!vcpu_has_ptrauth(vcpu))
0187         allow_mask &= ~(ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_APA3) |
0188                 ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_GPA3));
0189
0190     return id_aa64isar2_el1_sys_val & allow_mask;
0191 }
0192
0193 static u64 get_pvm_id_aa64mmfr0(const struct kvm_vcpu *vcpu)
0194 {
0195     u64 set_mask;
0196
0197     set_mask = get_restricted_features_unsigned(id_aa64mmfr0_el1_sys_val,
0198         PVM_ID_AA64MMFR0_RESTRICT_UNSIGNED);
0199
0200     return (id_aa64mmfr0_el1_sys_val & PVM_ID_AA64MMFR0_ALLOW) | set_mask;
0201 }
0202
0203 static u64 get_pvm_id_aa64mmfr1(const struct kvm_vcpu *vcpu)
0204 {
0205     return id_aa64mmfr1_el1_sys_val & PVM_ID_AA64MMFR1_ALLOW;
0206 }
0207
0208 static u64 get_pvm_id_aa64mmfr2(const struct kvm_vcpu *vcpu)
0209 {
0210     return id_aa64mmfr2_el1_sys_val & PVM_ID_AA64MMFR2_ALLOW;
0211 }
0212
0213 /* Read a sanitized cpufeature ID register by its encoding */
0214 u64 pvm_read_id_reg(const struct kvm_vcpu *vcpu, u32 id)
0215 {
0216     switch (id) {
0217     case SYS_ID_AA64PFR0_EL1:
0218         return get_pvm_id_aa64pfr0(vcpu);
0219     case SYS_ID_AA64PFR1_EL1:
0220         return get_pvm_id_aa64pfr1(vcpu);
0221     case SYS_ID_AA64ZFR0_EL1:
0222         return get_pvm_id_aa64zfr0(vcpu);
0223     case SYS_ID_AA64DFR0_EL1:
0224         return get_pvm_id_aa64dfr0(vcpu);
0225     case SYS_ID_AA64DFR1_EL1:
0226         return get_pvm_id_aa64dfr1(vcpu);
0227     case SYS_ID_AA64AFR0_EL1:
0228         return get_pvm_id_aa64afr0(vcpu);
0229     case SYS_ID_AA64AFR1_EL1:
0230         return get_pvm_id_aa64afr1(vcpu);
0231     case SYS_ID_AA64ISAR0_EL1:
0232         return get_pvm_id_aa64isar0(vcpu);
0233     case SYS_ID_AA64ISAR1_EL1:
0234         return get_pvm_id_aa64isar1(vcpu);
0235     case SYS_ID_AA64ISAR2_EL1:
0236         return get_pvm_id_aa64isar2(vcpu);
0237     case SYS_ID_AA64MMFR0_EL1:
0238         return get_pvm_id_aa64mmfr0(vcpu);
0239     case SYS_ID_AA64MMFR1_EL1:
0240         return get_pvm_id_aa64mmfr1(vcpu);
0241     case SYS_ID_AA64MMFR2_EL1:
0242         return get_pvm_id_aa64mmfr2(vcpu);
0243     default:
0244         /* Unhandled ID register, RAZ */
0245         return 0;
0246     }
0247 }
0248
0249 static u64 read_id_reg(const struct kvm_vcpu *vcpu,
0250                struct sys_reg_desc const *r)
0251 {
0252     return pvm_read_id_reg(vcpu, reg_to_encoding(r));
0253 }
0254
0255 /* Handler to RAZ/WI sysregs */
0256 static bool pvm_access_raz_wi(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
0257                   const struct sys_reg_desc *r)
0258 {
0259     if (!p->is_write)
0260         p->regval = 0;
0261
0262     return true;
0263 }
0264
0265 /*
0266  * Accessor for AArch32 feature id registers.
0267  *
0268  * The value of these registers is "unknown" according to the spec if AArch32
0269  * isn't supported.
0270  */
0271 static bool pvm_access_id_aarch32(struct kvm_vcpu *vcpu,
0272                   struct sys_reg_params *p,
0273                   const struct sys_reg_desc *r)
0274 {
0275     if (p->is_write) {
0276         inject_undef64(vcpu);
0277         return false;
0278     }
0279
0280     /*
0281      * No support for AArch32 guests, therefore, pKVM has no sanitized copy
0282      * of AArch32 feature id registers.
0283      */
0284     BUILD_BUG_ON(FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1),
0285              PVM_ID_AA64PFR0_RESTRICT_UNSIGNED) > ID_AA64PFR0_ELx_64BIT_ONLY);
0286
0287     return pvm_access_raz_wi(vcpu, p, r);
0288 }
0289
0290 /*
0291  * Accessor for AArch64 feature id registers.
0292  *
0293  * If access is allowed, set the regval to the protected VM's view of the
0294  * register and return true.
0295  * Otherwise, inject an undefined exception and return false.
0296  */
0297 static bool pvm_access_id_aarch64(struct kvm_vcpu *vcpu,
0298                   struct sys_reg_params *p,
0299                   const struct sys_reg_desc *r)
0300 {
0301     if (p->is_write) {
0302         inject_undef64(vcpu);
0303         return false;
0304     }
0305
0306     p->regval = read_id_reg(vcpu, r);
0307     return true;
0308 }
0309
0310 static bool pvm_gic_read_sre(struct kvm_vcpu *vcpu,
0311                  struct sys_reg_params *p,
0312                  const struct sys_reg_desc *r)
0313 {
0314     /* pVMs only support GICv3. 'nuf said. */
0315     if (!p->is_write)
0316         p->regval = ICC_SRE_EL1_DIB | ICC_SRE_EL1_DFB | ICC_SRE_EL1_SRE;
0317
0318     return true;
0319 }
0320
0321 /* Mark the specified system register as an AArch32 feature id register. */
0322 #define AARCH32(REG) { SYS_DESC(REG), .access = pvm_access_id_aarch32 }
0323
0324 /* Mark the specified system register as an AArch64 feature id register. */
0325 #define AARCH64(REG) { SYS_DESC(REG), .access = pvm_access_id_aarch64 }
0326
0327 /*
0328  * sys_reg_desc initialiser for architecturally unallocated cpufeature ID
0329  * register with encoding Op0=3, Op1=0, CRn=0, CRm=crm, Op2=op2
0330  * (1 <= crm < 8, 0 <= Op2 < 8).
0331  */
0332 #define ID_UNALLOCATED(crm, op2) {          \
0333     Op0(3), Op1(0), CRn(0), CRm(crm), Op2(op2), \
0334     .access = pvm_access_id_aarch64,        \
0335 }
0336
0337 /* Mark the specified system register as Read-As-Zero/Write-Ignored */
0338 #define RAZ_WI(REG) { SYS_DESC(REG), .access = pvm_access_raz_wi }
0339
0340 /* Mark the specified system register as not being handled in hyp. */
0341 #define HOST_HANDLED(REG) { SYS_DESC(REG), .access = NULL }
0342
0343 /*
0344  * Architected system registers.
0345  * Important: Must be sorted ascending by Op0, Op1, CRn, CRm, Op2
0346  *
0347  * NOTE: Anything not explicitly listed here is *restricted by default*, i.e.,
0348  * it will lead to injecting an exception into the guest.
0349  */
0350 static const struct sys_reg_desc pvm_sys_reg_descs[] = {
0351     /* Cache maintenance by set/way operations are restricted. */
0352
0353     /* Debug and Trace Registers are restricted. */
0354
0355     /* AArch64 mappings of the AArch32 ID registers */
0356     /* CRm=1 */
0357     AARCH32(SYS_ID_PFR0_EL1),
0358     AARCH32(SYS_ID_PFR1_EL1),
0359     AARCH32(SYS_ID_DFR0_EL1),
0360     AARCH32(SYS_ID_AFR0_EL1),
0361     AARCH32(SYS_ID_MMFR0_EL1),
0362     AARCH32(SYS_ID_MMFR1_EL1),
0363     AARCH32(SYS_ID_MMFR2_EL1),
0364     AARCH32(SYS_ID_MMFR3_EL1),
0365
0366     /* CRm=2 */
0367     AARCH32(SYS_ID_ISAR0_EL1),
0368     AARCH32(SYS_ID_ISAR1_EL1),
0369     AARCH32(SYS_ID_ISAR2_EL1),
0370     AARCH32(SYS_ID_ISAR3_EL1),
0371     AARCH32(SYS_ID_ISAR4_EL1),
0372     AARCH32(SYS_ID_ISAR5_EL1),
0373     AARCH32(SYS_ID_MMFR4_EL1),
0374     AARCH32(SYS_ID_ISAR6_EL1),
0375
0376     /* CRm=3 */
0377     AARCH32(SYS_MVFR0_EL1),
0378     AARCH32(SYS_MVFR1_EL1),
0379     AARCH32(SYS_MVFR2_EL1),
0380     ID_UNALLOCATED(3,3),
0381     AARCH32(SYS_ID_PFR2_EL1),
0382     AARCH32(SYS_ID_DFR1_EL1),
0383     AARCH32(SYS_ID_MMFR5_EL1),
0384     ID_UNALLOCATED(3,7),
0385
0386     /* AArch64 ID registers */
0387     /* CRm=4 */
0388     AARCH64(SYS_ID_AA64PFR0_EL1),
0389     AARCH64(SYS_ID_AA64PFR1_EL1),
0390     ID_UNALLOCATED(4,2),
0391     ID_UNALLOCATED(4,3),
0392     AARCH64(SYS_ID_AA64ZFR0_EL1),
0393     ID_UNALLOCATED(4,5),
0394     ID_UNALLOCATED(4,6),
0395     ID_UNALLOCATED(4,7),
0396     AARCH64(SYS_ID_AA64DFR0_EL1),
0397     AARCH64(SYS_ID_AA64DFR1_EL1),
0398     ID_UNALLOCATED(5,2),
0399     ID_UNALLOCATED(5,3),
0400     AARCH64(SYS_ID_AA64AFR0_EL1),
0401     AARCH64(SYS_ID_AA64AFR1_EL1),
0402     ID_UNALLOCATED(5,6),
0403     ID_UNALLOCATED(5,7),
0404     AARCH64(SYS_ID_AA64ISAR0_EL1),
0405     AARCH64(SYS_ID_AA64ISAR1_EL1),
0406     AARCH64(SYS_ID_AA64ISAR2_EL1),
0407     ID_UNALLOCATED(6,3),
0408     ID_UNALLOCATED(6,4),
0409     ID_UNALLOCATED(6,5),
0410     ID_UNALLOCATED(6,6),
0411     ID_UNALLOCATED(6,7),
0412     AARCH64(SYS_ID_AA64MMFR0_EL1),
0413     AARCH64(SYS_ID_AA64MMFR1_EL1),
0414     AARCH64(SYS_ID_AA64MMFR2_EL1),
0415     ID_UNALLOCATED(7,3),
0416     ID_UNALLOCATED(7,4),
0417     ID_UNALLOCATED(7,5),
0418     ID_UNALLOCATED(7,6),
0419     ID_UNALLOCATED(7,7),
0420
0421     /* Scalable Vector Registers are restricted. */
0422
0423     RAZ_WI(SYS_ERRIDR_EL1),
0424     RAZ_WI(SYS_ERRSELR_EL1),
0425     RAZ_WI(SYS_ERXFR_EL1),
0426     RAZ_WI(SYS_ERXCTLR_EL1),
0427     RAZ_WI(SYS_ERXSTATUS_EL1),
0428     RAZ_WI(SYS_ERXADDR_EL1),
0429     RAZ_WI(SYS_ERXMISC0_EL1),
0430     RAZ_WI(SYS_ERXMISC1_EL1),
0431
0432     /* Performance Monitoring Registers are restricted. */
0433
0434     /* Limited Ordering Regions Registers are restricted. */
0435
0436     HOST_HANDLED(SYS_ICC_SGI1R_EL1),
0437     HOST_HANDLED(SYS_ICC_ASGI1R_EL1),
0438     HOST_HANDLED(SYS_ICC_SGI0R_EL1),
0439     { SYS_DESC(SYS_ICC_SRE_EL1), .access = pvm_gic_read_sre, },
0440
0441     HOST_HANDLED(SYS_CCSIDR_EL1),
0442     HOST_HANDLED(SYS_CLIDR_EL1),
0443     HOST_HANDLED(SYS_CSSELR_EL1),
0444     HOST_HANDLED(SYS_CTR_EL0),
0445
0446     /* Performance Monitoring Registers are restricted. */
0447
0448     /* Activity Monitoring Registers are restricted. */
0449
0450     HOST_HANDLED(SYS_CNTP_TVAL_EL0),
0451     HOST_HANDLED(SYS_CNTP_CTL_EL0),
0452     HOST_HANDLED(SYS_CNTP_CVAL_EL0),
0453
0454     /* Performance Monitoring Registers are restricted. */
0455 };
0456
0457 /*
0458  * Checks that the sysreg table is unique and in-order.
0459  *
0460  * Returns 0 if the table is consistent, or 1 otherwise.
0461  */
0462 int kvm_check_pvm_sysreg_table(void)
0463 {
0464     unsigned int i;
0465
0466     for (i = 1; i < ARRAY_SIZE(pvm_sys_reg_descs); i++) {
0467         if (cmp_sys_reg(&pvm_sys_reg_descs[i-1], &pvm_sys_reg_descs[i]) >= 0)
0468             return 1;
0469     }
0470
0471     return 0;
0472 }
0473
0474 /*
0475  * Handler for protected VM MSR, MRS or System instruction execution.
0476  *
0477  * Returns true if the hypervisor has handled the exit, and control should go
0478  * back to the guest, or false if it hasn't, to be handled by the host.
0479  */
0480 bool kvm_handle_pvm_sysreg(struct kvm_vcpu *vcpu, u64 *exit_code)
0481 {
0482     const struct sys_reg_desc *r;
0483     struct sys_reg_params params;
0484     unsigned long esr = kvm_vcpu_get_esr(vcpu);
0485     int Rt = kvm_vcpu_sys_get_rt(vcpu);
0486
0487     params = esr_sys64_to_params(esr);
0488     params.regval = vcpu_get_reg(vcpu, Rt);
0489
0490     r = find_reg(&params, pvm_sys_reg_descs, ARRAY_SIZE(pvm_sys_reg_descs));
0491
0492     /* Undefined (RESTRICTED). */
0493     if (r == NULL) {
0494         inject_undef64(vcpu);
0495         return true;
0496     }
0497
0498     /* Handled by the host (HOST_HANDLED) */
0499     if (r->access == NULL)
0500         return false;
0501
0502     /* Handled by hyp: skip instruction if instructed to do so. */
0503     if (r->access(vcpu, &params, r))
0504         __kvm_skip_instr(vcpu);
0505
0506     if (!params.is_write)
0507         vcpu_set_reg(vcpu, Rt, params.regval);
0508
0509     return true;
0510 }
0511
0512 /*
0513  * Handler for protected VM restricted exceptions.
0514  *
0515  * Inject an undefined exception into the guest and return true to indicate that
0516  * the hypervisor has handled the exit, and control should go back to the guest.
0517  */
0518 bool kvm_handle_pvm_restricted(struct kvm_vcpu *vcpu, u64 *exit_code)
0519 {
0520     inject_undef64(vcpu);
0521     return true;
0522 }