hyp/nvhe/switch.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Copyright (C) 2015 - ARM Ltd
0004  * Author: Marc Zyngier <marc.zyngier@arm.com>
0005  */
0006
0007 #include <hyp/switch.h>
0008 #include <hyp/sysreg-sr.h>
0009
0010 #include <linux/arm-smccc.h>
0011 #include <linux/kvm_host.h>
0012 #include <linux/types.h>
0013 #include <linux/jump_label.h>
0014 #include <uapi/linux/psci.h>
0015
0016 #include <kvm/arm_psci.h>
0017
0018 #include <asm/barrier.h>
0019 #include <asm/cpufeature.h>
0020 #include <asm/kprobes.h>
0021 #include <asm/kvm_asm.h>
0022 #include <asm/kvm_emulate.h>
0023 #include <asm/kvm_hyp.h>
0024 #include <asm/kvm_mmu.h>
0025 #include <asm/fpsimd.h>
0026 #include <asm/debug-monitors.h>
0027 #include <asm/processor.h>
0028
0029 #include <nvhe/fixed_config.h>
0030 #include <nvhe/mem_protect.h>
0031
0032 /* Non-VHE specific context */
0033 DEFINE_PER_CPU(struct kvm_host_data, kvm_host_data);
0034 DEFINE_PER_CPU(struct kvm_cpu_context, kvm_hyp_ctxt);
0035 DEFINE_PER_CPU(unsigned long, kvm_hyp_vector);
0036
0037 extern void kvm_nvhe_prepare_backtrace(unsigned long fp, unsigned long pc);
0038
0039 static void __activate_traps(struct kvm_vcpu *vcpu)
0040 {
0041     u64 val;
0042
0043     ___activate_traps(vcpu);
0044     __activate_traps_common(vcpu);
0045
0046     val = vcpu->arch.cptr_el2;
0047     val |= CPTR_EL2_TTA | CPTR_EL2_TAM;
0048     if (!guest_owns_fp_regs(vcpu)) {
0049         val |= CPTR_EL2_TFP | CPTR_EL2_TZ;
0050         __activate_traps_fpsimd32(vcpu);
0051     }
0052     if (cpus_have_final_cap(ARM64_SME))
0053         val |= CPTR_EL2_TSM;
0054
0055     write_sysreg(val, cptr_el2);
0056     write_sysreg(__this_cpu_read(kvm_hyp_vector), vbar_el2);
0057
0058     if (cpus_have_final_cap(ARM64_SME)) {
0059         val = read_sysreg_s(SYS_HFGRTR_EL2);
0060         val &= ~(HFGxTR_EL2_nTPIDR2_EL0_MASK |
0061              HFGxTR_EL2_nSMPRI_EL1_MASK);
0062         write_sysreg_s(val, SYS_HFGRTR_EL2);
0063
0064         val = read_sysreg_s(SYS_HFGWTR_EL2);
0065         val &= ~(HFGxTR_EL2_nTPIDR2_EL0_MASK |
0066              HFGxTR_EL2_nSMPRI_EL1_MASK);
0067         write_sysreg_s(val, SYS_HFGWTR_EL2);
0068     }
0069
0070     if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT)) {
0071         struct kvm_cpu_context *ctxt = &vcpu->arch.ctxt;
0072
0073         isb();
0074         /*
0075          * At this stage, and thanks to the above isb(), S2 is
0076          * configured and enabled. We can now restore the guest's S1
0077          * configuration: SCTLR, and only then TCR.
0078          */
0079         write_sysreg_el1(ctxt_sys_reg(ctxt, SCTLR_EL1), SYS_SCTLR);
0080         isb();
0081         write_sysreg_el1(ctxt_sys_reg(ctxt, TCR_EL1),   SYS_TCR);
0082     }
0083 }
0084
0085 static void __deactivate_traps(struct kvm_vcpu *vcpu)
0086 {
0087     extern char __kvm_hyp_host_vector[];
0088     u64 cptr;
0089
0090     ___deactivate_traps(vcpu);
0091
0092     if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT)) {
0093         u64 val;
0094
0095         /*
0096          * Set the TCR and SCTLR registers in the exact opposite
0097          * sequence as __activate_traps (first prevent walks,
0098          * then force the MMU on). A generous sprinkling of isb()
0099          * ensure that things happen in this exact order.
0100          */
0101         val = read_sysreg_el1(SYS_TCR);
0102         write_sysreg_el1(val | TCR_EPD1_MASK | TCR_EPD0_MASK, SYS_TCR);
0103         isb();
0104         val = read_sysreg_el1(SYS_SCTLR);
0105         write_sysreg_el1(val | SCTLR_ELx_M, SYS_SCTLR);
0106         isb();
0107     }
0108
0109     __deactivate_traps_common(vcpu);
0110
0111     write_sysreg(this_cpu_ptr(&kvm_init_params)->hcr_el2, hcr_el2);
0112
0113     if (cpus_have_final_cap(ARM64_SME)) {
0114         u64 val;
0115
0116         val = read_sysreg_s(SYS_HFGRTR_EL2);
0117         val |= HFGxTR_EL2_nTPIDR2_EL0_MASK |
0118             HFGxTR_EL2_nSMPRI_EL1_MASK;
0119         write_sysreg_s(val, SYS_HFGRTR_EL2);
0120
0121         val = read_sysreg_s(SYS_HFGWTR_EL2);
0122         val |= HFGxTR_EL2_nTPIDR2_EL0_MASK |
0123             HFGxTR_EL2_nSMPRI_EL1_MASK;
0124         write_sysreg_s(val, SYS_HFGWTR_EL2);
0125     }
0126
0127     cptr = CPTR_EL2_DEFAULT;
0128     if (vcpu_has_sve(vcpu) && (vcpu->arch.fp_state == FP_STATE_GUEST_OWNED))
0129         cptr |= CPTR_EL2_TZ;
0130     if (cpus_have_final_cap(ARM64_SME))
0131         cptr &= ~CPTR_EL2_TSM;
0132
0133     write_sysreg(cptr, cptr_el2);
0134     write_sysreg(__kvm_hyp_host_vector, vbar_el2);
0135 }
0136
0137 /* Save VGICv3 state on non-VHE systems */
0138 static void __hyp_vgic_save_state(struct kvm_vcpu *vcpu)
0139 {
0140     if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) {
0141         __vgic_v3_save_state(&vcpu->arch.vgic_cpu.vgic_v3);
0142         __vgic_v3_deactivate_traps(&vcpu->arch.vgic_cpu.vgic_v3);
0143     }
0144 }
0145
0146 /* Restore VGICv3 state on non_VEH systems */
0147 static void __hyp_vgic_restore_state(struct kvm_vcpu *vcpu)
0148 {
0149     if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) {
0150         __vgic_v3_activate_traps(&vcpu->arch.vgic_cpu.vgic_v3);
0151         __vgic_v3_restore_state(&vcpu->arch.vgic_cpu.vgic_v3);
0152     }
0153 }
0154
0155 /*
0156  * Disable host events, enable guest events
0157  */
0158 #ifdef CONFIG_HW_PERF_EVENTS
0159 static bool __pmu_switch_to_guest(struct kvm_vcpu *vcpu)
0160 {
0161     struct kvm_pmu_events *pmu = &vcpu->arch.pmu.events;
0162
0163     if (pmu->events_host)
0164         write_sysreg(pmu->events_host, pmcntenclr_el0);
0165
0166     if (pmu->events_guest)
0167         write_sysreg(pmu->events_guest, pmcntenset_el0);
0168
0169     return (pmu->events_host || pmu->events_guest);
0170 }
0171
0172 /*
0173  * Disable guest events, enable host events
0174  */
0175 static void __pmu_switch_to_host(struct kvm_vcpu *vcpu)
0176 {
0177     struct kvm_pmu_events *pmu = &vcpu->arch.pmu.events;
0178
0179     if (pmu->events_guest)
0180         write_sysreg(pmu->events_guest, pmcntenclr_el0);
0181
0182     if (pmu->events_host)
0183         write_sysreg(pmu->events_host, pmcntenset_el0);
0184 }
0185 #else
0186 #define __pmu_switch_to_guest(v)    ({ false; })
0187 #define __pmu_switch_to_host(v)     do {} while (0)
0188 #endif
0189
0190 /*
0191  * Handler for protected VM MSR, MRS or System instruction execution in AArch64.
0192  *
0193  * Returns true if the hypervisor has handled the exit, and control should go
0194  * back to the guest, or false if it hasn't.
0195  */
0196 static bool kvm_handle_pvm_sys64(struct kvm_vcpu *vcpu, u64 *exit_code)
0197 {
0198     /*
0199      * Make sure we handle the exit for workarounds and ptrauth
0200      * before the pKVM handling, as the latter could decide to
0201      * UNDEF.
0202      */
0203     return (kvm_hyp_handle_sysreg(vcpu, exit_code) ||
0204         kvm_handle_pvm_sysreg(vcpu, exit_code));
0205 }
0206
0207 static const exit_handler_fn hyp_exit_handlers[] = {
0208     [0 ... ESR_ELx_EC_MAX]      = NULL,
0209     [ESR_ELx_EC_CP15_32]        = kvm_hyp_handle_cp15_32,
0210     [ESR_ELx_EC_SYS64]      = kvm_hyp_handle_sysreg,
0211     [ESR_ELx_EC_SVE]        = kvm_hyp_handle_fpsimd,
0212     [ESR_ELx_EC_FP_ASIMD]       = kvm_hyp_handle_fpsimd,
0213     [ESR_ELx_EC_IABT_LOW]       = kvm_hyp_handle_iabt_low,
0214     [ESR_ELx_EC_DABT_LOW]       = kvm_hyp_handle_dabt_low,
0215     [ESR_ELx_EC_PAC]        = kvm_hyp_handle_ptrauth,
0216 };
0217
0218 static const exit_handler_fn pvm_exit_handlers[] = {
0219     [0 ... ESR_ELx_EC_MAX]      = NULL,
0220     [ESR_ELx_EC_SYS64]      = kvm_handle_pvm_sys64,
0221     [ESR_ELx_EC_SVE]        = kvm_handle_pvm_restricted,
0222     [ESR_ELx_EC_FP_ASIMD]       = kvm_hyp_handle_fpsimd,
0223     [ESR_ELx_EC_IABT_LOW]       = kvm_hyp_handle_iabt_low,
0224     [ESR_ELx_EC_DABT_LOW]       = kvm_hyp_handle_dabt_low,
0225     [ESR_ELx_EC_PAC]        = kvm_hyp_handle_ptrauth,
0226 };
0227
0228 static const exit_handler_fn *kvm_get_exit_handler_array(struct kvm_vcpu *vcpu)
0229 {
0230     if (unlikely(kvm_vm_is_protected(kern_hyp_va(vcpu->kvm))))
0231         return pvm_exit_handlers;
0232
0233     return hyp_exit_handlers;
0234 }
0235
0236 /*
0237  * Some guests (e.g., protected VMs) are not be allowed to run in AArch32.
0238  * The ARMv8 architecture does not give the hypervisor a mechanism to prevent a
0239  * guest from dropping to AArch32 EL0 if implemented by the CPU. If the
0240  * hypervisor spots a guest in such a state ensure it is handled, and don't
0241  * trust the host to spot or fix it.  The check below is based on the one in
0242  * kvm_arch_vcpu_ioctl_run().
0243  *
0244  * Returns false if the guest ran in AArch32 when it shouldn't have, and
0245  * thus should exit to the host, or true if a the guest run loop can continue.
0246  */
0247 static void early_exit_filter(struct kvm_vcpu *vcpu, u64 *exit_code)
0248 {
0249     struct kvm *kvm = kern_hyp_va(vcpu->kvm);
0250
0251     if (kvm_vm_is_protected(kvm) && vcpu_mode_is_32bit(vcpu)) {
0252         /*
0253          * As we have caught the guest red-handed, decide that it isn't
0254          * fit for purpose anymore by making the vcpu invalid. The VMM
0255          * can try and fix it by re-initializing the vcpu with
0256          * KVM_ARM_VCPU_INIT, however, this is likely not possible for
0257          * protected VMs.
0258          */
0259         vcpu->arch.target = -1;
0260         *exit_code &= BIT(ARM_EXIT_WITH_SERROR_BIT);
0261         *exit_code |= ARM_EXCEPTION_IL;
0262     }
0263 }
0264
0265 /* Switch to the guest for legacy non-VHE systems */
0266 int __kvm_vcpu_run(struct kvm_vcpu *vcpu)
0267 {
0268     struct kvm_cpu_context *host_ctxt;
0269     struct kvm_cpu_context *guest_ctxt;
0270     struct kvm_s2_mmu *mmu;
0271     bool pmu_switch_needed;
0272     u64 exit_code;
0273
0274     /*
0275      * Having IRQs masked via PMR when entering the guest means the GIC
0276      * will not signal the CPU of interrupts of lower priority, and the
0277      * only way to get out will be via guest exceptions.
0278      * Naturally, we want to avoid this.
0279      */
0280     if (system_uses_irq_prio_masking()) {
0281         gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);
0282         pmr_sync();
0283     }
0284
0285     host_ctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt;
0286     host_ctxt->__hyp_running_vcpu = vcpu;
0287     guest_ctxt = &vcpu->arch.ctxt;
0288
0289     pmu_switch_needed = __pmu_switch_to_guest(vcpu);
0290
0291     __sysreg_save_state_nvhe(host_ctxt);
0292     /*
0293      * We must flush and disable the SPE buffer for nVHE, as
0294      * the translation regime(EL1&0) is going to be loaded with
0295      * that of the guest. And we must do this before we change the
0296      * translation regime to EL2 (via MDCR_EL2_E2PB == 0) and
0297      * before we load guest Stage1.
0298      */
0299     __debug_save_host_buffers_nvhe(vcpu);
0300
0301     __kvm_adjust_pc(vcpu);
0302
0303     /*
0304      * We must restore the 32-bit state before the sysregs, thanks
0305      * to erratum #852523 (Cortex-A57) or #853709 (Cortex-A72).
0306      *
0307      * Also, and in order to be able to deal with erratum #1319537 (A57)
0308      * and #1319367 (A72), we must ensure that all VM-related sysreg are
0309      * restored before we enable S2 translation.
0310      */
0311     __sysreg32_restore_state(vcpu);
0312     __sysreg_restore_state_nvhe(guest_ctxt);
0313
0314     mmu = kern_hyp_va(vcpu->arch.hw_mmu);
0315     __load_stage2(mmu, kern_hyp_va(mmu->arch));
0316     __activate_traps(vcpu);
0317
0318     __hyp_vgic_restore_state(vcpu);
0319     __timer_enable_traps(vcpu);
0320
0321     __debug_switch_to_guest(vcpu);
0322
0323     do {
0324         /* Jump in the fire! */
0325         exit_code = __guest_enter(vcpu);
0326
0327         /* And we're baaack! */
0328     } while (fixup_guest_exit(vcpu, &exit_code));
0329
0330     __sysreg_save_state_nvhe(guest_ctxt);
0331     __sysreg32_save_state(vcpu);
0332     __timer_disable_traps(vcpu);
0333     __hyp_vgic_save_state(vcpu);
0334
0335     __deactivate_traps(vcpu);
0336     __load_host_stage2();
0337
0338     __sysreg_restore_state_nvhe(host_ctxt);
0339
0340     if (vcpu->arch.fp_state == FP_STATE_GUEST_OWNED)
0341         __fpsimd_save_fpexc32(vcpu);
0342
0343     __debug_switch_to_host(vcpu);
0344     /*
0345      * This must come after restoring the host sysregs, since a non-VHE
0346      * system may enable SPE here and make use of the TTBRs.
0347      */
0348     __debug_restore_host_buffers_nvhe(vcpu);
0349
0350     if (pmu_switch_needed)
0351         __pmu_switch_to_host(vcpu);
0352
0353     /* Returning to host will clear PSR.I, remask PMR if needed */
0354     if (system_uses_irq_prio_masking())
0355         gic_write_pmr(GIC_PRIO_IRQOFF);
0356
0357     host_ctxt->__hyp_running_vcpu = NULL;
0358
0359     return exit_code;
0360 }
0361
0362 asmlinkage void __noreturn hyp_panic(void)
0363 {
0364     u64 spsr = read_sysreg_el2(SYS_SPSR);
0365     u64 elr = read_sysreg_el2(SYS_ELR);
0366     u64 par = read_sysreg_par();
0367     struct kvm_cpu_context *host_ctxt;
0368     struct kvm_vcpu *vcpu;
0369
0370     host_ctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt;
0371     vcpu = host_ctxt->__hyp_running_vcpu;
0372
0373     if (vcpu) {
0374         __timer_disable_traps(vcpu);
0375         __deactivate_traps(vcpu);
0376         __load_host_stage2();
0377         __sysreg_restore_state_nvhe(host_ctxt);
0378     }
0379
0380     /* Prepare to dump kvm nvhe hyp stacktrace */
0381     kvm_nvhe_prepare_backtrace((unsigned long)__builtin_frame_address(0),
0382                    _THIS_IP_);
0383
0384     __hyp_do_panic(host_ctxt, spsr, elr, par);
0385     unreachable();
0386 }
0387
0388 asmlinkage void __noreturn hyp_panic_bad_stack(void)
0389 {
0390     hyp_panic();
0391 }
0392
0393 asmlinkage void kvm_unexpected_el2_exception(void)
0394 {
0395     __kvm_unexpected_el2_exception();
0396 }