include/asm/kvm_asm.h

0001 /* SPDX-License-Identifier: GPL-2.0-only */
0002 /*
0003  * Copyright (C) 2012,2013 - ARM Ltd
0004  * Author: Marc Zyngier <marc.zyngier@arm.com>
0005  */
0006
0007 #ifndef __ARM_KVM_ASM_H__
0008 #define __ARM_KVM_ASM_H__
0009
0010 #include <asm/hyp_image.h>
0011 #include <asm/insn.h>
0012 #include <asm/virt.h>
0013
0014 #define ARM_EXIT_WITH_SERROR_BIT  31
0015 #define ARM_EXCEPTION_CODE(x)     ((x) & ~(1U << ARM_EXIT_WITH_SERROR_BIT))
0016 #define ARM_EXCEPTION_IS_TRAP(x)  (ARM_EXCEPTION_CODE((x)) == ARM_EXCEPTION_TRAP)
0017 #define ARM_SERROR_PENDING(x)     !!((x) & (1U << ARM_EXIT_WITH_SERROR_BIT))
0018
0019 #define ARM_EXCEPTION_IRQ     0
0020 #define ARM_EXCEPTION_EL1_SERROR  1
0021 #define ARM_EXCEPTION_TRAP    2
0022 #define ARM_EXCEPTION_IL      3
0023 /* The hyp-stub will return this for any kvm_call_hyp() call */
0024 #define ARM_EXCEPTION_HYP_GONE    HVC_STUB_ERR
0025
0026 #define kvm_arm_exception_type                  \
0027     {ARM_EXCEPTION_IRQ,     "IRQ"       },  \
0028     {ARM_EXCEPTION_EL1_SERROR,  "SERROR"    },  \
0029     {ARM_EXCEPTION_TRAP,        "TRAP"      },  \
0030     {ARM_EXCEPTION_HYP_GONE,    "HYP_GONE"  }
0031
0032 /*
0033  * Size of the HYP vectors preamble. kvm_patch_vector_branch() generates code
0034  * that jumps over this.
0035  */
0036 #define KVM_VECTOR_PREAMBLE (2 * AARCH64_INSN_SIZE)
0037
0038 #define KVM_HOST_SMCCC_ID(id)                       \
0039     ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL,             \
0040                ARM_SMCCC_SMC_64,                \
0041                ARM_SMCCC_OWNER_VENDOR_HYP,          \
0042                (id))
0043
0044 #define KVM_HOST_SMCCC_FUNC(name) KVM_HOST_SMCCC_ID(__KVM_HOST_SMCCC_FUNC_##name)
0045
0046 #define __KVM_HOST_SMCCC_FUNC___kvm_hyp_init            0
0047
0048 #ifndef __ASSEMBLY__
0049
0050 #include <linux/mm.h>
0051
0052 enum __kvm_host_smccc_func {
0053     /* Hypercalls available only prior to pKVM finalisation */
0054     /* __KVM_HOST_SMCCC_FUNC___kvm_hyp_init */
0055     __KVM_HOST_SMCCC_FUNC___kvm_get_mdcr_el2 = __KVM_HOST_SMCCC_FUNC___kvm_hyp_init + 1,
0056     __KVM_HOST_SMCCC_FUNC___pkvm_init,
0057     __KVM_HOST_SMCCC_FUNC___pkvm_create_private_mapping,
0058     __KVM_HOST_SMCCC_FUNC___pkvm_cpu_set_vector,
0059     __KVM_HOST_SMCCC_FUNC___kvm_enable_ssbs,
0060     __KVM_HOST_SMCCC_FUNC___vgic_v3_init_lrs,
0061     __KVM_HOST_SMCCC_FUNC___vgic_v3_get_gic_config,
0062     __KVM_HOST_SMCCC_FUNC___pkvm_prot_finalize,
0063
0064     /* Hypercalls available after pKVM finalisation */
0065     __KVM_HOST_SMCCC_FUNC___pkvm_host_share_hyp,
0066     __KVM_HOST_SMCCC_FUNC___pkvm_host_unshare_hyp,
0067     __KVM_HOST_SMCCC_FUNC___kvm_adjust_pc,
0068     __KVM_HOST_SMCCC_FUNC___kvm_vcpu_run,
0069     __KVM_HOST_SMCCC_FUNC___kvm_flush_vm_context,
0070     __KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid_ipa,
0071     __KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid,
0072     __KVM_HOST_SMCCC_FUNC___kvm_flush_cpu_context,
0073     __KVM_HOST_SMCCC_FUNC___kvm_timer_set_cntvoff,
0074     __KVM_HOST_SMCCC_FUNC___vgic_v3_read_vmcr,
0075     __KVM_HOST_SMCCC_FUNC___vgic_v3_write_vmcr,
0076     __KVM_HOST_SMCCC_FUNC___vgic_v3_save_aprs,
0077     __KVM_HOST_SMCCC_FUNC___vgic_v3_restore_aprs,
0078     __KVM_HOST_SMCCC_FUNC___pkvm_vcpu_init_traps,
0079 };
0080
0081 #define DECLARE_KVM_VHE_SYM(sym)    extern char sym[]
0082 #define DECLARE_KVM_NVHE_SYM(sym)   extern char kvm_nvhe_sym(sym)[]
0083
0084 /*
0085  * Define a pair of symbols sharing the same name but one defined in
0086  * VHE and the other in nVHE hyp implementations.
0087  */
0088 #define DECLARE_KVM_HYP_SYM(sym)        \
0089     DECLARE_KVM_VHE_SYM(sym);       \
0090     DECLARE_KVM_NVHE_SYM(sym)
0091
0092 #define DECLARE_KVM_VHE_PER_CPU(type, sym)  \
0093     DECLARE_PER_CPU(type, sym)
0094 #define DECLARE_KVM_NVHE_PER_CPU(type, sym) \
0095     DECLARE_PER_CPU(type, kvm_nvhe_sym(sym))
0096
0097 #define DECLARE_KVM_HYP_PER_CPU(type, sym)  \
0098     DECLARE_KVM_VHE_PER_CPU(type, sym); \
0099     DECLARE_KVM_NVHE_PER_CPU(type, sym)
0100
0101 /*
0102  * Compute pointer to a symbol defined in nVHE percpu region.
0103  * Returns NULL if percpu memory has not been allocated yet.
0104  */
0105 #define this_cpu_ptr_nvhe_sym(sym)  per_cpu_ptr_nvhe_sym(sym, smp_processor_id())
0106 #define per_cpu_ptr_nvhe_sym(sym, cpu)                      \
0107     ({                                  \
0108         unsigned long base, off;                    \
0109         base = kvm_arm_hyp_percpu_base[cpu];                \
0110         off = (unsigned long)&CHOOSE_NVHE_SYM(sym) -            \
0111               (unsigned long)&CHOOSE_NVHE_SYM(__per_cpu_start);     \
0112         base ? (typeof(CHOOSE_NVHE_SYM(sym))*)(base + off) : NULL;  \
0113     })
0114
0115 #if defined(__KVM_NVHE_HYPERVISOR__)
0116
0117 #define CHOOSE_NVHE_SYM(sym)    sym
0118 #define CHOOSE_HYP_SYM(sym) CHOOSE_NVHE_SYM(sym)
0119
0120 /* The nVHE hypervisor shouldn't even try to access VHE symbols */
0121 extern void *__nvhe_undefined_symbol;
0122 #define CHOOSE_VHE_SYM(sym)     __nvhe_undefined_symbol
0123 #define this_cpu_ptr_hyp_sym(sym)   (&__nvhe_undefined_symbol)
0124 #define per_cpu_ptr_hyp_sym(sym, cpu)   (&__nvhe_undefined_symbol)
0125
0126 #elif defined(__KVM_VHE_HYPERVISOR__)
0127
0128 #define CHOOSE_VHE_SYM(sym) sym
0129 #define CHOOSE_HYP_SYM(sym) CHOOSE_VHE_SYM(sym)
0130
0131 /* The VHE hypervisor shouldn't even try to access nVHE symbols */
0132 extern void *__vhe_undefined_symbol;
0133 #define CHOOSE_NVHE_SYM(sym)        __vhe_undefined_symbol
0134 #define this_cpu_ptr_hyp_sym(sym)   (&__vhe_undefined_symbol)
0135 #define per_cpu_ptr_hyp_sym(sym, cpu)   (&__vhe_undefined_symbol)
0136
0137 #else
0138
0139 /*
0140  * BIG FAT WARNINGS:
0141  *
0142  * - Don't be tempted to change the following is_kernel_in_hyp_mode()
0143  *   to has_vhe(). has_vhe() is implemented as a *final* capability,
0144  *   while this is used early at boot time, when the capabilities are
0145  *   not final yet....
0146  *
0147  * - Don't let the nVHE hypervisor have access to this, as it will
0148  *   pick the *wrong* symbol (yes, it runs at EL2...).
0149  */
0150 #define CHOOSE_HYP_SYM(sym)     (is_kernel_in_hyp_mode()    \
0151                        ? CHOOSE_VHE_SYM(sym)    \
0152                        : CHOOSE_NVHE_SYM(sym))
0153
0154 #define this_cpu_ptr_hyp_sym(sym)   (is_kernel_in_hyp_mode()    \
0155                        ? this_cpu_ptr(&sym)     \
0156                        : this_cpu_ptr_nvhe_sym(sym))
0157
0158 #define per_cpu_ptr_hyp_sym(sym, cpu)   (is_kernel_in_hyp_mode()    \
0159                        ? per_cpu_ptr(&sym, cpu) \
0160                        : per_cpu_ptr_nvhe_sym(sym, cpu))
0161
0162 #define CHOOSE_VHE_SYM(sym) sym
0163 #define CHOOSE_NVHE_SYM(sym)    kvm_nvhe_sym(sym)
0164
0165 #endif
0166
0167 struct kvm_nvhe_init_params {
0168     unsigned long mair_el2;
0169     unsigned long tcr_el2;
0170     unsigned long tpidr_el2;
0171     unsigned long stack_hyp_va;
0172     unsigned long stack_pa;
0173     phys_addr_t pgd_pa;
0174     unsigned long hcr_el2;
0175     unsigned long vttbr;
0176     unsigned long vtcr;
0177 };
0178
0179 /*
0180  * Used by the host in EL1 to dump the nVHE hypervisor backtrace on
0181  * hyp_panic() in non-protected mode.
0182  *
0183  * @stack_base:                 hyp VA of the hyp_stack base.
0184  * @overflow_stack_base:        hyp VA of the hyp_overflow_stack base.
0185  * @fp:                         hyp FP where the backtrace begins.
0186  * @pc:                         hyp PC where the backtrace begins.
0187  */
0188 struct kvm_nvhe_stacktrace_info {
0189     unsigned long stack_base;
0190     unsigned long overflow_stack_base;
0191     unsigned long fp;
0192     unsigned long pc;
0193 };
0194
0195 /* Translate a kernel address @ptr into its equivalent linear mapping */
0196 #define kvm_ksym_ref(ptr)                       \
0197     ({                              \
0198         void *val = (ptr);                  \
0199         if (!is_kernel_in_hyp_mode())               \
0200             val = lm_alias((ptr));              \
0201         val;                            \
0202      })
0203 #define kvm_ksym_ref_nvhe(sym)  kvm_ksym_ref(kvm_nvhe_sym(sym))
0204
0205 struct kvm;
0206 struct kvm_vcpu;
0207 struct kvm_s2_mmu;
0208
0209 DECLARE_KVM_NVHE_SYM(__kvm_hyp_init);
0210 DECLARE_KVM_HYP_SYM(__kvm_hyp_vector);
0211 #define __kvm_hyp_init      CHOOSE_NVHE_SYM(__kvm_hyp_init)
0212 #define __kvm_hyp_vector    CHOOSE_HYP_SYM(__kvm_hyp_vector)
0213
0214 extern unsigned long kvm_arm_hyp_percpu_base[NR_CPUS];
0215 DECLARE_KVM_NVHE_SYM(__per_cpu_start);
0216 DECLARE_KVM_NVHE_SYM(__per_cpu_end);
0217
0218 DECLARE_KVM_HYP_SYM(__bp_harden_hyp_vecs);
0219 #define __bp_harden_hyp_vecs    CHOOSE_HYP_SYM(__bp_harden_hyp_vecs)
0220
0221 extern void __kvm_flush_vm_context(void);
0222 extern void __kvm_flush_cpu_context(struct kvm_s2_mmu *mmu);
0223 extern void __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu, phys_addr_t ipa,
0224                      int level);
0225 extern void __kvm_tlb_flush_vmid(struct kvm_s2_mmu *mmu);
0226
0227 extern void __kvm_timer_set_cntvoff(u64 cntvoff);
0228
0229 extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu);
0230
0231 extern void __kvm_adjust_pc(struct kvm_vcpu *vcpu);
0232
0233 extern u64 __vgic_v3_get_gic_config(void);
0234 extern u64 __vgic_v3_read_vmcr(void);
0235 extern void __vgic_v3_write_vmcr(u32 vmcr);
0236 extern void __vgic_v3_init_lrs(void);
0237
0238 extern u64 __kvm_get_mdcr_el2(void);
0239
0240 #define __KVM_EXTABLE(from, to)                     \
0241     "   .pushsection    __kvm_ex_table, \"a\"\n"        \
0242     "   .align      3\n"                    \
0243     "   .long       (" #from " - .), (" #to " - .)\n"   \
0244     "   .popsection\n"
0245
0246
0247 #define __kvm_at(at_op, addr)                       \
0248 ( {                                     \
0249     int __kvm_at_err = 0;                       \
0250     u64 spsr, elr;                          \
0251     asm volatile(                           \
0252     "   mrs %1, spsr_el2\n"                 \
0253     "   mrs %2, elr_el2\n"                  \
0254     "1: at  "at_op", %3\n"                  \
0255     "   isb\n"                          \
0256     "   b   9f\n"                       \
0257     "2: msr spsr_el2, %1\n"                 \
0258     "   msr elr_el2, %2\n"                  \
0259     "   mov %w0, %4\n"                  \
0260     "9:\n"                              \
0261     __KVM_EXTABLE(1b, 2b)                       \
0262     : "+r" (__kvm_at_err), "=&r" (spsr), "=&r" (elr)        \
0263     : "r" (addr), "i" (-EFAULT));                   \
0264     __kvm_at_err;                           \
0265 } )
0266
0267
0268 #else /* __ASSEMBLY__ */
0269
0270 .macro get_host_ctxt reg, tmp
0271     adr_this_cpu \reg, kvm_host_data, \tmp
0272     add \reg, \reg, #HOST_DATA_CONTEXT
0273 .endm
0274
0275 .macro get_vcpu_ptr vcpu, ctxt
0276     get_host_ctxt \ctxt, \vcpu
0277     ldr \vcpu, [\ctxt, #HOST_CONTEXT_VCPU]
0278 .endm
0279
0280 .macro get_loaded_vcpu vcpu, ctxt
0281     adr_this_cpu \ctxt, kvm_hyp_ctxt, \vcpu
0282     ldr \vcpu, [\ctxt, #HOST_CONTEXT_VCPU]
0283 .endm
0284
0285 .macro set_loaded_vcpu vcpu, ctxt, tmp
0286     adr_this_cpu \ctxt, kvm_hyp_ctxt, \tmp
0287     str \vcpu, [\ctxt, #HOST_CONTEXT_VCPU]
0288 .endm
0289
0290 /*
0291  * KVM extable for unexpected exceptions.
0292  * Create a struct kvm_exception_table_entry output to a section that can be
0293  * mapped by EL2. The table is not sorted.
0294  *
0295  * The caller must ensure:
0296  * x18 has the hypervisor value to allow any Shadow-Call-Stack instrumented
0297  * code to write to it, and that SPSR_EL2 and ELR_EL2 are restored by the fixup.
0298  */
0299 .macro  _kvm_extable, from, to
0300     .pushsection    __kvm_ex_table, "a"
0301     .align      3
0302     .long       (\from - .), (\to - .)
0303     .popsection
0304 .endm
0305
0306 #define CPU_XREG_OFFSET(x)  (CPU_USER_PT_REGS + 8*x)
0307 #define CPU_LR_OFFSET       CPU_XREG_OFFSET(30)
0308 #define CPU_SP_EL0_OFFSET   (CPU_LR_OFFSET + 8)
0309
0310 /*
0311  * We treat x18 as callee-saved as the host may use it as a platform
0312  * register (e.g. for shadow call stack).
0313  */
0314 .macro save_callee_saved_regs ctxt
0315     str x18,      [\ctxt, #CPU_XREG_OFFSET(18)]
0316     stp x19, x20, [\ctxt, #CPU_XREG_OFFSET(19)]
0317     stp x21, x22, [\ctxt, #CPU_XREG_OFFSET(21)]
0318     stp x23, x24, [\ctxt, #CPU_XREG_OFFSET(23)]
0319     stp x25, x26, [\ctxt, #CPU_XREG_OFFSET(25)]
0320     stp x27, x28, [\ctxt, #CPU_XREG_OFFSET(27)]
0321     stp x29, lr,  [\ctxt, #CPU_XREG_OFFSET(29)]
0322 .endm
0323
0324 .macro restore_callee_saved_regs ctxt
0325     // We require \ctxt is not x18-x28
0326     ldr x18,      [\ctxt, #CPU_XREG_OFFSET(18)]
0327     ldp x19, x20, [\ctxt, #CPU_XREG_OFFSET(19)]
0328     ldp x21, x22, [\ctxt, #CPU_XREG_OFFSET(21)]
0329     ldp x23, x24, [\ctxt, #CPU_XREG_OFFSET(23)]
0330     ldp x25, x26, [\ctxt, #CPU_XREG_OFFSET(25)]
0331     ldp x27, x28, [\ctxt, #CPU_XREG_OFFSET(27)]
0332     ldp x29, lr,  [\ctxt, #CPU_XREG_OFFSET(29)]
0333 .endm
0334
0335 .macro save_sp_el0 ctxt, tmp
0336     mrs \tmp,   sp_el0
0337     str \tmp,   [\ctxt, #CPU_SP_EL0_OFFSET]
0338 .endm
0339
0340 .macro restore_sp_el0 ctxt, tmp
0341     ldr \tmp,     [\ctxt, #CPU_SP_EL0_OFFSET]
0342     msr sp_el0, \tmp
0343 .endm
0344
0345 #endif
0346
0347 #endif /* __ARM_KVM_ASM_H__ */