arm/kernel/hyp-stub.S

0001 /* SPDX-License-Identifier: GPL-2.0-or-later */
0002 /*
0003  * Copyright (c) 2012 Linaro Limited.
0004  */
0005
0006 #include <linux/init.h>
0007 #include <linux/irqchip/arm-gic-v3.h>
0008 #include <linux/linkage.h>
0009 #include <asm/assembler.h>
0010 #include <asm/virt.h>
0011
0012 #ifndef ZIMAGE
0013 /*
0014  * For the kernel proper, we need to find out the CPU boot mode long after
0015  * boot, so we need to store it in a writable variable.
0016  *
0017  * This is not in .bss, because we set it sufficiently early that the boot-time
0018  * zeroing of .bss would clobber it.
0019  */
0020 .data
0021     .align  2
0022 ENTRY(__boot_cpu_mode)
0023     .long   0
0024 .text
0025
0026     /*
0027      * Save the primary CPU boot mode. Requires 2 scratch registers.
0028      */
0029     .macro  store_primary_cpu_mode  reg1, reg2
0030     mrs \reg1, cpsr
0031     and \reg1, \reg1, #MODE_MASK
0032     str_l   \reg1, __boot_cpu_mode, \reg2
0033     .endm
0034
0035     /*
0036      * Compare the current mode with the one saved on the primary CPU.
0037      * If they don't match, record that fact. The Z bit indicates
0038      * if there's a match or not.
0039      * Requires 2 additional scratch registers.
0040      */
0041     .macro  compare_cpu_mode_with_primary mode, reg1, reg2
0042     adr_l   \reg2, __boot_cpu_mode
0043     ldr \reg1, [\reg2]
0044     cmp \mode, \reg1        @ matches primary CPU boot mode?
0045     orrne   \reg1, \reg1, #BOOT_CPU_MODE_MISMATCH
0046     strne   \reg1, [\reg2]      @ record what happened and give up
0047     .endm
0048
0049 #else   /* ZIMAGE */
0050
0051     .macro  store_primary_cpu_mode  reg1:req, reg2:req
0052     .endm
0053
0054 /*
0055  * The zImage loader only runs on one CPU, so we don't bother with mult-CPU
0056  * consistency checking:
0057  */
0058     .macro  compare_cpu_mode_with_primary mode, reg1, reg2
0059     cmp \mode, \mode
0060     .endm
0061
0062 #endif /* ZIMAGE */
0063
0064 /*
0065  * Hypervisor stub installation functions.
0066  *
0067  * These must be called with the MMU and D-cache off.
0068  * They are not ABI compliant and are only intended to be called from the kernel
0069  * entry points in head.S.
0070  */
0071 @ Call this from the primary CPU
0072 ENTRY(__hyp_stub_install)
0073     store_primary_cpu_mode  r4, r5
0074 ENDPROC(__hyp_stub_install)
0075
0076     @ fall through...
0077
0078 @ Secondary CPUs should call here
0079 ENTRY(__hyp_stub_install_secondary)
0080     mrs r4, cpsr
0081     and r4, r4, #MODE_MASK
0082
0083     /*
0084      * If the secondary has booted with a different mode, give up
0085      * immediately.
0086      */
0087     compare_cpu_mode_with_primary   r4, r5, r6
0088     retne   lr
0089
0090     /*
0091      * Once we have given up on one CPU, we do not try to install the
0092      * stub hypervisor on the remaining ones: because the saved boot mode
0093      * is modified, it can't compare equal to the CPSR mode field any
0094      * more.
0095      *
0096      * Otherwise...
0097      */
0098
0099     cmp r4, #HYP_MODE
0100     retne   lr          @ give up if the CPU is not in HYP mode
0101
0102 /*
0103  * Configure HSCTLR to set correct exception endianness/instruction set
0104  * state etc.
0105  * Turn off all traps
0106  * Eventually, CPU-specific code might be needed -- assume not for now
0107  *
0108  * This code relies on the "eret" instruction to synchronize the
0109  * various coprocessor accesses. This is done when we switch to SVC
0110  * (see safe_svcmode_maskall).
0111  */
0112     @ Now install the hypervisor stub:
0113     W(adr)  r7, __hyp_stub_vectors
0114     mcr p15, 4, r7, c12, c0, 0  @ set hypervisor vector base (HVBAR)
0115
0116     @ Disable all traps, so we don't get any nasty surprise
0117     mov r7, #0
0118     mcr p15, 4, r7, c1, c1, 0   @ HCR
0119     mcr p15, 4, r7, c1, c1, 2   @ HCPTR
0120     mcr p15, 4, r7, c1, c1, 3   @ HSTR
0121
0122 THUMB(  orr r7, #(1 << 30)  )   @ HSCTLR.TE
0123 ARM_BE8(orr r7, r7, #(1 << 25))     @ HSCTLR.EE
0124     mcr p15, 4, r7, c1, c0, 0   @ HSCTLR
0125
0126     mrc p15, 4, r7, c1, c1, 1   @ HDCR
0127     and r7, #0x1f       @ Preserve HPMN
0128     mcr p15, 4, r7, c1, c1, 1   @ HDCR
0129
0130     @ Make sure NS-SVC is initialised appropriately
0131     mrc p15, 0, r7, c1, c0, 0   @ SCTLR
0132     orr r7, #(1 << 5)       @ CP15 barriers enabled
0133     bic r7, #(3 << 7)       @ Clear SED/ITD for v8 (RES0 for v7)
0134     bic r7, #(3 << 19)      @ WXN and UWXN disabled
0135     mcr p15, 0, r7, c1, c0, 0   @ SCTLR
0136
0137     mrc p15, 0, r7, c0, c0, 0   @ MIDR
0138     mcr p15, 4, r7, c0, c0, 0   @ VPIDR
0139
0140     mrc p15, 0, r7, c0, c0, 5   @ MPIDR
0141     mcr p15, 4, r7, c0, c0, 5   @ VMPIDR
0142
0143 #if !defined(ZIMAGE) && defined(CONFIG_ARM_ARCH_TIMER)
0144     @ make CNTP_* and CNTPCT accessible from PL1
0145     mrc p15, 0, r7, c0, c1, 1   @ ID_PFR1
0146     ubfx    r7, r7, #16, #4
0147     teq r7, #0
0148     beq 1f
0149     mrc p15, 4, r7, c14, c1, 0  @ CNTHCTL
0150     orr r7, r7, #3      @ PL1PCEN | PL1PCTEN
0151     mcr p15, 4, r7, c14, c1, 0  @ CNTHCTL
0152     mov r7, #0
0153     mcrr    p15, 4, r7, r7, c14 @ CNTVOFF
0154
0155     @ Disable virtual timer in case it was counting
0156     mrc p15, 0, r7, c14, c3, 1  @ CNTV_CTL
0157     bic r7, #1          @ Clear ENABLE
0158     mcr p15, 0, r7, c14, c3, 1  @ CNTV_CTL
0159 1:
0160 #endif
0161
0162 #ifdef CONFIG_ARM_GIC_V3
0163     @ Check whether GICv3 system registers are available
0164     mrc p15, 0, r7, c0, c1, 1   @ ID_PFR1
0165     ubfx    r7, r7, #28, #4
0166     teq r7, #0
0167     beq 2f
0168
0169     @ Enable system register accesses
0170     mrc p15, 4, r7, c12, c9, 5  @ ICC_HSRE
0171     orr r7, r7, #(ICC_SRE_EL2_ENABLE | ICC_SRE_EL2_SRE)
0172     mcr p15, 4, r7, c12, c9, 5  @ ICC_HSRE
0173     isb
0174
0175     @ SRE bit could be forced to 0 by firmware.
0176     @ Check whether it sticks before accessing any other sysreg
0177     mrc p15, 4, r7, c12, c9, 5  @ ICC_HSRE
0178     tst r7, #ICC_SRE_EL2_SRE
0179     beq 2f
0180     mov r7, #0
0181     mcr p15, 4, r7, c12, c11, 0 @ ICH_HCR
0182 2:
0183 #endif
0184
0185     bx  lr          @ The boot CPU mode is left in r4.
0186 ENDPROC(__hyp_stub_install_secondary)
0187
0188 __hyp_stub_do_trap:
0189 #ifdef ZIMAGE
0190     teq r0, #HVC_SET_VECTORS
0191     bne 1f
0192     /* Only the ZIMAGE stubs can change the HYP vectors */
0193     mcr p15, 4, r1, c12, c0, 0  @ set HVBAR
0194     b   __hyp_stub_exit
0195 #endif
0196
0197 1:  teq r0, #HVC_SOFT_RESTART
0198     bne 2f
0199     bx  r1
0200
0201 2:  ldr r0, =HVC_STUB_ERR
0202     __ERET
0203
0204 __hyp_stub_exit:
0205     mov r0, #0
0206     __ERET
0207 ENDPROC(__hyp_stub_do_trap)
0208
0209 /*
0210  * __hyp_set_vectors is only used when ZIMAGE must bounce between HYP
0211  * and SVC. For the kernel itself, the vectors are set once and for
0212  * all by the stubs.
0213  */
0214 ENTRY(__hyp_set_vectors)
0215     mov r1, r0
0216     mov r0, #HVC_SET_VECTORS
0217     __HVC(0)
0218     ret lr
0219 ENDPROC(__hyp_set_vectors)
0220
0221 ENTRY(__hyp_soft_restart)
0222     mov r1, r0
0223     mov r0, #HVC_SOFT_RESTART
0224     __HVC(0)
0225     ret lr
0226 ENDPROC(__hyp_soft_restart)
0227
0228 .align 5
0229 ENTRY(__hyp_stub_vectors)
0230 __hyp_stub_reset:   W(b)    .
0231 __hyp_stub_und:     W(b)    .
0232 __hyp_stub_svc:     W(b)    .
0233 __hyp_stub_pabort:  W(b)    .
0234 __hyp_stub_dabort:  W(b)    .
0235 __hyp_stub_trap:    W(b)    __hyp_stub_do_trap
0236 __hyp_stub_irq:     W(b)    .
0237 __hyp_stub_fiq:     W(b)    .
0238 ENDPROC(__hyp_stub_vectors)
0239