arm64/kernel/sdei.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 // Copyright (C) 2017 Arm Ltd.
0003 #define pr_fmt(fmt) "sdei: " fmt
0004
0005 #include <linux/arm-smccc.h>
0006 #include <linux/arm_sdei.h>
0007 #include <linux/hardirq.h>
0008 #include <linux/irqflags.h>
0009 #include <linux/sched/task_stack.h>
0010 #include <linux/scs.h>
0011 #include <linux/uaccess.h>
0012
0013 #include <asm/alternative.h>
0014 #include <asm/exception.h>
0015 #include <asm/kprobes.h>
0016 #include <asm/mmu.h>
0017 #include <asm/ptrace.h>
0018 #include <asm/sections.h>
0019 #include <asm/stacktrace.h>
0020 #include <asm/sysreg.h>
0021 #include <asm/vmap_stack.h>
0022
0023 unsigned long sdei_exit_mode;
0024
0025 /*
0026  * VMAP'd stacks checking for stack overflow on exception using sp as a scratch
0027  * register, meaning SDEI has to switch to its own stack. We need two stacks as
0028  * a critical event may interrupt a normal event that has just taken a
0029  * synchronous exception, and is using sp as scratch register. For a critical
0030  * event interrupting a normal event, we can't reliably tell if we were on the
0031  * sdei stack.
0032  * For now, we allocate stacks when the driver is probed.
0033  */
0034 DECLARE_PER_CPU(unsigned long *, sdei_stack_normal_ptr);
0035 DECLARE_PER_CPU(unsigned long *, sdei_stack_critical_ptr);
0036
0037 #ifdef CONFIG_VMAP_STACK
0038 DEFINE_PER_CPU(unsigned long *, sdei_stack_normal_ptr);
0039 DEFINE_PER_CPU(unsigned long *, sdei_stack_critical_ptr);
0040 #endif
0041
0042 DECLARE_PER_CPU(unsigned long *, sdei_shadow_call_stack_normal_ptr);
0043 DECLARE_PER_CPU(unsigned long *, sdei_shadow_call_stack_critical_ptr);
0044
0045 #ifdef CONFIG_SHADOW_CALL_STACK
0046 DEFINE_PER_CPU(unsigned long *, sdei_shadow_call_stack_normal_ptr);
0047 DEFINE_PER_CPU(unsigned long *, sdei_shadow_call_stack_critical_ptr);
0048 #endif
0049
0050 static void _free_sdei_stack(unsigned long * __percpu *ptr, int cpu)
0051 {
0052     unsigned long *p;
0053
0054     p = per_cpu(*ptr, cpu);
0055     if (p) {
0056         per_cpu(*ptr, cpu) = NULL;
0057         vfree(p);
0058     }
0059 }
0060
0061 static void free_sdei_stacks(void)
0062 {
0063     int cpu;
0064
0065     if (!IS_ENABLED(CONFIG_VMAP_STACK))
0066         return;
0067
0068     for_each_possible_cpu(cpu) {
0069         _free_sdei_stack(&sdei_stack_normal_ptr, cpu);
0070         _free_sdei_stack(&sdei_stack_critical_ptr, cpu);
0071     }
0072 }
0073
0074 static int _init_sdei_stack(unsigned long * __percpu *ptr, int cpu)
0075 {
0076     unsigned long *p;
0077
0078     p = arch_alloc_vmap_stack(SDEI_STACK_SIZE, cpu_to_node(cpu));
0079     if (!p)
0080         return -ENOMEM;
0081     per_cpu(*ptr, cpu) = p;
0082
0083     return 0;
0084 }
0085
0086 static int init_sdei_stacks(void)
0087 {
0088     int cpu;
0089     int err = 0;
0090
0091     if (!IS_ENABLED(CONFIG_VMAP_STACK))
0092         return 0;
0093
0094     for_each_possible_cpu(cpu) {
0095         err = _init_sdei_stack(&sdei_stack_normal_ptr, cpu);
0096         if (err)
0097             break;
0098         err = _init_sdei_stack(&sdei_stack_critical_ptr, cpu);
0099         if (err)
0100             break;
0101     }
0102
0103     if (err)
0104         free_sdei_stacks();
0105
0106     return err;
0107 }
0108
0109 static void _free_sdei_scs(unsigned long * __percpu *ptr, int cpu)
0110 {
0111     void *s;
0112
0113     s = per_cpu(*ptr, cpu);
0114     if (s) {
0115         per_cpu(*ptr, cpu) = NULL;
0116         scs_free(s);
0117     }
0118 }
0119
0120 static void free_sdei_scs(void)
0121 {
0122     int cpu;
0123
0124     for_each_possible_cpu(cpu) {
0125         _free_sdei_scs(&sdei_shadow_call_stack_normal_ptr, cpu);
0126         _free_sdei_scs(&sdei_shadow_call_stack_critical_ptr, cpu);
0127     }
0128 }
0129
0130 static int _init_sdei_scs(unsigned long * __percpu *ptr, int cpu)
0131 {
0132     void *s;
0133
0134     s = scs_alloc(cpu_to_node(cpu));
0135     if (!s)
0136         return -ENOMEM;
0137     per_cpu(*ptr, cpu) = s;
0138
0139     return 0;
0140 }
0141
0142 static int init_sdei_scs(void)
0143 {
0144     int cpu;
0145     int err = 0;
0146
0147     if (!IS_ENABLED(CONFIG_SHADOW_CALL_STACK))
0148         return 0;
0149
0150     for_each_possible_cpu(cpu) {
0151         err = _init_sdei_scs(&sdei_shadow_call_stack_normal_ptr, cpu);
0152         if (err)
0153             break;
0154         err = _init_sdei_scs(&sdei_shadow_call_stack_critical_ptr, cpu);
0155         if (err)
0156             break;
0157     }
0158
0159     if (err)
0160         free_sdei_scs();
0161
0162     return err;
0163 }
0164
0165 static bool on_sdei_normal_stack(unsigned long sp, unsigned long size,
0166                  struct stack_info *info)
0167 {
0168     unsigned long low = (unsigned long)raw_cpu_read(sdei_stack_normal_ptr);
0169     unsigned long high = low + SDEI_STACK_SIZE;
0170
0171     return on_stack(sp, size, low, high, STACK_TYPE_SDEI_NORMAL, info);
0172 }
0173
0174 static bool on_sdei_critical_stack(unsigned long sp, unsigned long size,
0175                    struct stack_info *info)
0176 {
0177     unsigned long low = (unsigned long)raw_cpu_read(sdei_stack_critical_ptr);
0178     unsigned long high = low + SDEI_STACK_SIZE;
0179
0180     return on_stack(sp, size, low, high, STACK_TYPE_SDEI_CRITICAL, info);
0181 }
0182
0183 bool _on_sdei_stack(unsigned long sp, unsigned long size, struct stack_info *info)
0184 {
0185     if (!IS_ENABLED(CONFIG_VMAP_STACK))
0186         return false;
0187
0188     if (on_sdei_critical_stack(sp, size, info))
0189         return true;
0190
0191     if (on_sdei_normal_stack(sp, size, info))
0192         return true;
0193
0194     return false;
0195 }
0196
0197 unsigned long sdei_arch_get_entry_point(int conduit)
0198 {
0199     /*
0200      * SDEI works between adjacent exception levels. If we booted at EL1 we
0201      * assume a hypervisor is marshalling events. If we booted at EL2 and
0202      * dropped to EL1 because we don't support VHE, then we can't support
0203      * SDEI.
0204      */
0205     if (is_hyp_nvhe()) {
0206         pr_err("Not supported on this hardware/boot configuration\n");
0207         goto out_err;
0208     }
0209
0210     if (init_sdei_stacks())
0211         goto out_err;
0212
0213     if (init_sdei_scs())
0214         goto out_err_free_stacks;
0215
0216     sdei_exit_mode = (conduit == SMCCC_CONDUIT_HVC) ? SDEI_EXIT_HVC : SDEI_EXIT_SMC;
0217
0218 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
0219     if (arm64_kernel_unmapped_at_el0()) {
0220         unsigned long offset;
0221
0222         offset = (unsigned long)__sdei_asm_entry_trampoline -
0223              (unsigned long)__entry_tramp_text_start;
0224         return TRAMP_VALIAS + offset;
0225     } else
0226 #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
0227         return (unsigned long)__sdei_asm_handler;
0228
0229 out_err_free_stacks:
0230     free_sdei_stacks();
0231 out_err:
0232     return 0;
0233 }
0234
0235 /*
0236  * do_sdei_event() returns one of:
0237  *  SDEI_EV_HANDLED -  success, return to the interrupted context.
0238  *  SDEI_EV_FAILED  -  failure, return this error code to firmare.
0239  *  virtual-address -  success, return to this address.
0240  */
0241 unsigned long __kprobes do_sdei_event(struct pt_regs *regs,
0242                       struct sdei_registered_event *arg)
0243 {
0244     u32 mode;
0245     int i, err = 0;
0246     int clobbered_registers = 4;
0247     u64 elr = read_sysreg(elr_el1);
0248     u32 kernel_mode = read_sysreg(CurrentEL) | 1;   /* +SPSel */
0249     unsigned long vbar = read_sysreg(vbar_el1);
0250
0251     if (arm64_kernel_unmapped_at_el0())
0252         clobbered_registers++;
0253
0254     /* Retrieve the missing registers values */
0255     for (i = 0; i < clobbered_registers; i++) {
0256         /* from within the handler, this call always succeeds */
0257         sdei_api_event_context(i, &regs->regs[i]);
0258     }
0259
0260     err = sdei_event_handler(regs, arg);
0261     if (err)
0262         return SDEI_EV_FAILED;
0263
0264     if (elr != read_sysreg(elr_el1)) {
0265         /*
0266          * We took a synchronous exception from the SDEI handler.
0267          * This could deadlock, and if you interrupt KVM it will
0268          * hyp-panic instead.
0269          */
0270         pr_warn("unsafe: exception during handler\n");
0271     }
0272
0273     mode = regs->pstate & (PSR_MODE32_BIT | PSR_MODE_MASK);
0274
0275     /*
0276      * If we interrupted the kernel with interrupts masked, we always go
0277      * back to wherever we came from.
0278      */
0279     if (mode == kernel_mode && !interrupts_enabled(regs))
0280         return SDEI_EV_HANDLED;
0281
0282     /*
0283      * Otherwise, we pretend this was an IRQ. This lets user space tasks
0284      * receive signals before we return to them, and KVM to invoke it's
0285      * world switch to do the same.
0286      *
0287      * See DDI0487B.a Table D1-7 'Vector offsets from vector table base
0288      * address'.
0289      */
0290     if (mode == kernel_mode)
0291         return vbar + 0x280;
0292     else if (mode & PSR_MODE32_BIT)
0293         return vbar + 0x680;
0294
0295     return vbar + 0x480;
0296 }