arm/kernel/traps.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  *  linux/arch/arm/kernel/traps.c
0004  *
0005  *  Copyright (C) 1995-2009 Russell King
0006  *  Fragments that appear the same as linux/arch/i386/kernel/traps.c (C) Linus Torvalds
0007  *
0008  *  'traps.c' handles hardware exceptions after we have saved some state in
0009  *  'linux/arch/arm/lib/traps.S'.  Mostly a debugging aid, but will probably
0010  *  kill the offending process.
0011  */
0012 #include <linux/signal.h>
0013 #include <linux/personality.h>
0014 #include <linux/kallsyms.h>
0015 #include <linux/spinlock.h>
0016 #include <linux/uaccess.h>
0017 #include <linux/hardirq.h>
0018 #include <linux/kdebug.h>
0019 #include <linux/kprobes.h>
0020 #include <linux/module.h>
0021 #include <linux/kexec.h>
0022 #include <linux/bug.h>
0023 #include <linux/delay.h>
0024 #include <linux/init.h>
0025 #include <linux/sched/signal.h>
0026 #include <linux/sched/debug.h>
0027 #include <linux/sched/task_stack.h>
0028 #include <linux/irq.h>
0029
0030 #include <linux/atomic.h>
0031 #include <asm/cacheflush.h>
0032 #include <asm/exception.h>
0033 #include <asm/spectre.h>
0034 #include <asm/unistd.h>
0035 #include <asm/traps.h>
0036 #include <asm/ptrace.h>
0037 #include <asm/unwind.h>
0038 #include <asm/tls.h>
0039 #include <asm/stacktrace.h>
0040 #include <asm/system_misc.h>
0041 #include <asm/opcodes.h>
0042
0043
0044 static const char *handler[]= {
0045     "prefetch abort",
0046     "data abort",
0047     "address exception",
0048     "interrupt",
0049     "undefined instruction",
0050 };
0051
0052 void *vectors_page;
0053
0054 #ifdef CONFIG_DEBUG_USER
0055 unsigned int user_debug;
0056
0057 static int __init user_debug_setup(char *str)
0058 {
0059     get_option(&str, &user_debug);
0060     return 1;
0061 }
0062 __setup("user_debug=", user_debug_setup);
0063 #endif
0064
0065 void dump_backtrace_entry(unsigned long where, unsigned long from,
0066               unsigned long frame, const char *loglvl)
0067 {
0068     unsigned long end = frame + 4 + sizeof(struct pt_regs);
0069
0070     if (IS_ENABLED(CONFIG_UNWINDER_FRAME_POINTER) &&
0071         IS_ENABLED(CONFIG_CC_IS_GCC) &&
0072         end > ALIGN(frame, THREAD_SIZE)) {
0073         /*
0074          * If we are walking past the end of the stack, it may be due
0075          * to the fact that we are on an IRQ or overflow stack. In this
0076          * case, we can load the address of the other stack from the
0077          * frame record.
0078          */
0079         frame = ((unsigned long *)frame)[-2] - 4;
0080         end = frame + 4 + sizeof(struct pt_regs);
0081     }
0082
0083 #ifndef CONFIG_KALLSYMS
0084     printk("%sFunction entered at [<%08lx>] from [<%08lx>]\n",
0085         loglvl, where, from);
0086 #elif defined CONFIG_BACKTRACE_VERBOSE
0087     printk("%s[<%08lx>] (%ps) from [<%08lx>] (%pS)\n",
0088         loglvl, where, (void *)where, from, (void *)from);
0089 #else
0090     printk("%s %ps from %pS\n", loglvl, (void *)where, (void *)from);
0091 #endif
0092
0093     if (in_entry_text(from) && end <= ALIGN(frame, THREAD_SIZE))
0094         dump_mem(loglvl, "Exception stack", frame + 4, end);
0095 }
0096
0097 void dump_backtrace_stm(u32 *stack, u32 instruction, const char *loglvl)
0098 {
0099     char str[80], *p;
0100     unsigned int x;
0101     int reg;
0102
0103     for (reg = 10, x = 0, p = str; reg >= 0; reg--) {
0104         if (instruction & BIT(reg)) {
0105             p += sprintf(p, " r%d:%08x", reg, *stack--);
0106             if (++x == 6) {
0107                 x = 0;
0108                 p = str;
0109                 printk("%s%s\n", loglvl, str);
0110             }
0111         }
0112     }
0113     if (p != str)
0114         printk("%s%s\n", loglvl, str);
0115 }
0116
0117 #ifndef CONFIG_ARM_UNWIND
0118 /*
0119  * Stack pointers should always be within the kernels view of
0120  * physical memory.  If it is not there, then we can't dump
0121  * out any information relating to the stack.
0122  */
0123 static int verify_stack(unsigned long sp)
0124 {
0125     if (sp < PAGE_OFFSET ||
0126         (!IS_ENABLED(CONFIG_VMAP_STACK) &&
0127          sp > (unsigned long)high_memory && high_memory != NULL))
0128         return -EFAULT;
0129
0130     return 0;
0131 }
0132 #endif
0133
0134 /*
0135  * Dump out the contents of some memory nicely...
0136  */
0137 void dump_mem(const char *lvl, const char *str, unsigned long bottom,
0138           unsigned long top)
0139 {
0140     unsigned long first;
0141     int i;
0142
0143     printk("%s%s(0x%08lx to 0x%08lx)\n", lvl, str, bottom, top);
0144
0145     for (first = bottom & ~31; first < top; first += 32) {
0146         unsigned long p;
0147         char str[sizeof(" 12345678") * 8 + 1];
0148
0149         memset(str, ' ', sizeof(str));
0150         str[sizeof(str) - 1] = '\0';
0151
0152         for (p = first, i = 0; i < 8 && p < top; i++, p += 4) {
0153             if (p >= bottom && p < top) {
0154                 unsigned long val;
0155                 if (!get_kernel_nofault(val, (unsigned long *)p))
0156                     sprintf(str + i * 9, " %08lx", val);
0157                 else
0158                     sprintf(str + i * 9, " ????????");
0159             }
0160         }
0161         printk("%s%04lx:%s\n", lvl, first & 0xffff, str);
0162     }
0163 }
0164
0165 static void dump_instr(const char *lvl, struct pt_regs *regs)
0166 {
0167     unsigned long addr = instruction_pointer(regs);
0168     const int thumb = thumb_mode(regs);
0169     const int width = thumb ? 4 : 8;
0170     char str[sizeof("00000000 ") * 5 + 2 + 1], *p = str;
0171     int i;
0172
0173     /*
0174      * Note that we now dump the code first, just in case the backtrace
0175      * kills us.
0176      */
0177
0178     for (i = -4; i < 1 + !!thumb; i++) {
0179         unsigned int val, bad;
0180
0181         if (!user_mode(regs)) {
0182             if (thumb) {
0183                 u16 val16;
0184                 bad = get_kernel_nofault(val16, &((u16 *)addr)[i]);
0185                 val = val16;
0186             } else {
0187                 bad = get_kernel_nofault(val, &((u32 *)addr)[i]);
0188             }
0189         } else {
0190             if (thumb)
0191                 bad = get_user(val, &((u16 *)addr)[i]);
0192             else
0193                 bad = get_user(val, &((u32 *)addr)[i]);
0194         }
0195
0196         if (!bad)
0197             p += sprintf(p, i == 0 ? "(%0*x) " : "%0*x ",
0198                     width, val);
0199         else {
0200             p += sprintf(p, "bad PC value");
0201             break;
0202         }
0203     }
0204     printk("%sCode: %s\n", lvl, str);
0205 }
0206
0207 #ifdef CONFIG_ARM_UNWIND
0208 static inline void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk,
0209                   const char *loglvl)
0210 {
0211     unwind_backtrace(regs, tsk, loglvl);
0212 }
0213 #else
0214 static void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk,
0215                const char *loglvl)
0216 {
0217     unsigned int fp, mode;
0218     int ok = 1;
0219
0220     printk("%sBacktrace: ", loglvl);
0221
0222     if (!tsk)
0223         tsk = current;
0224
0225     if (regs) {
0226         fp = frame_pointer(regs);
0227         mode = processor_mode(regs);
0228     } else if (tsk != current) {
0229         fp = thread_saved_fp(tsk);
0230         mode = 0x10;
0231     } else {
0232         asm("mov %0, fp" : "=r" (fp) : : "cc");
0233         mode = 0x10;
0234     }
0235
0236     if (!fp) {
0237         pr_cont("no frame pointer");
0238         ok = 0;
0239     } else if (verify_stack(fp)) {
0240         pr_cont("invalid frame pointer 0x%08x", fp);
0241         ok = 0;
0242     } else if (fp < (unsigned long)end_of_stack(tsk))
0243         pr_cont("frame pointer underflow");
0244     pr_cont("\n");
0245
0246     if (ok)
0247         c_backtrace(fp, mode, loglvl);
0248 }
0249 #endif
0250
0251 void show_stack(struct task_struct *tsk, unsigned long *sp, const char *loglvl)
0252 {
0253     dump_backtrace(NULL, tsk, loglvl);
0254     barrier();
0255 }
0256
0257 #ifdef CONFIG_PREEMPT
0258 #define S_PREEMPT " PREEMPT"
0259 #elif defined(CONFIG_PREEMPT_RT)
0260 #define S_PREEMPT " PREEMPT_RT"
0261 #else
0262 #define S_PREEMPT ""
0263 #endif
0264 #ifdef CONFIG_SMP
0265 #define S_SMP " SMP"
0266 #else
0267 #define S_SMP ""
0268 #endif
0269 #ifdef CONFIG_THUMB2_KERNEL
0270 #define S_ISA " THUMB2"
0271 #else
0272 #define S_ISA " ARM"
0273 #endif
0274
0275 static int __die(const char *str, int err, struct pt_regs *regs)
0276 {
0277     struct task_struct *tsk = current;
0278     static int die_counter;
0279     int ret;
0280
0281     pr_emerg("Internal error: %s: %x [#%d]" S_PREEMPT S_SMP S_ISA "\n",
0282              str, err, ++die_counter);
0283
0284     /* trap and error numbers are mostly meaningless on ARM */
0285     ret = notify_die(DIE_OOPS, str, regs, err, tsk->thread.trap_no, SIGSEGV);
0286     if (ret == NOTIFY_STOP)
0287         return 1;
0288
0289     print_modules();
0290     __show_regs(regs);
0291     __show_regs_alloc_free(regs);
0292     pr_emerg("Process %.*s (pid: %d, stack limit = 0x%p)\n",
0293          TASK_COMM_LEN, tsk->comm, task_pid_nr(tsk), end_of_stack(tsk));
0294
0295     if (!user_mode(regs) || in_interrupt()) {
0296         dump_mem(KERN_EMERG, "Stack: ", regs->ARM_sp,
0297              ALIGN(regs->ARM_sp - THREAD_SIZE, THREAD_ALIGN)
0298              + THREAD_SIZE);
0299         dump_backtrace(regs, tsk, KERN_EMERG);
0300         dump_instr(KERN_EMERG, regs);
0301     }
0302
0303     return 0;
0304 }
0305
0306 static arch_spinlock_t die_lock = __ARCH_SPIN_LOCK_UNLOCKED;
0307 static int die_owner = -1;
0308 static unsigned int die_nest_count;
0309
0310 static unsigned long oops_begin(void)
0311 {
0312     int cpu;
0313     unsigned long flags;
0314
0315     oops_enter();
0316
0317     /* racy, but better than risking deadlock. */
0318     raw_local_irq_save(flags);
0319     cpu = smp_processor_id();
0320     if (!arch_spin_trylock(&die_lock)) {
0321         if (cpu == die_owner)
0322             /* nested oops. should stop eventually */;
0323         else
0324             arch_spin_lock(&die_lock);
0325     }
0326     die_nest_count++;
0327     die_owner = cpu;
0328     console_verbose();
0329     bust_spinlocks(1);
0330     return flags;
0331 }
0332
0333 static void oops_end(unsigned long flags, struct pt_regs *regs, int signr)
0334 {
0335     if (regs && kexec_should_crash(current))
0336         crash_kexec(regs);
0337
0338     bust_spinlocks(0);
0339     die_owner = -1;
0340     add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE);
0341     die_nest_count--;
0342     if (!die_nest_count)
0343         /* Nest count reaches zero, release the lock. */
0344         arch_spin_unlock(&die_lock);
0345     raw_local_irq_restore(flags);
0346     oops_exit();
0347
0348     if (in_interrupt())
0349         panic("Fatal exception in interrupt");
0350     if (panic_on_oops)
0351         panic("Fatal exception");
0352     if (signr)
0353         make_task_dead(signr);
0354 }
0355
0356 /*
0357  * This function is protected against re-entrancy.
0358  */
0359 void die(const char *str, struct pt_regs *regs, int err)
0360 {
0361     enum bug_trap_type bug_type = BUG_TRAP_TYPE_NONE;
0362     unsigned long flags = oops_begin();
0363     int sig = SIGSEGV;
0364
0365     if (!user_mode(regs))
0366         bug_type = report_bug(regs->ARM_pc, regs);
0367     if (bug_type != BUG_TRAP_TYPE_NONE)
0368         str = "Oops - BUG";
0369
0370     if (__die(str, err, regs))
0371         sig = 0;
0372
0373     oops_end(flags, regs, sig);
0374 }
0375
0376 void arm_notify_die(const char *str, struct pt_regs *regs,
0377         int signo, int si_code, void __user *addr,
0378         unsigned long err, unsigned long trap)
0379 {
0380     if (user_mode(regs)) {
0381         current->thread.error_code = err;
0382         current->thread.trap_no = trap;
0383
0384         force_sig_fault(signo, si_code, addr);
0385     } else {
0386         die(str, regs, err);
0387     }
0388 }
0389
0390 #ifdef CONFIG_GENERIC_BUG
0391
0392 int is_valid_bugaddr(unsigned long pc)
0393 {
0394 #ifdef CONFIG_THUMB2_KERNEL
0395     u16 bkpt;
0396     u16 insn = __opcode_to_mem_thumb16(BUG_INSTR_VALUE);
0397 #else
0398     u32 bkpt;
0399     u32 insn = __opcode_to_mem_arm(BUG_INSTR_VALUE);
0400 #endif
0401
0402     if (get_kernel_nofault(bkpt, (void *)pc))
0403         return 0;
0404
0405     return bkpt == insn;
0406 }
0407
0408 #endif
0409
0410 static LIST_HEAD(undef_hook);
0411 static DEFINE_RAW_SPINLOCK(undef_lock);
0412
0413 void register_undef_hook(struct undef_hook *hook)
0414 {
0415     unsigned long flags;
0416
0417     raw_spin_lock_irqsave(&undef_lock, flags);
0418     list_add(&hook->node, &undef_hook);
0419     raw_spin_unlock_irqrestore(&undef_lock, flags);
0420 }
0421
0422 void unregister_undef_hook(struct undef_hook *hook)
0423 {
0424     unsigned long flags;
0425
0426     raw_spin_lock_irqsave(&undef_lock, flags);
0427     list_del(&hook->node);
0428     raw_spin_unlock_irqrestore(&undef_lock, flags);
0429 }
0430
0431 static nokprobe_inline
0432 int call_undef_hook(struct pt_regs *regs, unsigned int instr)
0433 {
0434     struct undef_hook *hook;
0435     unsigned long flags;
0436     int (*fn)(struct pt_regs *regs, unsigned int instr) = NULL;
0437
0438     raw_spin_lock_irqsave(&undef_lock, flags);
0439     list_for_each_entry(hook, &undef_hook, node)
0440         if ((instr & hook->instr_mask) == hook->instr_val &&
0441             (regs->ARM_cpsr & hook->cpsr_mask) == hook->cpsr_val)
0442             fn = hook->fn;
0443     raw_spin_unlock_irqrestore(&undef_lock, flags);
0444
0445     return fn ? fn(regs, instr) : 1;
0446 }
0447
0448 asmlinkage void do_undefinstr(struct pt_regs *regs)
0449 {
0450     unsigned int instr;
0451     void __user *pc;
0452
0453     pc = (void __user *)instruction_pointer(regs);
0454
0455     if (processor_mode(regs) == SVC_MODE) {
0456 #ifdef CONFIG_THUMB2_KERNEL
0457         if (thumb_mode(regs)) {
0458             instr = __mem_to_opcode_thumb16(((u16 *)pc)[0]);
0459             if (is_wide_instruction(instr)) {
0460                 u16 inst2;
0461                 inst2 = __mem_to_opcode_thumb16(((u16 *)pc)[1]);
0462                 instr = __opcode_thumb32_compose(instr, inst2);
0463             }
0464         } else
0465 #endif
0466             instr = __mem_to_opcode_arm(*(u32 *) pc);
0467     } else if (thumb_mode(regs)) {
0468         if (get_user(instr, (u16 __user *)pc))
0469             goto die_sig;
0470         instr = __mem_to_opcode_thumb16(instr);
0471         if (is_wide_instruction(instr)) {
0472             unsigned int instr2;
0473             if (get_user(instr2, (u16 __user *)pc+1))
0474                 goto die_sig;
0475             instr2 = __mem_to_opcode_thumb16(instr2);
0476             instr = __opcode_thumb32_compose(instr, instr2);
0477         }
0478     } else {
0479         if (get_user(instr, (u32 __user *)pc))
0480             goto die_sig;
0481         instr = __mem_to_opcode_arm(instr);
0482     }
0483
0484     if (call_undef_hook(regs, instr) == 0)
0485         return;
0486
0487 die_sig:
0488 #ifdef CONFIG_DEBUG_USER
0489     if (user_debug & UDBG_UNDEFINED) {
0490         pr_info("%s (%d): undefined instruction: pc=%p\n",
0491             current->comm, task_pid_nr(current), pc);
0492         __show_regs(regs);
0493         dump_instr(KERN_INFO, regs);
0494     }
0495 #endif
0496     arm_notify_die("Oops - undefined instruction", regs,
0497                SIGILL, ILL_ILLOPC, pc, 0, 6);
0498 }
0499 NOKPROBE_SYMBOL(do_undefinstr)
0500
0501 /*
0502  * Handle FIQ similarly to NMI on x86 systems.
0503  *
0504  * The runtime environment for NMIs is extremely restrictive
0505  * (NMIs can pre-empt critical sections meaning almost all locking is
0506  * forbidden) meaning this default FIQ handling must only be used in
0507  * circumstances where non-maskability improves robustness, such as
0508  * watchdog or debug logic.
0509  *
0510  * This handler is not appropriate for general purpose use in drivers
0511  * platform code and can be overrideen using set_fiq_handler.
0512  */
0513 asmlinkage void __exception_irq_entry handle_fiq_as_nmi(struct pt_regs *regs)
0514 {
0515     struct pt_regs *old_regs = set_irq_regs(regs);
0516
0517     nmi_enter();
0518
0519     /* nop. FIQ handlers for special arch/arm features can be added here. */
0520
0521     nmi_exit();
0522
0523     set_irq_regs(old_regs);
0524 }
0525
0526 /*
0527  * bad_mode handles the impossible case in the vectors.  If you see one of
0528  * these, then it's extremely serious, and could mean you have buggy hardware.
0529  * It never returns, and never tries to sync.  We hope that we can at least
0530  * dump out some state information...
0531  */
0532 asmlinkage void bad_mode(struct pt_regs *regs, int reason)
0533 {
0534     console_verbose();
0535
0536     pr_crit("Bad mode in %s handler detected\n", handler[reason]);
0537
0538     die("Oops - bad mode", regs, 0);
0539     local_irq_disable();
0540     panic("bad mode");
0541 }
0542
0543 static int bad_syscall(int n, struct pt_regs *regs)
0544 {
0545     if ((current->personality & PER_MASK) != PER_LINUX) {
0546         send_sig(SIGSEGV, current, 1);
0547         return regs->ARM_r0;
0548     }
0549
0550 #ifdef CONFIG_DEBUG_USER
0551     if (user_debug & UDBG_SYSCALL) {
0552         pr_err("[%d] %s: obsolete system call %08x.\n",
0553             task_pid_nr(current), current->comm, n);
0554         dump_instr(KERN_ERR, regs);
0555     }
0556 #endif
0557
0558     arm_notify_die("Oops - bad syscall", regs, SIGILL, ILL_ILLTRP,
0559                (void __user *)instruction_pointer(regs) -
0560              (thumb_mode(regs) ? 2 : 4),
0561                n, 0);
0562
0563     return regs->ARM_r0;
0564 }
0565
0566 static inline int
0567 __do_cache_op(unsigned long start, unsigned long end)
0568 {
0569     int ret;
0570
0571     do {
0572         unsigned long chunk = min(PAGE_SIZE, end - start);
0573
0574         if (fatal_signal_pending(current))
0575             return 0;
0576
0577         ret = flush_icache_user_range(start, start + chunk);
0578         if (ret)
0579             return ret;
0580
0581         cond_resched();
0582         start += chunk;
0583     } while (start < end);
0584
0585     return 0;
0586 }
0587
0588 static inline int
0589 do_cache_op(unsigned long start, unsigned long end, int flags)
0590 {
0591     if (end < start || flags)
0592         return -EINVAL;
0593
0594     if (!access_ok((void __user *)start, end - start))
0595         return -EFAULT;
0596
0597     return __do_cache_op(start, end);
0598 }
0599
0600 /*
0601  * Handle all unrecognised system calls.
0602  *  0x9f0000 - 0x9fffff are some more esoteric system calls
0603  */
0604 #define NR(x) ((__ARM_NR_##x) - __ARM_NR_BASE)
0605 asmlinkage int arm_syscall(int no, struct pt_regs *regs)
0606 {
0607     if ((no >> 16) != (__ARM_NR_BASE>> 16))
0608         return bad_syscall(no, regs);
0609
0610     switch (no & 0xffff) {
0611     case 0: /* branch through 0 */
0612         arm_notify_die("branch through zero", regs,
0613                    SIGSEGV, SEGV_MAPERR, NULL, 0, 0);
0614         return 0;
0615
0616     case NR(breakpoint): /* SWI BREAK_POINT */
0617         regs->ARM_pc -= thumb_mode(regs) ? 2 : 4;
0618         ptrace_break(regs);
0619         return regs->ARM_r0;
0620
0621     /*
0622      * Flush a region from virtual address 'r0' to virtual address 'r1'
0623      * _exclusive_.  There is no alignment requirement on either address;
0624      * user space does not need to know the hardware cache layout.
0625      *
0626      * r2 contains flags.  It should ALWAYS be passed as ZERO until it
0627      * is defined to be something else.  For now we ignore it, but may
0628      * the fires of hell burn in your belly if you break this rule. ;)
0629      *
0630      * (at a later date, we may want to allow this call to not flush
0631      * various aspects of the cache.  Passing '0' will guarantee that
0632      * everything necessary gets flushed to maintain consistency in
0633      * the specified region).
0634      */
0635     case NR(cacheflush):
0636         return do_cache_op(regs->ARM_r0, regs->ARM_r1, regs->ARM_r2);
0637
0638     case NR(usr26):
0639         if (!(elf_hwcap & HWCAP_26BIT))
0640             break;
0641         regs->ARM_cpsr &= ~MODE32_BIT;
0642         return regs->ARM_r0;
0643
0644     case NR(usr32):
0645         if (!(elf_hwcap & HWCAP_26BIT))
0646             break;
0647         regs->ARM_cpsr |= MODE32_BIT;
0648         return regs->ARM_r0;
0649
0650     case NR(set_tls):
0651         set_tls(regs->ARM_r0);
0652         return 0;
0653
0654     case NR(get_tls):
0655         return current_thread_info()->tp_value[0];
0656
0657     default:
0658         /* Calls 9f00xx..9f07ff are defined to return -ENOSYS
0659            if not implemented, rather than raising SIGILL.  This
0660            way the calling program can gracefully determine whether
0661            a feature is supported.  */
0662         if ((no & 0xffff) <= 0x7ff)
0663             return -ENOSYS;
0664         break;
0665     }
0666 #ifdef CONFIG_DEBUG_USER
0667     /*
0668      * experience shows that these seem to indicate that
0669      * something catastrophic has happened
0670      */
0671     if (user_debug & UDBG_SYSCALL) {
0672         pr_err("[%d] %s: arm syscall %d\n",
0673                task_pid_nr(current), current->comm, no);
0674         dump_instr(KERN_ERR, regs);
0675         if (user_mode(regs)) {
0676             __show_regs(regs);
0677             c_backtrace(frame_pointer(regs), processor_mode(regs), KERN_ERR);
0678         }
0679     }
0680 #endif
0681     arm_notify_die("Oops - bad syscall(2)", regs, SIGILL, ILL_ILLTRP,
0682                (void __user *)instruction_pointer(regs) -
0683              (thumb_mode(regs) ? 2 : 4),
0684                no, 0);
0685     return 0;
0686 }
0687
0688 #ifdef CONFIG_TLS_REG_EMUL
0689
0690 /*
0691  * We might be running on an ARMv6+ processor which should have the TLS
0692  * register but for some reason we can't use it, or maybe an SMP system
0693  * using a pre-ARMv6 processor (there are apparently a few prototypes like
0694  * that in existence) and therefore access to that register must be
0695  * emulated.
0696  */
0697
0698 static int get_tp_trap(struct pt_regs *regs, unsigned int instr)
0699 {
0700     int reg = (instr >> 12) & 15;
0701     if (reg == 15)
0702         return 1;
0703     regs->uregs[reg] = current_thread_info()->tp_value[0];
0704     regs->ARM_pc += 4;
0705     return 0;
0706 }
0707
0708 static struct undef_hook arm_mrc_hook = {
0709     .instr_mask = 0x0fff0fff,
0710     .instr_val  = 0x0e1d0f70,
0711     .cpsr_mask  = PSR_T_BIT,
0712     .cpsr_val   = 0,
0713     .fn     = get_tp_trap,
0714 };
0715
0716 static int __init arm_mrc_hook_init(void)
0717 {
0718     register_undef_hook(&arm_mrc_hook);
0719     return 0;
0720 }
0721
0722 late_initcall(arm_mrc_hook_init);
0723
0724 #endif
0725
0726 /*
0727  * A data abort trap was taken, but we did not handle the instruction.
0728  * Try to abort the user program, or panic if it was the kernel.
0729  */
0730 asmlinkage void
0731 baddataabort(int code, unsigned long instr, struct pt_regs *regs)
0732 {
0733     unsigned long addr = instruction_pointer(regs);
0734
0735 #ifdef CONFIG_DEBUG_USER
0736     if (user_debug & UDBG_BADABORT) {
0737         pr_err("8<--- cut here ---\n");
0738         pr_err("[%d] %s: bad data abort: code %d instr 0x%08lx\n",
0739                task_pid_nr(current), current->comm, code, instr);
0740         dump_instr(KERN_ERR, regs);
0741         show_pte(KERN_ERR, current->mm, addr);
0742     }
0743 #endif
0744
0745     arm_notify_die("unknown data abort code", regs,
0746                SIGILL, ILL_ILLOPC, (void __user *)addr, instr, 0);
0747 }
0748
0749 void __readwrite_bug(const char *fn)
0750 {
0751     pr_err("%s called, but not implemented\n", fn);
0752     BUG();
0753 }
0754 EXPORT_SYMBOL(__readwrite_bug);
0755
0756 void __pte_error(const char *file, int line, pte_t pte)
0757 {
0758     pr_err("%s:%d: bad pte %08llx.\n", file, line, (long long)pte_val(pte));
0759 }
0760
0761 void __pmd_error(const char *file, int line, pmd_t pmd)
0762 {
0763     pr_err("%s:%d: bad pmd %08llx.\n", file, line, (long long)pmd_val(pmd));
0764 }
0765
0766 void __pgd_error(const char *file, int line, pgd_t pgd)
0767 {
0768     pr_err("%s:%d: bad pgd %08llx.\n", file, line, (long long)pgd_val(pgd));
0769 }
0770
0771 asmlinkage void __div0(void)
0772 {
0773     pr_err("Division by zero in kernel.\n");
0774     dump_stack();
0775 }
0776 EXPORT_SYMBOL(__div0);
0777
0778 void abort(void)
0779 {
0780     BUG();
0781
0782     /* if that doesn't kill us, halt */
0783     panic("Oops failed to kill thread");
0784 }
0785
0786 #ifdef CONFIG_KUSER_HELPERS
0787 static void __init kuser_init(void *vectors)
0788 {
0789     extern char __kuser_helper_start[], __kuser_helper_end[];
0790     int kuser_sz = __kuser_helper_end - __kuser_helper_start;
0791
0792     memcpy(vectors + 0x1000 - kuser_sz, __kuser_helper_start, kuser_sz);
0793
0794     /*
0795      * vectors + 0xfe0 = __kuser_get_tls
0796      * vectors + 0xfe8 = hardware TLS instruction at 0xffff0fe8
0797      */
0798     if (tls_emu || has_tls_reg)
0799         memcpy(vectors + 0xfe0, vectors + 0xfe8, 4);
0800 }
0801 #else
0802 static inline void __init kuser_init(void *vectors)
0803 {
0804 }
0805 #endif
0806
0807 #ifndef CONFIG_CPU_V7M
0808 static void copy_from_lma(void *vma, void *lma_start, void *lma_end)
0809 {
0810     memcpy(vma, lma_start, lma_end - lma_start);
0811 }
0812
0813 static void flush_vectors(void *vma, size_t offset, size_t size)
0814 {
0815     unsigned long start = (unsigned long)vma + offset;
0816     unsigned long end = start + size;
0817
0818     flush_icache_range(start, end);
0819 }
0820
0821 #ifdef CONFIG_HARDEN_BRANCH_HISTORY
0822 int spectre_bhb_update_vectors(unsigned int method)
0823 {
0824     extern char __vectors_bhb_bpiall_start[], __vectors_bhb_bpiall_end[];
0825     extern char __vectors_bhb_loop8_start[], __vectors_bhb_loop8_end[];
0826     void *vec_start, *vec_end;
0827
0828     if (system_state >= SYSTEM_FREEING_INITMEM) {
0829         pr_err("CPU%u: Spectre BHB workaround too late - system vulnerable\n",
0830                smp_processor_id());
0831         return SPECTRE_VULNERABLE;
0832     }
0833
0834     switch (method) {
0835     case SPECTRE_V2_METHOD_LOOP8:
0836         vec_start = __vectors_bhb_loop8_start;
0837         vec_end = __vectors_bhb_loop8_end;
0838         break;
0839
0840     case SPECTRE_V2_METHOD_BPIALL:
0841         vec_start = __vectors_bhb_bpiall_start;
0842         vec_end = __vectors_bhb_bpiall_end;
0843         break;
0844
0845     default:
0846         pr_err("CPU%u: unknown Spectre BHB state %d\n",
0847                smp_processor_id(), method);
0848         return SPECTRE_VULNERABLE;
0849     }
0850
0851     copy_from_lma(vectors_page, vec_start, vec_end);
0852     flush_vectors(vectors_page, 0, vec_end - vec_start);
0853
0854     return SPECTRE_MITIGATED;
0855 }
0856 #endif
0857
0858 void __init early_trap_init(void *vectors_base)
0859 {
0860     extern char __stubs_start[], __stubs_end[];
0861     extern char __vectors_start[], __vectors_end[];
0862     unsigned i;
0863
0864     vectors_page = vectors_base;
0865
0866     /*
0867      * Poison the vectors page with an undefined instruction.  This
0868      * instruction is chosen to be undefined for both ARM and Thumb
0869      * ISAs.  The Thumb version is an undefined instruction with a
0870      * branch back to the undefined instruction.
0871      */
0872     for (i = 0; i < PAGE_SIZE / sizeof(u32); i++)
0873         ((u32 *)vectors_base)[i] = 0xe7fddef1;
0874
0875     /*
0876      * Copy the vectors, stubs and kuser helpers (in entry-armv.S)
0877      * into the vector page, mapped at 0xffff0000, and ensure these
0878      * are visible to the instruction stream.
0879      */
0880     copy_from_lma(vectors_base, __vectors_start, __vectors_end);
0881     copy_from_lma(vectors_base + 0x1000, __stubs_start, __stubs_end);
0882
0883     kuser_init(vectors_base);
0884
0885     flush_vectors(vectors_base, 0, PAGE_SIZE * 2);
0886 }
0887 #else /* ifndef CONFIG_CPU_V7M */
0888 void __init early_trap_init(void *vectors_base)
0889 {
0890     /*
0891      * on V7-M there is no need to copy the vector table to a dedicated
0892      * memory area. The address is configurable and so a table in the kernel
0893      * image can be used.
0894      */
0895 }
0896 #endif
0897
0898 #ifdef CONFIG_VMAP_STACK
0899
0900 DECLARE_PER_CPU(u8 *, irq_stack_ptr);
0901
0902 asmlinkage DEFINE_PER_CPU(u8 *, overflow_stack_ptr);
0903
0904 static int __init allocate_overflow_stacks(void)
0905 {
0906     u8 *stack;
0907     int cpu;
0908
0909     for_each_possible_cpu(cpu) {
0910         stack = (u8 *)__get_free_page(GFP_KERNEL);
0911         if (WARN_ON(!stack))
0912             return -ENOMEM;
0913         per_cpu(overflow_stack_ptr, cpu) = &stack[OVERFLOW_STACK_SIZE];
0914     }
0915     return 0;
0916 }
0917 early_initcall(allocate_overflow_stacks);
0918
0919 asmlinkage void handle_bad_stack(struct pt_regs *regs)
0920 {
0921     unsigned long tsk_stk = (unsigned long)current->stack;
0922 #ifdef CONFIG_IRQSTACKS
0923     unsigned long irq_stk = (unsigned long)this_cpu_read(irq_stack_ptr);
0924 #endif
0925     unsigned long ovf_stk = (unsigned long)this_cpu_read(overflow_stack_ptr);
0926
0927     console_verbose();
0928     pr_emerg("Insufficient stack space to handle exception!");
0929
0930     pr_emerg("Task stack:     [0x%08lx..0x%08lx]\n",
0931          tsk_stk, tsk_stk + THREAD_SIZE);
0932 #ifdef CONFIG_IRQSTACKS
0933     pr_emerg("IRQ stack:      [0x%08lx..0x%08lx]\n",
0934          irq_stk - THREAD_SIZE, irq_stk);
0935 #endif
0936     pr_emerg("Overflow stack: [0x%08lx..0x%08lx]\n",
0937          ovf_stk - OVERFLOW_STACK_SIZE, ovf_stk);
0938
0939     die("kernel stack overflow", regs, 0);
0940 }
0941
0942 #ifndef CONFIG_ARM_LPAE
0943 /*
0944  * Normally, we rely on the logic in do_translation_fault() to update stale PMD
0945  * entries covering the vmalloc space in a task's page tables when it first
0946  * accesses the region in question. Unfortunately, this is not sufficient when
0947  * the task stack resides in the vmalloc region, as do_translation_fault() is a
0948  * C function that needs a stack to run.
0949  *
0950  * So we need to ensure that these PMD entries are up to date *before* the MM
0951  * switch. As we already have some logic in the MM switch path that takes care
0952  * of this, let's trigger it by bumping the counter every time the core vmalloc
0953  * code modifies a PMD entry in the vmalloc region. Use release semantics on
0954  * the store so that other CPUs observing the counter's new value are
0955  * guaranteed to see the updated page table entries as well.
0956  */
0957 void arch_sync_kernel_mappings(unsigned long start, unsigned long end)
0958 {
0959     if (start < VMALLOC_END && end > VMALLOC_START)
0960         atomic_inc_return_release(&init_mm.context.vmalloc_seq);
0961 }
0962 #endif
0963 #endif