probes/kprobes/core.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * arch/arm/kernel/kprobes.c
0004  *
0005  * Kprobes on ARM
0006  *
0007  * Abhishek Sagar <sagar.abhishek@gmail.com>
0008  * Copyright (C) 2006, 2007 Motorola Inc.
0009  *
0010  * Nicolas Pitre <nico@marvell.com>
0011  * Copyright (C) 2007 Marvell Ltd.
0012  */
0013
0014 #define pr_fmt(fmt) "kprobes: " fmt
0015
0016 #include <linux/kernel.h>
0017 #include <linux/kprobes.h>
0018 #include <linux/module.h>
0019 #include <linux/slab.h>
0020 #include <linux/stop_machine.h>
0021 #include <linux/sched/debug.h>
0022 #include <linux/stringify.h>
0023 #include <asm/traps.h>
0024 #include <asm/opcodes.h>
0025 #include <asm/cacheflush.h>
0026 #include <linux/percpu.h>
0027 #include <linux/bug.h>
0028 #include <asm/patch.h>
0029 #include <asm/sections.h>
0030
0031 #include "../decode-arm.h"
0032 #include "../decode-thumb.h"
0033 #include "core.h"
0034
0035 #define MIN_STACK_SIZE(addr)                \
0036     min((unsigned long)MAX_STACK_SIZE,      \
0037         (unsigned long)current_thread_info() + THREAD_START_SP - (addr))
0038
0039 #define flush_insns(addr, size)             \
0040     flush_icache_range((unsigned long)(addr),   \
0041                (unsigned long)(addr) +  \
0042                (size))
0043
0044 DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
0045 DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
0046
0047
0048 int __kprobes arch_prepare_kprobe(struct kprobe *p)
0049 {
0050     kprobe_opcode_t insn;
0051     kprobe_opcode_t tmp_insn[MAX_INSN_SIZE];
0052     unsigned long addr = (unsigned long)p->addr;
0053     bool thumb;
0054     kprobe_decode_insn_t *decode_insn;
0055     const union decode_action *actions;
0056     int is;
0057     const struct decode_checker **checkers;
0058
0059 #ifdef CONFIG_THUMB2_KERNEL
0060     thumb = true;
0061     addr &= ~1; /* Bit 0 would normally be set to indicate Thumb code */
0062     insn = __mem_to_opcode_thumb16(((u16 *)addr)[0]);
0063     if (is_wide_instruction(insn)) {
0064         u16 inst2 = __mem_to_opcode_thumb16(((u16 *)addr)[1]);
0065         insn = __opcode_thumb32_compose(insn, inst2);
0066         decode_insn = thumb32_probes_decode_insn;
0067         actions = kprobes_t32_actions;
0068         checkers = kprobes_t32_checkers;
0069     } else {
0070         decode_insn = thumb16_probes_decode_insn;
0071         actions = kprobes_t16_actions;
0072         checkers = kprobes_t16_checkers;
0073     }
0074 #else /* !CONFIG_THUMB2_KERNEL */
0075     thumb = false;
0076     if (addr & 0x3)
0077         return -EINVAL;
0078     insn = __mem_to_opcode_arm(*p->addr);
0079     decode_insn = arm_probes_decode_insn;
0080     actions = kprobes_arm_actions;
0081     checkers = kprobes_arm_checkers;
0082 #endif
0083
0084     p->opcode = insn;
0085     p->ainsn.insn = tmp_insn;
0086
0087     switch ((*decode_insn)(insn, &p->ainsn, true, actions, checkers)) {
0088     case INSN_REJECTED: /* not supported */
0089         return -EINVAL;
0090
0091     case INSN_GOOD:     /* instruction uses slot */
0092         p->ainsn.insn = get_insn_slot();
0093         if (!p->ainsn.insn)
0094             return -ENOMEM;
0095         for (is = 0; is < MAX_INSN_SIZE; ++is)
0096             p->ainsn.insn[is] = tmp_insn[is];
0097         flush_insns(p->ainsn.insn,
0098                 sizeof(p->ainsn.insn[0]) * MAX_INSN_SIZE);
0099         p->ainsn.insn_fn = (probes_insn_fn_t *)
0100                     ((uintptr_t)p->ainsn.insn | thumb);
0101         break;
0102
0103     case INSN_GOOD_NO_SLOT: /* instruction doesn't need insn slot */
0104         p->ainsn.insn = NULL;
0105         break;
0106     }
0107
0108     /*
0109      * Never instrument insn like 'str r0, [sp, +/-r1]'. Also, insn likes
0110      * 'str r0, [sp, #-68]' should also be prohibited.
0111      * See __und_svc.
0112      */
0113     if ((p->ainsn.stack_space < 0) ||
0114             (p->ainsn.stack_space > MAX_STACK_SIZE))
0115         return -EINVAL;
0116
0117     return 0;
0118 }
0119
0120 void __kprobes arch_arm_kprobe(struct kprobe *p)
0121 {
0122     unsigned int brkp;
0123     void *addr;
0124
0125     if (IS_ENABLED(CONFIG_THUMB2_KERNEL)) {
0126         /* Remove any Thumb flag */
0127         addr = (void *)((uintptr_t)p->addr & ~1);
0128
0129         if (is_wide_instruction(p->opcode))
0130             brkp = KPROBE_THUMB32_BREAKPOINT_INSTRUCTION;
0131         else
0132             brkp = KPROBE_THUMB16_BREAKPOINT_INSTRUCTION;
0133     } else {
0134         kprobe_opcode_t insn = p->opcode;
0135
0136         addr = p->addr;
0137         brkp = KPROBE_ARM_BREAKPOINT_INSTRUCTION;
0138
0139         if (insn >= 0xe0000000)
0140             brkp |= 0xe0000000;  /* Unconditional instruction */
0141         else
0142             brkp |= insn & 0xf0000000;  /* Copy condition from insn */
0143     }
0144
0145     patch_text(addr, brkp);
0146 }
0147
0148 /*
0149  * The actual disarming is done here on each CPU and synchronized using
0150  * stop_machine. This synchronization is necessary on SMP to avoid removing
0151  * a probe between the moment the 'Undefined Instruction' exception is raised
0152  * and the moment the exception handler reads the faulting instruction from
0153  * memory. It is also needed to atomically set the two half-words of a 32-bit
0154  * Thumb breakpoint.
0155  */
0156 struct patch {
0157     void *addr;
0158     unsigned int insn;
0159 };
0160
0161 static int __kprobes_remove_breakpoint(void *data)
0162 {
0163     struct patch *p = data;
0164     __patch_text(p->addr, p->insn);
0165     return 0;
0166 }
0167
0168 void __kprobes kprobes_remove_breakpoint(void *addr, unsigned int insn)
0169 {
0170     struct patch p = {
0171         .addr = addr,
0172         .insn = insn,
0173     };
0174     stop_machine_cpuslocked(__kprobes_remove_breakpoint, &p,
0175                 cpu_online_mask);
0176 }
0177
0178 void __kprobes arch_disarm_kprobe(struct kprobe *p)
0179 {
0180     kprobes_remove_breakpoint((void *)((uintptr_t)p->addr & ~1),
0181             p->opcode);
0182 }
0183
0184 void __kprobes arch_remove_kprobe(struct kprobe *p)
0185 {
0186     if (p->ainsn.insn) {
0187         free_insn_slot(p->ainsn.insn, 0);
0188         p->ainsn.insn = NULL;
0189     }
0190 }
0191
0192 static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb)
0193 {
0194     kcb->prev_kprobe.kp = kprobe_running();
0195     kcb->prev_kprobe.status = kcb->kprobe_status;
0196 }
0197
0198 static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb)
0199 {
0200     __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
0201     kcb->kprobe_status = kcb->prev_kprobe.status;
0202 }
0203
0204 static void __kprobes set_current_kprobe(struct kprobe *p)
0205 {
0206     __this_cpu_write(current_kprobe, p);
0207 }
0208
0209 static void __kprobes
0210 singlestep_skip(struct kprobe *p, struct pt_regs *regs)
0211 {
0212 #ifdef CONFIG_THUMB2_KERNEL
0213     regs->ARM_cpsr = it_advance(regs->ARM_cpsr);
0214     if (is_wide_instruction(p->opcode))
0215         regs->ARM_pc += 4;
0216     else
0217         regs->ARM_pc += 2;
0218 #else
0219     regs->ARM_pc += 4;
0220 #endif
0221 }
0222
0223 static inline void __kprobes
0224 singlestep(struct kprobe *p, struct pt_regs *regs, struct kprobe_ctlblk *kcb)
0225 {
0226     p->ainsn.insn_singlestep(p->opcode, &p->ainsn, regs);
0227 }
0228
0229 /*
0230  * Called with IRQs disabled. IRQs must remain disabled from that point
0231  * all the way until processing this kprobe is complete.  The current
0232  * kprobes implementation cannot process more than one nested level of
0233  * kprobe, and that level is reserved for user kprobe handlers, so we can't
0234  * risk encountering a new kprobe in an interrupt handler.
0235  */
0236 void __kprobes kprobe_handler(struct pt_regs *regs)
0237 {
0238     struct kprobe *p, *cur;
0239     struct kprobe_ctlblk *kcb;
0240
0241     kcb = get_kprobe_ctlblk();
0242     cur = kprobe_running();
0243
0244 #ifdef CONFIG_THUMB2_KERNEL
0245     /*
0246      * First look for a probe which was registered using an address with
0247      * bit 0 set, this is the usual situation for pointers to Thumb code.
0248      * If not found, fallback to looking for one with bit 0 clear.
0249      */
0250     p = get_kprobe((kprobe_opcode_t *)(regs->ARM_pc | 1));
0251     if (!p)
0252         p = get_kprobe((kprobe_opcode_t *)regs->ARM_pc);
0253
0254 #else /* ! CONFIG_THUMB2_KERNEL */
0255     p = get_kprobe((kprobe_opcode_t *)regs->ARM_pc);
0256 #endif
0257
0258     if (p) {
0259         if (!p->ainsn.insn_check_cc(regs->ARM_cpsr)) {
0260             /*
0261              * Probe hit but conditional execution check failed,
0262              * so just skip the instruction and continue as if
0263              * nothing had happened.
0264              * In this case, we can skip recursing check too.
0265              */
0266             singlestep_skip(p, regs);
0267         } else if (cur) {
0268             /* Kprobe is pending, so we're recursing. */
0269             switch (kcb->kprobe_status) {
0270             case KPROBE_HIT_ACTIVE:
0271             case KPROBE_HIT_SSDONE:
0272             case KPROBE_HIT_SS:
0273                 /* A pre- or post-handler probe got us here. */
0274                 kprobes_inc_nmissed_count(p);
0275                 save_previous_kprobe(kcb);
0276                 set_current_kprobe(p);
0277                 kcb->kprobe_status = KPROBE_REENTER;
0278                 singlestep(p, regs, kcb);
0279                 restore_previous_kprobe(kcb);
0280                 break;
0281             case KPROBE_REENTER:
0282                 /* A nested probe was hit in FIQ, it is a BUG */
0283                 pr_warn("Failed to recover from reentered kprobes.\n");
0284                 dump_kprobe(p);
0285                 fallthrough;
0286             default:
0287                 /* impossible cases */
0288                 BUG();
0289             }
0290         } else {
0291             /* Probe hit and conditional execution check ok. */
0292             set_current_kprobe(p);
0293             kcb->kprobe_status = KPROBE_HIT_ACTIVE;
0294
0295             /*
0296              * If we have no pre-handler or it returned 0, we
0297              * continue with normal processing. If we have a
0298              * pre-handler and it returned non-zero, it will
0299              * modify the execution path and no need to single
0300              * stepping. Let's just reset current kprobe and exit.
0301              */
0302             if (!p->pre_handler || !p->pre_handler(p, regs)) {
0303                 kcb->kprobe_status = KPROBE_HIT_SS;
0304                 singlestep(p, regs, kcb);
0305                 if (p->post_handler) {
0306                     kcb->kprobe_status = KPROBE_HIT_SSDONE;
0307                     p->post_handler(p, regs, 0);
0308                 }
0309             }
0310             reset_current_kprobe();
0311         }
0312     } else {
0313         /*
0314          * The probe was removed and a race is in progress.
0315          * There is nothing we can do about it.  Let's restart
0316          * the instruction.  By the time we can restart, the
0317          * real instruction will be there.
0318          */
0319     }
0320 }
0321
0322 static int __kprobes kprobe_trap_handler(struct pt_regs *regs, unsigned int instr)
0323 {
0324     unsigned long flags;
0325     local_irq_save(flags);
0326     kprobe_handler(regs);
0327     local_irq_restore(flags);
0328     return 0;
0329 }
0330
0331 int __kprobes kprobe_fault_handler(struct pt_regs *regs, unsigned int fsr)
0332 {
0333     struct kprobe *cur = kprobe_running();
0334     struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
0335
0336     switch (kcb->kprobe_status) {
0337     case KPROBE_HIT_SS:
0338     case KPROBE_REENTER:
0339         /*
0340          * We are here because the instruction being single
0341          * stepped caused a page fault. We reset the current
0342          * kprobe and the PC to point back to the probe address
0343          * and allow the page fault handler to continue as a
0344          * normal page fault.
0345          */
0346         regs->ARM_pc = (long)cur->addr;
0347         if (kcb->kprobe_status == KPROBE_REENTER) {
0348             restore_previous_kprobe(kcb);
0349         } else {
0350             reset_current_kprobe();
0351         }
0352         break;
0353     }
0354
0355     return 0;
0356 }
0357
0358 int __kprobes kprobe_exceptions_notify(struct notifier_block *self,
0359                        unsigned long val, void *data)
0360 {
0361     /*
0362      * notify_die() is currently never called on ARM,
0363      * so this callback is currently empty.
0364      */
0365     return NOTIFY_DONE;
0366 }
0367
0368 /*
0369  * When a retprobed function returns, trampoline_handler() is called,
0370  * calling the kretprobe's handler. We construct a struct pt_regs to
0371  * give a view of registers r0-r11, sp, lr, and pc to the user
0372  * return-handler. This is not a complete pt_regs structure, but that
0373  * should be enough for stacktrace from the return handler with or
0374  * without pt_regs.
0375  */
0376 void __naked __kprobes __kretprobe_trampoline(void)
0377 {
0378     __asm__ __volatile__ (
0379 #ifdef CONFIG_FRAME_POINTER
0380         "ldr    lr, =__kretprobe_trampoline \n\t"
0381     /* __kretprobe_trampoline makes a framepointer on pt_regs. */
0382 #ifdef CONFIG_CC_IS_CLANG
0383         "stmdb  sp, {sp, lr, pc}    \n\t"
0384         "sub    sp, sp, #12     \n\t"
0385         /* In clang case, pt_regs->ip = lr. */
0386         "stmdb  sp!, {r0 - r11, lr} \n\t"
0387         /* fp points regs->r11 (fp) */
0388         "add    fp, sp, #44     \n\t"
0389 #else /* !CONFIG_CC_IS_CLANG */
0390         /* In gcc case, pt_regs->ip = fp. */
0391         "stmdb  sp, {fp, sp, lr, pc}    \n\t"
0392         "sub    sp, sp, #16     \n\t"
0393         "stmdb  sp!, {r0 - r11}     \n\t"
0394         /* fp points regs->r15 (pc) */
0395         "add    fp, sp, #60     \n\t"
0396 #endif /* CONFIG_CC_IS_CLANG */
0397 #else /* !CONFIG_FRAME_POINTER */
0398         "sub    sp, sp, #16     \n\t"
0399         "stmdb  sp!, {r0 - r11}     \n\t"
0400 #endif /* CONFIG_FRAME_POINTER */
0401         "mov    r0, sp          \n\t"
0402         "bl trampoline_handler  \n\t"
0403         "mov    lr, r0          \n\t"
0404         "ldmia  sp!, {r0 - r11}     \n\t"
0405         "add    sp, sp, #16     \n\t"
0406 #ifdef CONFIG_THUMB2_KERNEL
0407         "bx lr          \n\t"
0408 #else
0409         "mov    pc, lr          \n\t"
0410 #endif
0411         : : : "memory");
0412 }
0413
0414 /* Called from __kretprobe_trampoline */
0415 static __used __kprobes void *trampoline_handler(struct pt_regs *regs)
0416 {
0417     return (void *)kretprobe_trampoline_handler(regs, (void *)regs->ARM_fp);
0418 }
0419
0420 void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri,
0421                       struct pt_regs *regs)
0422 {
0423     ri->ret_addr = (kprobe_opcode_t *)regs->ARM_lr;
0424     ri->fp = (void *)regs->ARM_fp;
0425
0426     /* Replace the return addr with trampoline addr. */
0427     regs->ARM_lr = (unsigned long)&__kretprobe_trampoline;
0428 }
0429
0430 int __kprobes arch_trampoline_kprobe(struct kprobe *p)
0431 {
0432     return 0;
0433 }
0434
0435 #ifdef CONFIG_THUMB2_KERNEL
0436
0437 static struct undef_hook kprobes_thumb16_break_hook = {
0438     .instr_mask = 0xffff,
0439     .instr_val  = KPROBE_THUMB16_BREAKPOINT_INSTRUCTION,
0440     .cpsr_mask  = MODE_MASK,
0441     .cpsr_val   = SVC_MODE,
0442     .fn     = kprobe_trap_handler,
0443 };
0444
0445 static struct undef_hook kprobes_thumb32_break_hook = {
0446     .instr_mask = 0xffffffff,
0447     .instr_val  = KPROBE_THUMB32_BREAKPOINT_INSTRUCTION,
0448     .cpsr_mask  = MODE_MASK,
0449     .cpsr_val   = SVC_MODE,
0450     .fn     = kprobe_trap_handler,
0451 };
0452
0453 #else  /* !CONFIG_THUMB2_KERNEL */
0454
0455 static struct undef_hook kprobes_arm_break_hook = {
0456     .instr_mask = 0x0fffffff,
0457     .instr_val  = KPROBE_ARM_BREAKPOINT_INSTRUCTION,
0458     .cpsr_mask  = MODE_MASK,
0459     .cpsr_val   = SVC_MODE,
0460     .fn     = kprobe_trap_handler,
0461 };
0462
0463 #endif /* !CONFIG_THUMB2_KERNEL */
0464
0465 int __init arch_init_kprobes(void)
0466 {
0467     arm_probes_decode_init();
0468 #ifdef CONFIG_THUMB2_KERNEL
0469     register_undef_hook(&kprobes_thumb16_break_hook);
0470     register_undef_hook(&kprobes_thumb32_break_hook);
0471 #else
0472     register_undef_hook(&kprobes_arm_break_hook);
0473 #endif
0474     return 0;
0475 }
0476
0477 bool arch_within_kprobe_blacklist(unsigned long addr)
0478 {
0479     void *a = (void *)addr;
0480
0481     return __in_irqentry_text(addr) ||
0482            in_entry_text(addr) ||
0483            in_idmap_text(addr) ||
0484            memory_contains(__kprobes_text_start, __kprobes_text_end, a, 1);
0485 }