sparc/kernel/kprobes.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /* arch/sparc64/kernel/kprobes.c
0003  *
0004  * Copyright (C) 2004 David S. Miller <davem@davemloft.net>
0005  */
0006
0007 #include <linux/kernel.h>
0008 #include <linux/kprobes.h>
0009 #include <linux/extable.h>
0010 #include <linux/kdebug.h>
0011 #include <linux/slab.h>
0012 #include <linux/context_tracking.h>
0013 #include <asm/signal.h>
0014 #include <asm/cacheflush.h>
0015 #include <linux/uaccess.h>
0016
0017 /* We do not have hardware single-stepping on sparc64.
0018  * So we implement software single-stepping with breakpoint
0019  * traps.  The top-level scheme is similar to that used
0020  * in the x86 kprobes implementation.
0021  *
0022  * In the kprobe->ainsn.insn[] array we store the original
0023  * instruction at index zero and a break instruction at
0024  * index one.
0025  *
0026  * When we hit a kprobe we:
0027  * - Run the pre-handler
0028  * - Remember "regs->tnpc" and interrupt level stored in
0029  *   "regs->tstate" so we can restore them later
0030  * - Disable PIL interrupts
0031  * - Set regs->tpc to point to kprobe->ainsn.insn[0]
0032  * - Set regs->tnpc to point to kprobe->ainsn.insn[1]
0033  * - Mark that we are actively in a kprobe
0034  *
0035  * At this point we wait for the second breakpoint at
0036  * kprobe->ainsn.insn[1] to hit.  When it does we:
0037  * - Run the post-handler
0038  * - Set regs->tpc to "remembered" regs->tnpc stored above,
0039  *   restore the PIL interrupt level in "regs->tstate" as well
0040  * - Make any adjustments necessary to regs->tnpc in order
0041  *   to handle relative branches correctly.  See below.
0042  * - Mark that we are no longer actively in a kprobe.
0043  */
0044
0045 DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
0046 DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
0047
0048 struct kretprobe_blackpoint kretprobe_blacklist[] = {{NULL, NULL}};
0049
0050 int __kprobes arch_prepare_kprobe(struct kprobe *p)
0051 {
0052     if ((unsigned long) p->addr & 0x3UL)
0053         return -EILSEQ;
0054
0055     p->ainsn.insn[0] = *p->addr;
0056     flushi(&p->ainsn.insn[0]);
0057
0058     p->ainsn.insn[1] = BREAKPOINT_INSTRUCTION_2;
0059     flushi(&p->ainsn.insn[1]);
0060
0061     p->opcode = *p->addr;
0062     return 0;
0063 }
0064
0065 void __kprobes arch_arm_kprobe(struct kprobe *p)
0066 {
0067     *p->addr = BREAKPOINT_INSTRUCTION;
0068     flushi(p->addr);
0069 }
0070
0071 void __kprobes arch_disarm_kprobe(struct kprobe *p)
0072 {
0073     *p->addr = p->opcode;
0074     flushi(p->addr);
0075 }
0076
0077 static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb)
0078 {
0079     kcb->prev_kprobe.kp = kprobe_running();
0080     kcb->prev_kprobe.status = kcb->kprobe_status;
0081     kcb->prev_kprobe.orig_tnpc = kcb->kprobe_orig_tnpc;
0082     kcb->prev_kprobe.orig_tstate_pil = kcb->kprobe_orig_tstate_pil;
0083 }
0084
0085 static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb)
0086 {
0087     __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
0088     kcb->kprobe_status = kcb->prev_kprobe.status;
0089     kcb->kprobe_orig_tnpc = kcb->prev_kprobe.orig_tnpc;
0090     kcb->kprobe_orig_tstate_pil = kcb->prev_kprobe.orig_tstate_pil;
0091 }
0092
0093 static void __kprobes set_current_kprobe(struct kprobe *p, struct pt_regs *regs,
0094                 struct kprobe_ctlblk *kcb)
0095 {
0096     __this_cpu_write(current_kprobe, p);
0097     kcb->kprobe_orig_tnpc = regs->tnpc;
0098     kcb->kprobe_orig_tstate_pil = (regs->tstate & TSTATE_PIL);
0099 }
0100
0101 static void __kprobes prepare_singlestep(struct kprobe *p, struct pt_regs *regs,
0102             struct kprobe_ctlblk *kcb)
0103 {
0104     regs->tstate |= TSTATE_PIL;
0105
0106     /*single step inline, if it a breakpoint instruction*/
0107     if (p->opcode == BREAKPOINT_INSTRUCTION) {
0108         regs->tpc = (unsigned long) p->addr;
0109         regs->tnpc = kcb->kprobe_orig_tnpc;
0110     } else {
0111         regs->tpc = (unsigned long) &p->ainsn.insn[0];
0112         regs->tnpc = (unsigned long) &p->ainsn.insn[1];
0113     }
0114 }
0115
0116 static int __kprobes kprobe_handler(struct pt_regs *regs)
0117 {
0118     struct kprobe *p;
0119     void *addr = (void *) regs->tpc;
0120     int ret = 0;
0121     struct kprobe_ctlblk *kcb;
0122
0123     /*
0124      * We don't want to be preempted for the entire
0125      * duration of kprobe processing
0126      */
0127     preempt_disable();
0128     kcb = get_kprobe_ctlblk();
0129
0130     if (kprobe_running()) {
0131         p = get_kprobe(addr);
0132         if (p) {
0133             if (kcb->kprobe_status == KPROBE_HIT_SS) {
0134                 regs->tstate = ((regs->tstate & ~TSTATE_PIL) |
0135                     kcb->kprobe_orig_tstate_pil);
0136                 goto no_kprobe;
0137             }
0138             /* We have reentered the kprobe_handler(), since
0139              * another probe was hit while within the handler.
0140              * We here save the original kprobes variables and
0141              * just single step on the instruction of the new probe
0142              * without calling any user handlers.
0143              */
0144             save_previous_kprobe(kcb);
0145             set_current_kprobe(p, regs, kcb);
0146             kprobes_inc_nmissed_count(p);
0147             kcb->kprobe_status = KPROBE_REENTER;
0148             prepare_singlestep(p, regs, kcb);
0149             return 1;
0150         } else if (*(u32 *)addr != BREAKPOINT_INSTRUCTION) {
0151             /* The breakpoint instruction was removed by
0152              * another cpu right after we hit, no further
0153              * handling of this interrupt is appropriate
0154              */
0155             ret = 1;
0156         }
0157         goto no_kprobe;
0158     }
0159
0160     p = get_kprobe(addr);
0161     if (!p) {
0162         if (*(u32 *)addr != BREAKPOINT_INSTRUCTION) {
0163             /*
0164              * The breakpoint instruction was removed right
0165              * after we hit it.  Another cpu has removed
0166              * either a probepoint or a debugger breakpoint
0167              * at this address.  In either case, no further
0168              * handling of this interrupt is appropriate.
0169              */
0170             ret = 1;
0171         }
0172         /* Not one of ours: let kernel handle it */
0173         goto no_kprobe;
0174     }
0175
0176     set_current_kprobe(p, regs, kcb);
0177     kcb->kprobe_status = KPROBE_HIT_ACTIVE;
0178     if (p->pre_handler && p->pre_handler(p, regs)) {
0179         reset_current_kprobe();
0180         preempt_enable_no_resched();
0181         return 1;
0182     }
0183
0184     prepare_singlestep(p, regs, kcb);
0185     kcb->kprobe_status = KPROBE_HIT_SS;
0186     return 1;
0187
0188 no_kprobe:
0189     preempt_enable_no_resched();
0190     return ret;
0191 }
0192
0193 /* If INSN is a relative control transfer instruction,
0194  * return the corrected branch destination value.
0195  *
0196  * regs->tpc and regs->tnpc still hold the values of the
0197  * program counters at the time of trap due to the execution
0198  * of the BREAKPOINT_INSTRUCTION_2 at p->ainsn.insn[1]
0199  *
0200  */
0201 static unsigned long __kprobes relbranch_fixup(u32 insn, struct kprobe *p,
0202                            struct pt_regs *regs)
0203 {
0204     unsigned long real_pc = (unsigned long) p->addr;
0205
0206     /* Branch not taken, no mods necessary.  */
0207     if (regs->tnpc == regs->tpc + 0x4UL)
0208         return real_pc + 0x8UL;
0209
0210     /* The three cases are call, branch w/prediction,
0211      * and traditional branch.
0212      */
0213     if ((insn & 0xc0000000) == 0x40000000 ||
0214         (insn & 0xc1c00000) == 0x00400000 ||
0215         (insn & 0xc1c00000) == 0x00800000) {
0216         unsigned long ainsn_addr;
0217
0218         ainsn_addr = (unsigned long) &p->ainsn.insn[0];
0219
0220         /* The instruction did all the work for us
0221          * already, just apply the offset to the correct
0222          * instruction location.
0223          */
0224         return (real_pc + (regs->tnpc - ainsn_addr));
0225     }
0226
0227     /* It is jmpl or some other absolute PC modification instruction,
0228      * leave NPC as-is.
0229      */
0230     return regs->tnpc;
0231 }
0232
0233 /* If INSN is an instruction which writes it's PC location
0234  * into a destination register, fix that up.
0235  */
0236 static void __kprobes retpc_fixup(struct pt_regs *regs, u32 insn,
0237                   unsigned long real_pc)
0238 {
0239     unsigned long *slot = NULL;
0240
0241     /* Simplest case is 'call', which always uses %o7 */
0242     if ((insn & 0xc0000000) == 0x40000000) {
0243         slot = &regs->u_regs[UREG_I7];
0244     }
0245
0246     /* 'jmpl' encodes the register inside of the opcode */
0247     if ((insn & 0xc1f80000) == 0x81c00000) {
0248         unsigned long rd = ((insn >> 25) & 0x1f);
0249
0250         if (rd <= 15) {
0251             slot = &regs->u_regs[rd];
0252         } else {
0253             /* Hard case, it goes onto the stack. */
0254             flushw_all();
0255
0256             rd -= 16;
0257             slot = (unsigned long *)
0258                 (regs->u_regs[UREG_FP] + STACK_BIAS);
0259             slot += rd;
0260         }
0261     }
0262     if (slot != NULL)
0263         *slot = real_pc;
0264 }
0265
0266 /*
0267  * Called after single-stepping.  p->addr is the address of the
0268  * instruction which has been replaced by the breakpoint
0269  * instruction.  To avoid the SMP problems that can occur when we
0270  * temporarily put back the original opcode to single-step, we
0271  * single-stepped a copy of the instruction.  The address of this
0272  * copy is &p->ainsn.insn[0].
0273  *
0274  * This function prepares to return from the post-single-step
0275  * breakpoint trap.
0276  */
0277 static void __kprobes resume_execution(struct kprobe *p,
0278         struct pt_regs *regs, struct kprobe_ctlblk *kcb)
0279 {
0280     u32 insn = p->ainsn.insn[0];
0281
0282     regs->tnpc = relbranch_fixup(insn, p, regs);
0283
0284     /* This assignment must occur after relbranch_fixup() */
0285     regs->tpc = kcb->kprobe_orig_tnpc;
0286
0287     retpc_fixup(regs, insn, (unsigned long) p->addr);
0288
0289     regs->tstate = ((regs->tstate & ~TSTATE_PIL) |
0290             kcb->kprobe_orig_tstate_pil);
0291 }
0292
0293 static int __kprobes post_kprobe_handler(struct pt_regs *regs)
0294 {
0295     struct kprobe *cur = kprobe_running();
0296     struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
0297
0298     if (!cur)
0299         return 0;
0300
0301     if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) {
0302         kcb->kprobe_status = KPROBE_HIT_SSDONE;
0303         cur->post_handler(cur, regs, 0);
0304     }
0305
0306     resume_execution(cur, regs, kcb);
0307
0308     /*Restore back the original saved kprobes variables and continue. */
0309     if (kcb->kprobe_status == KPROBE_REENTER) {
0310         restore_previous_kprobe(kcb);
0311         goto out;
0312     }
0313     reset_current_kprobe();
0314 out:
0315     preempt_enable_no_resched();
0316
0317     return 1;
0318 }
0319
0320 int __kprobes kprobe_fault_handler(struct pt_regs *regs, int trapnr)
0321 {
0322     struct kprobe *cur = kprobe_running();
0323     struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
0324     const struct exception_table_entry *entry;
0325
0326     switch(kcb->kprobe_status) {
0327     case KPROBE_HIT_SS:
0328     case KPROBE_REENTER:
0329         /*
0330          * We are here because the instruction being single
0331          * stepped caused a page fault. We reset the current
0332          * kprobe and the tpc points back to the probe address
0333          * and allow the page fault handler to continue as a
0334          * normal page fault.
0335          */
0336         regs->tpc = (unsigned long)cur->addr;
0337         regs->tnpc = kcb->kprobe_orig_tnpc;
0338         regs->tstate = ((regs->tstate & ~TSTATE_PIL) |
0339                 kcb->kprobe_orig_tstate_pil);
0340         if (kcb->kprobe_status == KPROBE_REENTER)
0341             restore_previous_kprobe(kcb);
0342         else
0343             reset_current_kprobe();
0344         preempt_enable_no_resched();
0345         break;
0346     case KPROBE_HIT_ACTIVE:
0347     case KPROBE_HIT_SSDONE:
0348         /*
0349          * In case the user-specified fault handler returned
0350          * zero, try to fix up.
0351          */
0352
0353         entry = search_exception_tables(regs->tpc);
0354         if (entry) {
0355             regs->tpc = entry->fixup;
0356             regs->tnpc = regs->tpc + 4;
0357             return 1;
0358         }
0359
0360         /*
0361          * fixup_exception() could not handle it,
0362          * Let do_page_fault() fix it.
0363          */
0364         break;
0365     default:
0366         break;
0367     }
0368
0369     return 0;
0370 }
0371
0372 /*
0373  * Wrapper routine to for handling exceptions.
0374  */
0375 int __kprobes kprobe_exceptions_notify(struct notifier_block *self,
0376                        unsigned long val, void *data)
0377 {
0378     struct die_args *args = (struct die_args *)data;
0379     int ret = NOTIFY_DONE;
0380
0381     if (args->regs && user_mode(args->regs))
0382         return ret;
0383
0384     switch (val) {
0385     case DIE_DEBUG:
0386         if (kprobe_handler(args->regs))
0387             ret = NOTIFY_STOP;
0388         break;
0389     case DIE_DEBUG_2:
0390         if (post_kprobe_handler(args->regs))
0391             ret = NOTIFY_STOP;
0392         break;
0393     default:
0394         break;
0395     }
0396     return ret;
0397 }
0398
0399 asmlinkage void __kprobes kprobe_trap(unsigned long trap_level,
0400                       struct pt_regs *regs)
0401 {
0402     enum ctx_state prev_state = exception_enter();
0403
0404     BUG_ON(trap_level != 0x170 && trap_level != 0x171);
0405
0406     if (user_mode(regs)) {
0407         local_irq_enable();
0408         bad_trap(regs, trap_level);
0409         goto out;
0410     }
0411
0412     /* trap_level == 0x170 --> ta 0x70
0413      * trap_level == 0x171 --> ta 0x71
0414      */
0415     if (notify_die((trap_level == 0x170) ? DIE_DEBUG : DIE_DEBUG_2,
0416                (trap_level == 0x170) ? "debug" : "debug_2",
0417                regs, 0, trap_level, SIGTRAP) != NOTIFY_STOP)
0418         bad_trap(regs, trap_level);
0419 out:
0420     exception_exit(prev_state);
0421 }
0422
0423 /* The value stored in the return address register is actually 2
0424  * instructions before where the callee will return to.
0425  * Sequences usually look something like this
0426  *
0427  *      call    some_function   <--- return register points here
0428  *       nop            <--- call delay slot
0429  *      whatever        <--- where callee returns to
0430  *
0431  * To keep trampoline_probe_handler logic simpler, we normalize the
0432  * value kept in ri->ret_addr so we don't need to keep adjusting it
0433  * back and forth.
0434  */
0435 void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri,
0436                       struct pt_regs *regs)
0437 {
0438     ri->ret_addr = (kprobe_opcode_t *)(regs->u_regs[UREG_RETPC] + 8);
0439     ri->fp = NULL;
0440
0441     /* Replace the return addr with trampoline addr */
0442     regs->u_regs[UREG_RETPC] =
0443         ((unsigned long)__kretprobe_trampoline) - 8;
0444 }
0445
0446 /*
0447  * Called when the probe at kretprobe trampoline is hit
0448  */
0449 static int __kprobes trampoline_probe_handler(struct kprobe *p,
0450                           struct pt_regs *regs)
0451 {
0452     unsigned long orig_ret_address = 0;
0453
0454     orig_ret_address = __kretprobe_trampoline_handler(regs, NULL);
0455     regs->tpc = orig_ret_address;
0456     regs->tnpc = orig_ret_address + 4;
0457
0458     /*
0459      * By returning a non-zero value, we are telling
0460      * kprobe_handler() that we don't want the post_handler
0461      * to run (and have re-enabled preemption)
0462      */
0463     return 1;
0464 }
0465
0466 static void __used kretprobe_trampoline_holder(void)
0467 {
0468     asm volatile(".global __kretprobe_trampoline\n"
0469              "__kretprobe_trampoline:\n"
0470              "\tnop\n"
0471              "\tnop\n");
0472 }
0473 static struct kprobe trampoline_p = {
0474     .addr = (kprobe_opcode_t *) &__kretprobe_trampoline,
0475     .pre_handler = trampoline_probe_handler
0476 };
0477
0478 int __init arch_init_kprobes(void)
0479 {
0480     return register_kprobe(&trampoline_p);
0481 }
0482
0483 int __kprobes arch_trampoline_kprobe(struct kprobe *p)
0484 {
0485     if (p->addr == (kprobe_opcode_t *)&__kretprobe_trampoline)
0486         return 1;
0487
0488     return 0;
0489 }