mips/kernel/kprobes.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  *  Kernel Probes (KProbes)
0004  *  arch/mips/kernel/kprobes.c
0005  *
0006  *  Copyright 2006 Sony Corp.
0007  *  Copyright 2010 Cavium Networks
0008  *
0009  *  Some portions copied from the powerpc version.
0010  *
0011  *   Copyright (C) IBM Corporation, 2002, 2004
0012  */
0013
0014 #define pr_fmt(fmt) "kprobes: " fmt
0015
0016 #include <linux/kprobes.h>
0017 #include <linux/preempt.h>
0018 #include <linux/uaccess.h>
0019 #include <linux/kdebug.h>
0020 #include <linux/slab.h>
0021
0022 #include <asm/ptrace.h>
0023 #include <asm/branch.h>
0024 #include <asm/break.h>
0025
0026 #include "probes-common.h"
0027
0028 static const union mips_instruction breakpoint_insn = {
0029     .b_format = {
0030         .opcode = spec_op,
0031         .code = BRK_KPROBE_BP,
0032         .func = break_op
0033     }
0034 };
0035
0036 static const union mips_instruction breakpoint2_insn = {
0037     .b_format = {
0038         .opcode = spec_op,
0039         .code = BRK_KPROBE_SSTEPBP,
0040         .func = break_op
0041     }
0042 };
0043
0044 DEFINE_PER_CPU(struct kprobe *, current_kprobe);
0045 DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
0046
0047 static int insn_has_delayslot(union mips_instruction insn)
0048 {
0049     return __insn_has_delay_slot(insn);
0050 }
0051 NOKPROBE_SYMBOL(insn_has_delayslot);
0052
0053 /*
0054  * insn_has_ll_or_sc function checks whether instruction is ll or sc
0055  * one; putting breakpoint on top of atomic ll/sc pair is bad idea;
0056  * so we need to prevent it and refuse kprobes insertion for such
0057  * instructions; cannot do much about breakpoint in the middle of
0058  * ll/sc pair; it is upto user to avoid those places
0059  */
0060 static int insn_has_ll_or_sc(union mips_instruction insn)
0061 {
0062     int ret = 0;
0063
0064     switch (insn.i_format.opcode) {
0065     case ll_op:
0066     case lld_op:
0067     case sc_op:
0068     case scd_op:
0069         ret = 1;
0070         break;
0071     default:
0072         break;
0073     }
0074     return ret;
0075 }
0076 NOKPROBE_SYMBOL(insn_has_ll_or_sc);
0077
0078 int arch_prepare_kprobe(struct kprobe *p)
0079 {
0080     union mips_instruction insn;
0081     union mips_instruction prev_insn;
0082     int ret = 0;
0083
0084     insn = p->addr[0];
0085
0086     if (insn_has_ll_or_sc(insn)) {
0087         pr_notice("Kprobes for ll and sc instructions are not supported\n");
0088         ret = -EINVAL;
0089         goto out;
0090     }
0091
0092     if (copy_from_kernel_nofault(&prev_insn, p->addr - 1,
0093             sizeof(mips_instruction)) == 0 &&
0094         insn_has_delayslot(prev_insn)) {
0095         pr_notice("Kprobes for branch delayslot are not supported\n");
0096         ret = -EINVAL;
0097         goto out;
0098     }
0099
0100     if (__insn_is_compact_branch(insn)) {
0101         pr_notice("Kprobes for compact branches are not supported\n");
0102         ret = -EINVAL;
0103         goto out;
0104     }
0105
0106     /* insn: must be on special executable page on mips. */
0107     p->ainsn.insn = get_insn_slot();
0108     if (!p->ainsn.insn) {
0109         ret = -ENOMEM;
0110         goto out;
0111     }
0112
0113     /*
0114      * In the kprobe->ainsn.insn[] array we store the original
0115      * instruction at index zero and a break trap instruction at
0116      * index one.
0117      *
0118      * On MIPS arch if the instruction at probed address is a
0119      * branch instruction, we need to execute the instruction at
0120      * Branch Delayslot (BD) at the time of probe hit. As MIPS also
0121      * doesn't have single stepping support, the BD instruction can
0122      * not be executed in-line and it would be executed on SSOL slot
0123      * using a normal breakpoint instruction in the next slot.
0124      * So, read the instruction and save it for later execution.
0125      */
0126     if (insn_has_delayslot(insn))
0127         memcpy(&p->ainsn.insn[0], p->addr + 1, sizeof(kprobe_opcode_t));
0128     else
0129         memcpy(&p->ainsn.insn[0], p->addr, sizeof(kprobe_opcode_t));
0130
0131     p->ainsn.insn[1] = breakpoint2_insn;
0132     p->opcode = *p->addr;
0133
0134 out:
0135     return ret;
0136 }
0137 NOKPROBE_SYMBOL(arch_prepare_kprobe);
0138
0139 void arch_arm_kprobe(struct kprobe *p)
0140 {
0141     *p->addr = breakpoint_insn;
0142     flush_insn_slot(p);
0143 }
0144 NOKPROBE_SYMBOL(arch_arm_kprobe);
0145
0146 void arch_disarm_kprobe(struct kprobe *p)
0147 {
0148     *p->addr = p->opcode;
0149     flush_insn_slot(p);
0150 }
0151 NOKPROBE_SYMBOL(arch_disarm_kprobe);
0152
0153 void arch_remove_kprobe(struct kprobe *p)
0154 {
0155     if (p->ainsn.insn) {
0156         free_insn_slot(p->ainsn.insn, 0);
0157         p->ainsn.insn = NULL;
0158     }
0159 }
0160 NOKPROBE_SYMBOL(arch_remove_kprobe);
0161
0162 static void save_previous_kprobe(struct kprobe_ctlblk *kcb)
0163 {
0164     kcb->prev_kprobe.kp = kprobe_running();
0165     kcb->prev_kprobe.status = kcb->kprobe_status;
0166     kcb->prev_kprobe.old_SR = kcb->kprobe_old_SR;
0167     kcb->prev_kprobe.saved_SR = kcb->kprobe_saved_SR;
0168     kcb->prev_kprobe.saved_epc = kcb->kprobe_saved_epc;
0169 }
0170
0171 static void restore_previous_kprobe(struct kprobe_ctlblk *kcb)
0172 {
0173     __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
0174     kcb->kprobe_status = kcb->prev_kprobe.status;
0175     kcb->kprobe_old_SR = kcb->prev_kprobe.old_SR;
0176     kcb->kprobe_saved_SR = kcb->prev_kprobe.saved_SR;
0177     kcb->kprobe_saved_epc = kcb->prev_kprobe.saved_epc;
0178 }
0179
0180 static void set_current_kprobe(struct kprobe *p, struct pt_regs *regs,
0181                    struct kprobe_ctlblk *kcb)
0182 {
0183     __this_cpu_write(current_kprobe, p);
0184     kcb->kprobe_saved_SR = kcb->kprobe_old_SR = (regs->cp0_status & ST0_IE);
0185     kcb->kprobe_saved_epc = regs->cp0_epc;
0186 }
0187
0188 /**
0189  * evaluate_branch_instrucion -
0190  *
0191  * Evaluate the branch instruction at probed address during probe hit. The
0192  * result of evaluation would be the updated epc. The insturction in delayslot
0193  * would actually be single stepped using a normal breakpoint) on SSOL slot.
0194  *
0195  * The result is also saved in the kprobe control block for later use,
0196  * in case we need to execute the delayslot instruction. The latter will be
0197  * false for NOP instruction in dealyslot and the branch-likely instructions
0198  * when the branch is taken. And for those cases we set a flag as
0199  * SKIP_DELAYSLOT in the kprobe control block
0200  */
0201 static int evaluate_branch_instruction(struct kprobe *p, struct pt_regs *regs,
0202                     struct kprobe_ctlblk *kcb)
0203 {
0204     union mips_instruction insn = p->opcode;
0205     long epc;
0206     int ret = 0;
0207
0208     epc = regs->cp0_epc;
0209     if (epc & 3)
0210         goto unaligned;
0211
0212     if (p->ainsn.insn->word == 0)
0213         kcb->flags |= SKIP_DELAYSLOT;
0214     else
0215         kcb->flags &= ~SKIP_DELAYSLOT;
0216
0217     ret = __compute_return_epc_for_insn(regs, insn);
0218     if (ret < 0)
0219         return ret;
0220
0221     if (ret == BRANCH_LIKELY_TAKEN)
0222         kcb->flags |= SKIP_DELAYSLOT;
0223
0224     kcb->target_epc = regs->cp0_epc;
0225
0226     return 0;
0227
0228 unaligned:
0229     pr_notice("Failed to emulate branch instruction because of unaligned epc - sending SIGBUS to %s.\n", current->comm);
0230     force_sig(SIGBUS);
0231     return -EFAULT;
0232
0233 }
0234
0235 static void prepare_singlestep(struct kprobe *p, struct pt_regs *regs,
0236                         struct kprobe_ctlblk *kcb)
0237 {
0238     int ret = 0;
0239
0240     regs->cp0_status &= ~ST0_IE;
0241
0242     /* single step inline if the instruction is a break */
0243     if (p->opcode.word == breakpoint_insn.word ||
0244         p->opcode.word == breakpoint2_insn.word)
0245         regs->cp0_epc = (unsigned long)p->addr;
0246     else if (insn_has_delayslot(p->opcode)) {
0247         ret = evaluate_branch_instruction(p, regs, kcb);
0248         if (ret < 0)
0249             return;
0250     }
0251     regs->cp0_epc = (unsigned long)&p->ainsn.insn[0];
0252 }
0253
0254 /*
0255  * Called after single-stepping.  p->addr is the address of the
0256  * instruction whose first byte has been replaced by the "break 0"
0257  * instruction.  To avoid the SMP problems that can occur when we
0258  * temporarily put back the original opcode to single-step, we
0259  * single-stepped a copy of the instruction.  The address of this
0260  * copy is p->ainsn.insn.
0261  *
0262  * This function prepares to return from the post-single-step
0263  * breakpoint trap. In case of branch instructions, the target
0264  * epc to be restored.
0265  */
0266 static void resume_execution(struct kprobe *p,
0267                        struct pt_regs *regs,
0268                        struct kprobe_ctlblk *kcb)
0269 {
0270     if (insn_has_delayslot(p->opcode))
0271         regs->cp0_epc = kcb->target_epc;
0272     else {
0273         unsigned long orig_epc = kcb->kprobe_saved_epc;
0274         regs->cp0_epc = orig_epc + 4;
0275     }
0276 }
0277 NOKPROBE_SYMBOL(resume_execution);
0278
0279 static int kprobe_handler(struct pt_regs *regs)
0280 {
0281     struct kprobe *p;
0282     int ret = 0;
0283     kprobe_opcode_t *addr;
0284     struct kprobe_ctlblk *kcb;
0285
0286     addr = (kprobe_opcode_t *) regs->cp0_epc;
0287
0288     /*
0289      * We don't want to be preempted for the entire
0290      * duration of kprobe processing
0291      */
0292     preempt_disable();
0293     kcb = get_kprobe_ctlblk();
0294
0295     /* Check we're not actually recursing */
0296     if (kprobe_running()) {
0297         p = get_kprobe(addr);
0298         if (p) {
0299             if (kcb->kprobe_status == KPROBE_HIT_SS &&
0300                 p->ainsn.insn->word == breakpoint_insn.word) {
0301                 regs->cp0_status &= ~ST0_IE;
0302                 regs->cp0_status |= kcb->kprobe_saved_SR;
0303                 goto no_kprobe;
0304             }
0305             /*
0306              * We have reentered the kprobe_handler(), since
0307              * another probe was hit while within the handler.
0308              * We here save the original kprobes variables and
0309              * just single step on the instruction of the new probe
0310              * without calling any user handlers.
0311              */
0312             save_previous_kprobe(kcb);
0313             set_current_kprobe(p, regs, kcb);
0314             kprobes_inc_nmissed_count(p);
0315             prepare_singlestep(p, regs, kcb);
0316             kcb->kprobe_status = KPROBE_REENTER;
0317             if (kcb->flags & SKIP_DELAYSLOT) {
0318                 resume_execution(p, regs, kcb);
0319                 restore_previous_kprobe(kcb);
0320                 preempt_enable_no_resched();
0321             }
0322             return 1;
0323         } else if (addr->word != breakpoint_insn.word) {
0324             /*
0325              * The breakpoint instruction was removed by
0326              * another cpu right after we hit, no further
0327              * handling of this interrupt is appropriate
0328              */
0329             ret = 1;
0330         }
0331         goto no_kprobe;
0332     }
0333
0334     p = get_kprobe(addr);
0335     if (!p) {
0336         if (addr->word != breakpoint_insn.word) {
0337             /*
0338              * The breakpoint instruction was removed right
0339              * after we hit it.  Another cpu has removed
0340              * either a probepoint or a debugger breakpoint
0341              * at this address.  In either case, no further
0342              * handling of this interrupt is appropriate.
0343              */
0344             ret = 1;
0345         }
0346         /* Not one of ours: let kernel handle it */
0347         goto no_kprobe;
0348     }
0349
0350     set_current_kprobe(p, regs, kcb);
0351     kcb->kprobe_status = KPROBE_HIT_ACTIVE;
0352
0353     if (p->pre_handler && p->pre_handler(p, regs)) {
0354         /* handler has already set things up, so skip ss setup */
0355         reset_current_kprobe();
0356         preempt_enable_no_resched();
0357         return 1;
0358     }
0359
0360     prepare_singlestep(p, regs, kcb);
0361     if (kcb->flags & SKIP_DELAYSLOT) {
0362         kcb->kprobe_status = KPROBE_HIT_SSDONE;
0363         if (p->post_handler)
0364             p->post_handler(p, regs, 0);
0365         resume_execution(p, regs, kcb);
0366         preempt_enable_no_resched();
0367     } else
0368         kcb->kprobe_status = KPROBE_HIT_SS;
0369
0370     return 1;
0371
0372 no_kprobe:
0373     preempt_enable_no_resched();
0374     return ret;
0375
0376 }
0377 NOKPROBE_SYMBOL(kprobe_handler);
0378
0379 static inline int post_kprobe_handler(struct pt_regs *regs)
0380 {
0381     struct kprobe *cur = kprobe_running();
0382     struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
0383
0384     if (!cur)
0385         return 0;
0386
0387     if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) {
0388         kcb->kprobe_status = KPROBE_HIT_SSDONE;
0389         cur->post_handler(cur, regs, 0);
0390     }
0391
0392     resume_execution(cur, regs, kcb);
0393
0394     regs->cp0_status |= kcb->kprobe_saved_SR;
0395
0396     /* Restore back the original saved kprobes variables and continue. */
0397     if (kcb->kprobe_status == KPROBE_REENTER) {
0398         restore_previous_kprobe(kcb);
0399         goto out;
0400     }
0401     reset_current_kprobe();
0402 out:
0403     preempt_enable_no_resched();
0404
0405     return 1;
0406 }
0407
0408 int kprobe_fault_handler(struct pt_regs *regs, int trapnr)
0409 {
0410     struct kprobe *cur = kprobe_running();
0411     struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
0412
0413     if (kcb->kprobe_status & KPROBE_HIT_SS) {
0414         resume_execution(cur, regs, kcb);
0415         regs->cp0_status |= kcb->kprobe_old_SR;
0416
0417         reset_current_kprobe();
0418         preempt_enable_no_resched();
0419     }
0420     return 0;
0421 }
0422
0423 /*
0424  * Wrapper routine for handling exceptions.
0425  */
0426 int kprobe_exceptions_notify(struct notifier_block *self,
0427                        unsigned long val, void *data)
0428 {
0429
0430     struct die_args *args = (struct die_args *)data;
0431     int ret = NOTIFY_DONE;
0432
0433     switch (val) {
0434     case DIE_BREAK:
0435         if (kprobe_handler(args->regs))
0436             ret = NOTIFY_STOP;
0437         break;
0438     case DIE_SSTEPBP:
0439         if (post_kprobe_handler(args->regs))
0440             ret = NOTIFY_STOP;
0441         break;
0442
0443     case DIE_PAGE_FAULT:
0444         /* kprobe_running() needs smp_processor_id() */
0445         preempt_disable();
0446
0447         if (kprobe_running()
0448             && kprobe_fault_handler(args->regs, args->trapnr))
0449             ret = NOTIFY_STOP;
0450         preempt_enable();
0451         break;
0452     default:
0453         break;
0454     }
0455     return ret;
0456 }
0457 NOKPROBE_SYMBOL(kprobe_exceptions_notify);
0458
0459 /*
0460  * Function return probe trampoline:
0461  *  - init_kprobes() establishes a probepoint here
0462  *  - When the probed function returns, this probe causes the
0463  *    handlers to fire
0464  */
0465 static void __used kretprobe_trampoline_holder(void)
0466 {
0467     asm volatile(
0468         ".set push\n\t"
0469         /* Keep the assembler from reordering and placing JR here. */
0470         ".set noreorder\n\t"
0471         "nop\n\t"
0472         ".global __kretprobe_trampoline\n"
0473         "__kretprobe_trampoline:\n\t"
0474         "nop\n\t"
0475         ".set pop"
0476         : : : "memory");
0477 }
0478
0479 void __kretprobe_trampoline(void);
0480
0481 void arch_prepare_kretprobe(struct kretprobe_instance *ri,
0482                       struct pt_regs *regs)
0483 {
0484     ri->ret_addr = (kprobe_opcode_t *) regs->regs[31];
0485     ri->fp = NULL;
0486
0487     /* Replace the return addr with trampoline addr */
0488     regs->regs[31] = (unsigned long)__kretprobe_trampoline;
0489 }
0490 NOKPROBE_SYMBOL(arch_prepare_kretprobe);
0491
0492 /*
0493  * Called when the probe at kretprobe trampoline is hit
0494  */
0495 static int trampoline_probe_handler(struct kprobe *p,
0496                         struct pt_regs *regs)
0497 {
0498     instruction_pointer(regs) = __kretprobe_trampoline_handler(regs, NULL);
0499     /*
0500      * By returning a non-zero value, we are telling
0501      * kprobe_handler() that we don't want the post_handler
0502      * to run (and have re-enabled preemption)
0503      */
0504     return 1;
0505 }
0506 NOKPROBE_SYMBOL(trampoline_probe_handler);
0507
0508 int arch_trampoline_kprobe(struct kprobe *p)
0509 {
0510     if (p->addr == (kprobe_opcode_t *)__kretprobe_trampoline)
0511         return 1;
0512
0513     return 0;
0514 }
0515 NOKPROBE_SYMBOL(arch_trampoline_kprobe);
0516
0517 static struct kprobe trampoline_p = {
0518     .addr = (kprobe_opcode_t *)__kretprobe_trampoline,
0519     .pre_handler = trampoline_probe_handler
0520 };
0521
0522 int __init arch_init_kprobes(void)
0523 {
0524     return register_kprobe(&trampoline_p);
0525 }