arm64/kernel/debug-monitors.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * ARMv8 single-step debug support and mdscr context switching.
0004  *
0005  * Copyright (C) 2012 ARM Limited
0006  *
0007  * Author: Will Deacon <will.deacon@arm.com>
0008  */
0009
0010 #include <linux/cpu.h>
0011 #include <linux/debugfs.h>
0012 #include <linux/hardirq.h>
0013 #include <linux/init.h>
0014 #include <linux/ptrace.h>
0015 #include <linux/kprobes.h>
0016 #include <linux/stat.h>
0017 #include <linux/uaccess.h>
0018 #include <linux/sched/task_stack.h>
0019
0020 #include <asm/cpufeature.h>
0021 #include <asm/cputype.h>
0022 #include <asm/daifflags.h>
0023 #include <asm/debug-monitors.h>
0024 #include <asm/system_misc.h>
0025 #include <asm/traps.h>
0026
0027 /* Determine debug architecture. */
0028 u8 debug_monitors_arch(void)
0029 {
0030     return cpuid_feature_extract_unsigned_field(read_sanitised_ftr_reg(SYS_ID_AA64DFR0_EL1),
0031                         ID_AA64DFR0_DEBUGVER_SHIFT);
0032 }
0033
0034 /*
0035  * MDSCR access routines.
0036  */
0037 static void mdscr_write(u32 mdscr)
0038 {
0039     unsigned long flags;
0040     flags = local_daif_save();
0041     write_sysreg(mdscr, mdscr_el1);
0042     local_daif_restore(flags);
0043 }
0044 NOKPROBE_SYMBOL(mdscr_write);
0045
0046 static u32 mdscr_read(void)
0047 {
0048     return read_sysreg(mdscr_el1);
0049 }
0050 NOKPROBE_SYMBOL(mdscr_read);
0051
0052 /*
0053  * Allow root to disable self-hosted debug from userspace.
0054  * This is useful if you want to connect an external JTAG debugger.
0055  */
0056 static bool debug_enabled = true;
0057
0058 static int create_debug_debugfs_entry(void)
0059 {
0060     debugfs_create_bool("debug_enabled", 0644, NULL, &debug_enabled);
0061     return 0;
0062 }
0063 fs_initcall(create_debug_debugfs_entry);
0064
0065 static int __init early_debug_disable(char *buf)
0066 {
0067     debug_enabled = false;
0068     return 0;
0069 }
0070
0071 early_param("nodebugmon", early_debug_disable);
0072
0073 /*
0074  * Keep track of debug users on each core.
0075  * The ref counts are per-cpu so we use a local_t type.
0076  */
0077 static DEFINE_PER_CPU(int, mde_ref_count);
0078 static DEFINE_PER_CPU(int, kde_ref_count);
0079
0080 void enable_debug_monitors(enum dbg_active_el el)
0081 {
0082     u32 mdscr, enable = 0;
0083
0084     WARN_ON(preemptible());
0085
0086     if (this_cpu_inc_return(mde_ref_count) == 1)
0087         enable = DBG_MDSCR_MDE;
0088
0089     if (el == DBG_ACTIVE_EL1 &&
0090         this_cpu_inc_return(kde_ref_count) == 1)
0091         enable |= DBG_MDSCR_KDE;
0092
0093     if (enable && debug_enabled) {
0094         mdscr = mdscr_read();
0095         mdscr |= enable;
0096         mdscr_write(mdscr);
0097     }
0098 }
0099 NOKPROBE_SYMBOL(enable_debug_monitors);
0100
0101 void disable_debug_monitors(enum dbg_active_el el)
0102 {
0103     u32 mdscr, disable = 0;
0104
0105     WARN_ON(preemptible());
0106
0107     if (this_cpu_dec_return(mde_ref_count) == 0)
0108         disable = ~DBG_MDSCR_MDE;
0109
0110     if (el == DBG_ACTIVE_EL1 &&
0111         this_cpu_dec_return(kde_ref_count) == 0)
0112         disable &= ~DBG_MDSCR_KDE;
0113
0114     if (disable) {
0115         mdscr = mdscr_read();
0116         mdscr &= disable;
0117         mdscr_write(mdscr);
0118     }
0119 }
0120 NOKPROBE_SYMBOL(disable_debug_monitors);
0121
0122 /*
0123  * OS lock clearing.
0124  */
0125 static int clear_os_lock(unsigned int cpu)
0126 {
0127     write_sysreg(0, osdlr_el1);
0128     write_sysreg(0, oslar_el1);
0129     isb();
0130     return 0;
0131 }
0132
0133 static int __init debug_monitors_init(void)
0134 {
0135     return cpuhp_setup_state(CPUHP_AP_ARM64_DEBUG_MONITORS_STARTING,
0136                  "arm64/debug_monitors:starting",
0137                  clear_os_lock, NULL);
0138 }
0139 postcore_initcall(debug_monitors_init);
0140
0141 /*
0142  * Single step API and exception handling.
0143  */
0144 static void set_user_regs_spsr_ss(struct user_pt_regs *regs)
0145 {
0146     regs->pstate |= DBG_SPSR_SS;
0147 }
0148 NOKPROBE_SYMBOL(set_user_regs_spsr_ss);
0149
0150 static void clear_user_regs_spsr_ss(struct user_pt_regs *regs)
0151 {
0152     regs->pstate &= ~DBG_SPSR_SS;
0153 }
0154 NOKPROBE_SYMBOL(clear_user_regs_spsr_ss);
0155
0156 #define set_regs_spsr_ss(r) set_user_regs_spsr_ss(&(r)->user_regs)
0157 #define clear_regs_spsr_ss(r)   clear_user_regs_spsr_ss(&(r)->user_regs)
0158
0159 static DEFINE_SPINLOCK(debug_hook_lock);
0160 static LIST_HEAD(user_step_hook);
0161 static LIST_HEAD(kernel_step_hook);
0162
0163 static void register_debug_hook(struct list_head *node, struct list_head *list)
0164 {
0165     spin_lock(&debug_hook_lock);
0166     list_add_rcu(node, list);
0167     spin_unlock(&debug_hook_lock);
0168
0169 }
0170
0171 static void unregister_debug_hook(struct list_head *node)
0172 {
0173     spin_lock(&debug_hook_lock);
0174     list_del_rcu(node);
0175     spin_unlock(&debug_hook_lock);
0176     synchronize_rcu();
0177 }
0178
0179 void register_user_step_hook(struct step_hook *hook)
0180 {
0181     register_debug_hook(&hook->node, &user_step_hook);
0182 }
0183
0184 void unregister_user_step_hook(struct step_hook *hook)
0185 {
0186     unregister_debug_hook(&hook->node);
0187 }
0188
0189 void register_kernel_step_hook(struct step_hook *hook)
0190 {
0191     register_debug_hook(&hook->node, &kernel_step_hook);
0192 }
0193
0194 void unregister_kernel_step_hook(struct step_hook *hook)
0195 {
0196     unregister_debug_hook(&hook->node);
0197 }
0198
0199 /*
0200  * Call registered single step handlers
0201  * There is no Syndrome info to check for determining the handler.
0202  * So we call all the registered handlers, until the right handler is
0203  * found which returns zero.
0204  */
0205 static int call_step_hook(struct pt_regs *regs, unsigned long esr)
0206 {
0207     struct step_hook *hook;
0208     struct list_head *list;
0209     int retval = DBG_HOOK_ERROR;
0210
0211     list = user_mode(regs) ? &user_step_hook : &kernel_step_hook;
0212
0213     /*
0214      * Since single-step exception disables interrupt, this function is
0215      * entirely not preemptible, and we can use rcu list safely here.
0216      */
0217     list_for_each_entry_rcu(hook, list, node)   {
0218         retval = hook->fn(regs, esr);
0219         if (retval == DBG_HOOK_HANDLED)
0220             break;
0221     }
0222
0223     return retval;
0224 }
0225 NOKPROBE_SYMBOL(call_step_hook);
0226
0227 static void send_user_sigtrap(int si_code)
0228 {
0229     struct pt_regs *regs = current_pt_regs();
0230
0231     if (WARN_ON(!user_mode(regs)))
0232         return;
0233
0234     if (interrupts_enabled(regs))
0235         local_irq_enable();
0236
0237     arm64_force_sig_fault(SIGTRAP, si_code, instruction_pointer(regs),
0238                   "User debug trap");
0239 }
0240
0241 static int single_step_handler(unsigned long unused, unsigned long esr,
0242                    struct pt_regs *regs)
0243 {
0244     bool handler_found = false;
0245
0246     /*
0247      * If we are stepping a pending breakpoint, call the hw_breakpoint
0248      * handler first.
0249      */
0250     if (!reinstall_suspended_bps(regs))
0251         return 0;
0252
0253     if (!handler_found && call_step_hook(regs, esr) == DBG_HOOK_HANDLED)
0254         handler_found = true;
0255
0256     if (!handler_found && user_mode(regs)) {
0257         send_user_sigtrap(TRAP_TRACE);
0258
0259         /*
0260          * ptrace will disable single step unless explicitly
0261          * asked to re-enable it. For other clients, it makes
0262          * sense to leave it enabled (i.e. rewind the controls
0263          * to the active-not-pending state).
0264          */
0265         user_rewind_single_step(current);
0266     } else if (!handler_found) {
0267         pr_warn("Unexpected kernel single-step exception at EL1\n");
0268         /*
0269          * Re-enable stepping since we know that we will be
0270          * returning to regs.
0271          */
0272         set_regs_spsr_ss(regs);
0273     }
0274
0275     return 0;
0276 }
0277 NOKPROBE_SYMBOL(single_step_handler);
0278
0279 static LIST_HEAD(user_break_hook);
0280 static LIST_HEAD(kernel_break_hook);
0281
0282 void register_user_break_hook(struct break_hook *hook)
0283 {
0284     register_debug_hook(&hook->node, &user_break_hook);
0285 }
0286
0287 void unregister_user_break_hook(struct break_hook *hook)
0288 {
0289     unregister_debug_hook(&hook->node);
0290 }
0291
0292 void register_kernel_break_hook(struct break_hook *hook)
0293 {
0294     register_debug_hook(&hook->node, &kernel_break_hook);
0295 }
0296
0297 void unregister_kernel_break_hook(struct break_hook *hook)
0298 {
0299     unregister_debug_hook(&hook->node);
0300 }
0301
0302 static int call_break_hook(struct pt_regs *regs, unsigned long esr)
0303 {
0304     struct break_hook *hook;
0305     struct list_head *list;
0306     int (*fn)(struct pt_regs *regs, unsigned long esr) = NULL;
0307
0308     list = user_mode(regs) ? &user_break_hook : &kernel_break_hook;
0309
0310     /*
0311      * Since brk exception disables interrupt, this function is
0312      * entirely not preemptible, and we can use rcu list safely here.
0313      */
0314     list_for_each_entry_rcu(hook, list, node) {
0315         unsigned long comment = esr & ESR_ELx_BRK64_ISS_COMMENT_MASK;
0316
0317         if ((comment & ~hook->mask) == hook->imm)
0318             fn = hook->fn;
0319     }
0320
0321     return fn ? fn(regs, esr) : DBG_HOOK_ERROR;
0322 }
0323 NOKPROBE_SYMBOL(call_break_hook);
0324
0325 static int brk_handler(unsigned long unused, unsigned long esr,
0326                struct pt_regs *regs)
0327 {
0328     if (call_break_hook(regs, esr) == DBG_HOOK_HANDLED)
0329         return 0;
0330
0331     if (user_mode(regs)) {
0332         send_user_sigtrap(TRAP_BRKPT);
0333     } else {
0334         pr_warn("Unexpected kernel BRK exception at EL1\n");
0335         return -EFAULT;
0336     }
0337
0338     return 0;
0339 }
0340 NOKPROBE_SYMBOL(brk_handler);
0341
0342 int aarch32_break_handler(struct pt_regs *regs)
0343 {
0344     u32 arm_instr;
0345     u16 thumb_instr;
0346     bool bp = false;
0347     void __user *pc = (void __user *)instruction_pointer(regs);
0348
0349     if (!compat_user_mode(regs))
0350         return -EFAULT;
0351
0352     if (compat_thumb_mode(regs)) {
0353         /* get 16-bit Thumb instruction */
0354         __le16 instr;
0355         get_user(instr, (__le16 __user *)pc);
0356         thumb_instr = le16_to_cpu(instr);
0357         if (thumb_instr == AARCH32_BREAK_THUMB2_LO) {
0358             /* get second half of 32-bit Thumb-2 instruction */
0359             get_user(instr, (__le16 __user *)(pc + 2));
0360             thumb_instr = le16_to_cpu(instr);
0361             bp = thumb_instr == AARCH32_BREAK_THUMB2_HI;
0362         } else {
0363             bp = thumb_instr == AARCH32_BREAK_THUMB;
0364         }
0365     } else {
0366         /* 32-bit ARM instruction */
0367         __le32 instr;
0368         get_user(instr, (__le32 __user *)pc);
0369         arm_instr = le32_to_cpu(instr);
0370         bp = (arm_instr & ~0xf0000000) == AARCH32_BREAK_ARM;
0371     }
0372
0373     if (!bp)
0374         return -EFAULT;
0375
0376     send_user_sigtrap(TRAP_BRKPT);
0377     return 0;
0378 }
0379 NOKPROBE_SYMBOL(aarch32_break_handler);
0380
0381 void __init debug_traps_init(void)
0382 {
0383     hook_debug_fault_code(DBG_ESR_EVT_HWSS, single_step_handler, SIGTRAP,
0384                   TRAP_TRACE, "single-step handler");
0385     hook_debug_fault_code(DBG_ESR_EVT_BRK, brk_handler, SIGTRAP,
0386                   TRAP_BRKPT, "BRK handler");
0387 }
0388
0389 /* Re-enable single step for syscall restarting. */
0390 void user_rewind_single_step(struct task_struct *task)
0391 {
0392     /*
0393      * If single step is active for this thread, then set SPSR.SS
0394      * to 1 to avoid returning to the active-pending state.
0395      */
0396     if (test_tsk_thread_flag(task, TIF_SINGLESTEP))
0397         set_regs_spsr_ss(task_pt_regs(task));
0398 }
0399 NOKPROBE_SYMBOL(user_rewind_single_step);
0400
0401 void user_fastforward_single_step(struct task_struct *task)
0402 {
0403     if (test_tsk_thread_flag(task, TIF_SINGLESTEP))
0404         clear_regs_spsr_ss(task_pt_regs(task));
0405 }
0406
0407 void user_regs_reset_single_step(struct user_pt_regs *regs,
0408                  struct task_struct *task)
0409 {
0410     if (test_tsk_thread_flag(task, TIF_SINGLESTEP))
0411         set_user_regs_spsr_ss(regs);
0412     else
0413         clear_user_regs_spsr_ss(regs);
0414 }
0415
0416 /* Kernel API */
0417 void kernel_enable_single_step(struct pt_regs *regs)
0418 {
0419     WARN_ON(!irqs_disabled());
0420     set_regs_spsr_ss(regs);
0421     mdscr_write(mdscr_read() | DBG_MDSCR_SS);
0422     enable_debug_monitors(DBG_ACTIVE_EL1);
0423 }
0424 NOKPROBE_SYMBOL(kernel_enable_single_step);
0425
0426 void kernel_disable_single_step(void)
0427 {
0428     WARN_ON(!irqs_disabled());
0429     mdscr_write(mdscr_read() & ~DBG_MDSCR_SS);
0430     disable_debug_monitors(DBG_ACTIVE_EL1);
0431 }
0432 NOKPROBE_SYMBOL(kernel_disable_single_step);
0433
0434 int kernel_active_single_step(void)
0435 {
0436     WARN_ON(!irqs_disabled());
0437     return mdscr_read() & DBG_MDSCR_SS;
0438 }
0439 NOKPROBE_SYMBOL(kernel_active_single_step);
0440
0441 /* ptrace API */
0442 void user_enable_single_step(struct task_struct *task)
0443 {
0444     struct thread_info *ti = task_thread_info(task);
0445
0446     if (!test_and_set_ti_thread_flag(ti, TIF_SINGLESTEP))
0447         set_regs_spsr_ss(task_pt_regs(task));
0448 }
0449 NOKPROBE_SYMBOL(user_enable_single_step);
0450
0451 void user_disable_single_step(struct task_struct *task)
0452 {
0453     clear_ti_thread_flag(task_thread_info(task), TIF_SINGLESTEP);
0454 }
0455 NOKPROBE_SYMBOL(user_disable_single_step);