Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  *
0004  * Copyright (C) 2009, 2010 ARM Limited
0005  *
0006  * Author: Will Deacon <will.deacon@arm.com>
0007  */
0008 
0009 /*
0010  * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility,
0011  * using the CPU's debug registers.
0012  */
0013 #define pr_fmt(fmt) "hw-breakpoint: " fmt
0014 
0015 #include <linux/errno.h>
0016 #include <linux/hardirq.h>
0017 #include <linux/perf_event.h>
0018 #include <linux/hw_breakpoint.h>
0019 #include <linux/smp.h>
0020 #include <linux/cpu_pm.h>
0021 #include <linux/coresight.h>
0022 
0023 #include <asm/cacheflush.h>
0024 #include <asm/cputype.h>
0025 #include <asm/current.h>
0026 #include <asm/hw_breakpoint.h>
0027 #include <asm/traps.h>
0028 
0029 /* Breakpoint currently in use for each BRP. */
0030 static DEFINE_PER_CPU(struct perf_event *, bp_on_reg[ARM_MAX_BRP]);
0031 
0032 /* Watchpoint currently in use for each WRP. */
0033 static DEFINE_PER_CPU(struct perf_event *, wp_on_reg[ARM_MAX_WRP]);
0034 
0035 /* Number of BRP/WRP registers on this CPU. */
0036 static int core_num_brps __ro_after_init;
0037 static int core_num_wrps __ro_after_init;
0038 
0039 /* Debug architecture version. */
0040 static u8 debug_arch __ro_after_init;
0041 
0042 /* Does debug architecture support OS Save and Restore? */
0043 static bool has_ossr __ro_after_init;
0044 
0045 /* Maximum supported watchpoint length. */
0046 static u8 max_watchpoint_len __ro_after_init;
0047 
0048 #define READ_WB_REG_CASE(OP2, M, VAL)           \
0049     case ((OP2 << 4) + M):              \
0050         ARM_DBG_READ(c0, c ## M, OP2, VAL); \
0051         break
0052 
0053 #define WRITE_WB_REG_CASE(OP2, M, VAL)          \
0054     case ((OP2 << 4) + M):              \
0055         ARM_DBG_WRITE(c0, c ## M, OP2, VAL);    \
0056         break
0057 
0058 #define GEN_READ_WB_REG_CASES(OP2, VAL)     \
0059     READ_WB_REG_CASE(OP2, 0, VAL);      \
0060     READ_WB_REG_CASE(OP2, 1, VAL);      \
0061     READ_WB_REG_CASE(OP2, 2, VAL);      \
0062     READ_WB_REG_CASE(OP2, 3, VAL);      \
0063     READ_WB_REG_CASE(OP2, 4, VAL);      \
0064     READ_WB_REG_CASE(OP2, 5, VAL);      \
0065     READ_WB_REG_CASE(OP2, 6, VAL);      \
0066     READ_WB_REG_CASE(OP2, 7, VAL);      \
0067     READ_WB_REG_CASE(OP2, 8, VAL);      \
0068     READ_WB_REG_CASE(OP2, 9, VAL);      \
0069     READ_WB_REG_CASE(OP2, 10, VAL);     \
0070     READ_WB_REG_CASE(OP2, 11, VAL);     \
0071     READ_WB_REG_CASE(OP2, 12, VAL);     \
0072     READ_WB_REG_CASE(OP2, 13, VAL);     \
0073     READ_WB_REG_CASE(OP2, 14, VAL);     \
0074     READ_WB_REG_CASE(OP2, 15, VAL)
0075 
0076 #define GEN_WRITE_WB_REG_CASES(OP2, VAL)    \
0077     WRITE_WB_REG_CASE(OP2, 0, VAL);     \
0078     WRITE_WB_REG_CASE(OP2, 1, VAL);     \
0079     WRITE_WB_REG_CASE(OP2, 2, VAL);     \
0080     WRITE_WB_REG_CASE(OP2, 3, VAL);     \
0081     WRITE_WB_REG_CASE(OP2, 4, VAL);     \
0082     WRITE_WB_REG_CASE(OP2, 5, VAL);     \
0083     WRITE_WB_REG_CASE(OP2, 6, VAL);     \
0084     WRITE_WB_REG_CASE(OP2, 7, VAL);     \
0085     WRITE_WB_REG_CASE(OP2, 8, VAL);     \
0086     WRITE_WB_REG_CASE(OP2, 9, VAL);     \
0087     WRITE_WB_REG_CASE(OP2, 10, VAL);    \
0088     WRITE_WB_REG_CASE(OP2, 11, VAL);    \
0089     WRITE_WB_REG_CASE(OP2, 12, VAL);    \
0090     WRITE_WB_REG_CASE(OP2, 13, VAL);    \
0091     WRITE_WB_REG_CASE(OP2, 14, VAL);    \
0092     WRITE_WB_REG_CASE(OP2, 15, VAL)
0093 
0094 static u32 read_wb_reg(int n)
0095 {
0096     u32 val = 0;
0097 
0098     switch (n) {
0099     GEN_READ_WB_REG_CASES(ARM_OP2_BVR, val);
0100     GEN_READ_WB_REG_CASES(ARM_OP2_BCR, val);
0101     GEN_READ_WB_REG_CASES(ARM_OP2_WVR, val);
0102     GEN_READ_WB_REG_CASES(ARM_OP2_WCR, val);
0103     default:
0104         pr_warn("attempt to read from unknown breakpoint register %d\n",
0105             n);
0106     }
0107 
0108     return val;
0109 }
0110 
0111 static void write_wb_reg(int n, u32 val)
0112 {
0113     switch (n) {
0114     GEN_WRITE_WB_REG_CASES(ARM_OP2_BVR, val);
0115     GEN_WRITE_WB_REG_CASES(ARM_OP2_BCR, val);
0116     GEN_WRITE_WB_REG_CASES(ARM_OP2_WVR, val);
0117     GEN_WRITE_WB_REG_CASES(ARM_OP2_WCR, val);
0118     default:
0119         pr_warn("attempt to write to unknown breakpoint register %d\n",
0120             n);
0121     }
0122     isb();
0123 }
0124 
0125 /* Determine debug architecture. */
0126 static u8 get_debug_arch(void)
0127 {
0128     u32 didr;
0129 
0130     /* Do we implement the extended CPUID interface? */
0131     if (((read_cpuid_id() >> 16) & 0xf) != 0xf) {
0132         pr_warn_once("CPUID feature registers not supported. "
0133                  "Assuming v6 debug is present.\n");
0134         return ARM_DEBUG_ARCH_V6;
0135     }
0136 
0137     ARM_DBG_READ(c0, c0, 0, didr);
0138     return (didr >> 16) & 0xf;
0139 }
0140 
0141 u8 arch_get_debug_arch(void)
0142 {
0143     return debug_arch;
0144 }
0145 
0146 static int debug_arch_supported(void)
0147 {
0148     u8 arch = get_debug_arch();
0149 
0150     /* We don't support the memory-mapped interface. */
0151     return (arch >= ARM_DEBUG_ARCH_V6 && arch <= ARM_DEBUG_ARCH_V7_ECP14) ||
0152         arch >= ARM_DEBUG_ARCH_V7_1;
0153 }
0154 
0155 /* Can we determine the watchpoint access type from the fsr? */
0156 static int debug_exception_updates_fsr(void)
0157 {
0158     return get_debug_arch() >= ARM_DEBUG_ARCH_V8;
0159 }
0160 
0161 /* Determine number of WRP registers available. */
0162 static int get_num_wrp_resources(void)
0163 {
0164     u32 didr;
0165     ARM_DBG_READ(c0, c0, 0, didr);
0166     return ((didr >> 28) & 0xf) + 1;
0167 }
0168 
0169 /* Determine number of BRP registers available. */
0170 static int get_num_brp_resources(void)
0171 {
0172     u32 didr;
0173     ARM_DBG_READ(c0, c0, 0, didr);
0174     return ((didr >> 24) & 0xf) + 1;
0175 }
0176 
0177 /* Does this core support mismatch breakpoints? */
0178 static int core_has_mismatch_brps(void)
0179 {
0180     return (get_debug_arch() >= ARM_DEBUG_ARCH_V7_ECP14 &&
0181         get_num_brp_resources() > 1);
0182 }
0183 
0184 /* Determine number of usable WRPs available. */
0185 static int get_num_wrps(void)
0186 {
0187     /*
0188      * On debug architectures prior to 7.1, when a watchpoint fires, the
0189      * only way to work out which watchpoint it was is by disassembling
0190      * the faulting instruction and working out the address of the memory
0191      * access.
0192      *
0193      * Furthermore, we can only do this if the watchpoint was precise
0194      * since imprecise watchpoints prevent us from calculating register
0195      * based addresses.
0196      *
0197      * Providing we have more than 1 breakpoint register, we only report
0198      * a single watchpoint register for the time being. This way, we always
0199      * know which watchpoint fired. In the future we can either add a
0200      * disassembler and address generation emulator, or we can insert a
0201      * check to see if the DFAR is set on watchpoint exception entry
0202      * [the ARM ARM states that the DFAR is UNKNOWN, but experience shows
0203      * that it is set on some implementations].
0204      */
0205     if (get_debug_arch() < ARM_DEBUG_ARCH_V7_1)
0206         return 1;
0207 
0208     return get_num_wrp_resources();
0209 }
0210 
0211 /* Determine number of usable BRPs available. */
0212 static int get_num_brps(void)
0213 {
0214     int brps = get_num_brp_resources();
0215     return core_has_mismatch_brps() ? brps - 1 : brps;
0216 }
0217 
0218 /*
0219  * In order to access the breakpoint/watchpoint control registers,
0220  * we must be running in debug monitor mode. Unfortunately, we can
0221  * be put into halting debug mode at any time by an external debugger
0222  * but there is nothing we can do to prevent that.
0223  */
0224 static int monitor_mode_enabled(void)
0225 {
0226     u32 dscr;
0227     ARM_DBG_READ(c0, c1, 0, dscr);
0228     return !!(dscr & ARM_DSCR_MDBGEN);
0229 }
0230 
0231 static int enable_monitor_mode(void)
0232 {
0233     u32 dscr;
0234     ARM_DBG_READ(c0, c1, 0, dscr);
0235 
0236     /* If monitor mode is already enabled, just return. */
0237     if (dscr & ARM_DSCR_MDBGEN)
0238         goto out;
0239 
0240     /* Write to the corresponding DSCR. */
0241     switch (get_debug_arch()) {
0242     case ARM_DEBUG_ARCH_V6:
0243     case ARM_DEBUG_ARCH_V6_1:
0244         ARM_DBG_WRITE(c0, c1, 0, (dscr | ARM_DSCR_MDBGEN));
0245         break;
0246     case ARM_DEBUG_ARCH_V7_ECP14:
0247     case ARM_DEBUG_ARCH_V7_1:
0248     case ARM_DEBUG_ARCH_V8:
0249     case ARM_DEBUG_ARCH_V8_1:
0250     case ARM_DEBUG_ARCH_V8_2:
0251     case ARM_DEBUG_ARCH_V8_4:
0252         ARM_DBG_WRITE(c0, c2, 2, (dscr | ARM_DSCR_MDBGEN));
0253         isb();
0254         break;
0255     default:
0256         return -ENODEV;
0257     }
0258 
0259     /* Check that the write made it through. */
0260     ARM_DBG_READ(c0, c1, 0, dscr);
0261     if (!(dscr & ARM_DSCR_MDBGEN)) {
0262         pr_warn_once("Failed to enable monitor mode on CPU %d.\n",
0263                 smp_processor_id());
0264         return -EPERM;
0265     }
0266 
0267 out:
0268     return 0;
0269 }
0270 
0271 int hw_breakpoint_slots(int type)
0272 {
0273     if (!debug_arch_supported())
0274         return 0;
0275 
0276     /*
0277      * We can be called early, so don't rely on
0278      * our static variables being initialised.
0279      */
0280     switch (type) {
0281     case TYPE_INST:
0282         return get_num_brps();
0283     case TYPE_DATA:
0284         return get_num_wrps();
0285     default:
0286         pr_warn("unknown slot type: %d\n", type);
0287         return 0;
0288     }
0289 }
0290 
0291 /*
0292  * Check if 8-bit byte-address select is available.
0293  * This clobbers WRP 0.
0294  */
0295 static u8 get_max_wp_len(void)
0296 {
0297     u32 ctrl_reg;
0298     struct arch_hw_breakpoint_ctrl ctrl;
0299     u8 size = 4;
0300 
0301     if (debug_arch < ARM_DEBUG_ARCH_V7_ECP14)
0302         goto out;
0303 
0304     memset(&ctrl, 0, sizeof(ctrl));
0305     ctrl.len = ARM_BREAKPOINT_LEN_8;
0306     ctrl_reg = encode_ctrl_reg(ctrl);
0307 
0308     write_wb_reg(ARM_BASE_WVR, 0);
0309     write_wb_reg(ARM_BASE_WCR, ctrl_reg);
0310     if ((read_wb_reg(ARM_BASE_WCR) & ctrl_reg) == ctrl_reg)
0311         size = 8;
0312 
0313 out:
0314     return size;
0315 }
0316 
0317 u8 arch_get_max_wp_len(void)
0318 {
0319     return max_watchpoint_len;
0320 }
0321 
0322 /*
0323  * Install a perf counter breakpoint.
0324  */
0325 int arch_install_hw_breakpoint(struct perf_event *bp)
0326 {
0327     struct arch_hw_breakpoint *info = counter_arch_bp(bp);
0328     struct perf_event **slot, **slots;
0329     int i, max_slots, ctrl_base, val_base;
0330     u32 addr, ctrl;
0331 
0332     addr = info->address;
0333     ctrl = encode_ctrl_reg(info->ctrl) | 0x1;
0334 
0335     if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) {
0336         /* Breakpoint */
0337         ctrl_base = ARM_BASE_BCR;
0338         val_base = ARM_BASE_BVR;
0339         slots = this_cpu_ptr(bp_on_reg);
0340         max_slots = core_num_brps;
0341     } else {
0342         /* Watchpoint */
0343         ctrl_base = ARM_BASE_WCR;
0344         val_base = ARM_BASE_WVR;
0345         slots = this_cpu_ptr(wp_on_reg);
0346         max_slots = core_num_wrps;
0347     }
0348 
0349     for (i = 0; i < max_slots; ++i) {
0350         slot = &slots[i];
0351 
0352         if (!*slot) {
0353             *slot = bp;
0354             break;
0355         }
0356     }
0357 
0358     if (i == max_slots) {
0359         pr_warn("Can't find any breakpoint slot\n");
0360         return -EBUSY;
0361     }
0362 
0363     /* Override the breakpoint data with the step data. */
0364     if (info->step_ctrl.enabled) {
0365         addr = info->trigger & ~0x3;
0366         ctrl = encode_ctrl_reg(info->step_ctrl);
0367         if (info->ctrl.type != ARM_BREAKPOINT_EXECUTE) {
0368             i = 0;
0369             ctrl_base = ARM_BASE_BCR + core_num_brps;
0370             val_base = ARM_BASE_BVR + core_num_brps;
0371         }
0372     }
0373 
0374     /* Setup the address register. */
0375     write_wb_reg(val_base + i, addr);
0376 
0377     /* Setup the control register. */
0378     write_wb_reg(ctrl_base + i, ctrl);
0379     return 0;
0380 }
0381 
0382 void arch_uninstall_hw_breakpoint(struct perf_event *bp)
0383 {
0384     struct arch_hw_breakpoint *info = counter_arch_bp(bp);
0385     struct perf_event **slot, **slots;
0386     int i, max_slots, base;
0387 
0388     if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) {
0389         /* Breakpoint */
0390         base = ARM_BASE_BCR;
0391         slots = this_cpu_ptr(bp_on_reg);
0392         max_slots = core_num_brps;
0393     } else {
0394         /* Watchpoint */
0395         base = ARM_BASE_WCR;
0396         slots = this_cpu_ptr(wp_on_reg);
0397         max_slots = core_num_wrps;
0398     }
0399 
0400     /* Remove the breakpoint. */
0401     for (i = 0; i < max_slots; ++i) {
0402         slot = &slots[i];
0403 
0404         if (*slot == bp) {
0405             *slot = NULL;
0406             break;
0407         }
0408     }
0409 
0410     if (i == max_slots) {
0411         pr_warn("Can't find any breakpoint slot\n");
0412         return;
0413     }
0414 
0415     /* Ensure that we disable the mismatch breakpoint. */
0416     if (info->ctrl.type != ARM_BREAKPOINT_EXECUTE &&
0417         info->step_ctrl.enabled) {
0418         i = 0;
0419         base = ARM_BASE_BCR + core_num_brps;
0420     }
0421 
0422     /* Reset the control register. */
0423     write_wb_reg(base + i, 0);
0424 }
0425 
0426 static int get_hbp_len(u8 hbp_len)
0427 {
0428     unsigned int len_in_bytes = 0;
0429 
0430     switch (hbp_len) {
0431     case ARM_BREAKPOINT_LEN_1:
0432         len_in_bytes = 1;
0433         break;
0434     case ARM_BREAKPOINT_LEN_2:
0435         len_in_bytes = 2;
0436         break;
0437     case ARM_BREAKPOINT_LEN_4:
0438         len_in_bytes = 4;
0439         break;
0440     case ARM_BREAKPOINT_LEN_8:
0441         len_in_bytes = 8;
0442         break;
0443     }
0444 
0445     return len_in_bytes;
0446 }
0447 
0448 /*
0449  * Check whether bp virtual address is in kernel space.
0450  */
0451 int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw)
0452 {
0453     unsigned int len;
0454     unsigned long va;
0455 
0456     va = hw->address;
0457     len = get_hbp_len(hw->ctrl.len);
0458 
0459     return (va >= TASK_SIZE) && ((va + len - 1) >= TASK_SIZE);
0460 }
0461 
0462 /*
0463  * Extract generic type and length encodings from an arch_hw_breakpoint_ctrl.
0464  * Hopefully this will disappear when ptrace can bypass the conversion
0465  * to generic breakpoint descriptions.
0466  */
0467 int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl,
0468                int *gen_len, int *gen_type)
0469 {
0470     /* Type */
0471     switch (ctrl.type) {
0472     case ARM_BREAKPOINT_EXECUTE:
0473         *gen_type = HW_BREAKPOINT_X;
0474         break;
0475     case ARM_BREAKPOINT_LOAD:
0476         *gen_type = HW_BREAKPOINT_R;
0477         break;
0478     case ARM_BREAKPOINT_STORE:
0479         *gen_type = HW_BREAKPOINT_W;
0480         break;
0481     case ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE:
0482         *gen_type = HW_BREAKPOINT_RW;
0483         break;
0484     default:
0485         return -EINVAL;
0486     }
0487 
0488     /* Len */
0489     switch (ctrl.len) {
0490     case ARM_BREAKPOINT_LEN_1:
0491         *gen_len = HW_BREAKPOINT_LEN_1;
0492         break;
0493     case ARM_BREAKPOINT_LEN_2:
0494         *gen_len = HW_BREAKPOINT_LEN_2;
0495         break;
0496     case ARM_BREAKPOINT_LEN_4:
0497         *gen_len = HW_BREAKPOINT_LEN_4;
0498         break;
0499     case ARM_BREAKPOINT_LEN_8:
0500         *gen_len = HW_BREAKPOINT_LEN_8;
0501         break;
0502     default:
0503         return -EINVAL;
0504     }
0505 
0506     return 0;
0507 }
0508 
0509 /*
0510  * Construct an arch_hw_breakpoint from a perf_event.
0511  */
0512 static int arch_build_bp_info(struct perf_event *bp,
0513                   const struct perf_event_attr *attr,
0514                   struct arch_hw_breakpoint *hw)
0515 {
0516     /* Type */
0517     switch (attr->bp_type) {
0518     case HW_BREAKPOINT_X:
0519         hw->ctrl.type = ARM_BREAKPOINT_EXECUTE;
0520         break;
0521     case HW_BREAKPOINT_R:
0522         hw->ctrl.type = ARM_BREAKPOINT_LOAD;
0523         break;
0524     case HW_BREAKPOINT_W:
0525         hw->ctrl.type = ARM_BREAKPOINT_STORE;
0526         break;
0527     case HW_BREAKPOINT_RW:
0528         hw->ctrl.type = ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE;
0529         break;
0530     default:
0531         return -EINVAL;
0532     }
0533 
0534     /* Len */
0535     switch (attr->bp_len) {
0536     case HW_BREAKPOINT_LEN_1:
0537         hw->ctrl.len = ARM_BREAKPOINT_LEN_1;
0538         break;
0539     case HW_BREAKPOINT_LEN_2:
0540         hw->ctrl.len = ARM_BREAKPOINT_LEN_2;
0541         break;
0542     case HW_BREAKPOINT_LEN_4:
0543         hw->ctrl.len = ARM_BREAKPOINT_LEN_4;
0544         break;
0545     case HW_BREAKPOINT_LEN_8:
0546         hw->ctrl.len = ARM_BREAKPOINT_LEN_8;
0547         if ((hw->ctrl.type != ARM_BREAKPOINT_EXECUTE)
0548             && max_watchpoint_len >= 8)
0549             break;
0550         fallthrough;
0551     default:
0552         return -EINVAL;
0553     }
0554 
0555     /*
0556      * Breakpoints must be of length 2 (thumb) or 4 (ARM) bytes.
0557      * Watchpoints can be of length 1, 2, 4 or 8 bytes if supported
0558      * by the hardware and must be aligned to the appropriate number of
0559      * bytes.
0560      */
0561     if (hw->ctrl.type == ARM_BREAKPOINT_EXECUTE &&
0562         hw->ctrl.len != ARM_BREAKPOINT_LEN_2 &&
0563         hw->ctrl.len != ARM_BREAKPOINT_LEN_4)
0564         return -EINVAL;
0565 
0566     /* Address */
0567     hw->address = attr->bp_addr;
0568 
0569     /* Privilege */
0570     hw->ctrl.privilege = ARM_BREAKPOINT_USER;
0571     if (arch_check_bp_in_kernelspace(hw))
0572         hw->ctrl.privilege |= ARM_BREAKPOINT_PRIV;
0573 
0574     /* Enabled? */
0575     hw->ctrl.enabled = !attr->disabled;
0576 
0577     /* Mismatch */
0578     hw->ctrl.mismatch = 0;
0579 
0580     return 0;
0581 }
0582 
0583 /*
0584  * Validate the arch-specific HW Breakpoint register settings.
0585  */
0586 int hw_breakpoint_arch_parse(struct perf_event *bp,
0587                  const struct perf_event_attr *attr,
0588                  struct arch_hw_breakpoint *hw)
0589 {
0590     int ret = 0;
0591     u32 offset, alignment_mask = 0x3;
0592 
0593     /* Ensure that we are in monitor debug mode. */
0594     if (!monitor_mode_enabled())
0595         return -ENODEV;
0596 
0597     /* Build the arch_hw_breakpoint. */
0598     ret = arch_build_bp_info(bp, attr, hw);
0599     if (ret)
0600         goto out;
0601 
0602     /* Check address alignment. */
0603     if (hw->ctrl.len == ARM_BREAKPOINT_LEN_8)
0604         alignment_mask = 0x7;
0605     offset = hw->address & alignment_mask;
0606     switch (offset) {
0607     case 0:
0608         /* Aligned */
0609         break;
0610     case 1:
0611     case 2:
0612         /* Allow halfword watchpoints and breakpoints. */
0613         if (hw->ctrl.len == ARM_BREAKPOINT_LEN_2)
0614             break;
0615         fallthrough;
0616     case 3:
0617         /* Allow single byte watchpoint. */
0618         if (hw->ctrl.len == ARM_BREAKPOINT_LEN_1)
0619             break;
0620         fallthrough;
0621     default:
0622         ret = -EINVAL;
0623         goto out;
0624     }
0625 
0626     hw->address &= ~alignment_mask;
0627     hw->ctrl.len <<= offset;
0628 
0629     if (is_default_overflow_handler(bp)) {
0630         /*
0631          * Mismatch breakpoints are required for single-stepping
0632          * breakpoints.
0633          */
0634         if (!core_has_mismatch_brps())
0635             return -EINVAL;
0636 
0637         /* We don't allow mismatch breakpoints in kernel space. */
0638         if (arch_check_bp_in_kernelspace(hw))
0639             return -EPERM;
0640 
0641         /*
0642          * Per-cpu breakpoints are not supported by our stepping
0643          * mechanism.
0644          */
0645         if (!bp->hw.target)
0646             return -EINVAL;
0647 
0648         /*
0649          * We only support specific access types if the fsr
0650          * reports them.
0651          */
0652         if (!debug_exception_updates_fsr() &&
0653             (hw->ctrl.type == ARM_BREAKPOINT_LOAD ||
0654              hw->ctrl.type == ARM_BREAKPOINT_STORE))
0655             return -EINVAL;
0656     }
0657 
0658 out:
0659     return ret;
0660 }
0661 
0662 /*
0663  * Enable/disable single-stepping over the breakpoint bp at address addr.
0664  */
0665 static void enable_single_step(struct perf_event *bp, u32 addr)
0666 {
0667     struct arch_hw_breakpoint *info = counter_arch_bp(bp);
0668 
0669     arch_uninstall_hw_breakpoint(bp);
0670     info->step_ctrl.mismatch  = 1;
0671     info->step_ctrl.len   = ARM_BREAKPOINT_LEN_4;
0672     info->step_ctrl.type      = ARM_BREAKPOINT_EXECUTE;
0673     info->step_ctrl.privilege = info->ctrl.privilege;
0674     info->step_ctrl.enabled   = 1;
0675     info->trigger         = addr;
0676     arch_install_hw_breakpoint(bp);
0677 }
0678 
0679 static void disable_single_step(struct perf_event *bp)
0680 {
0681     arch_uninstall_hw_breakpoint(bp);
0682     counter_arch_bp(bp)->step_ctrl.enabled = 0;
0683     arch_install_hw_breakpoint(bp);
0684 }
0685 
0686 /*
0687  * Arm32 hardware does not always report a watchpoint hit address that matches
0688  * one of the watchpoints set. It can also report an address "near" the
0689  * watchpoint if a single instruction access both watched and unwatched
0690  * addresses. There is no straight-forward way, short of disassembling the
0691  * offending instruction, to map that address back to the watchpoint. This
0692  * function computes the distance of the memory access from the watchpoint as a
0693  * heuristic for the likelyhood that a given access triggered the watchpoint.
0694  *
0695  * See this same function in the arm64 platform code, which has the same
0696  * problem.
0697  *
0698  * The function returns the distance of the address from the bytes watched by
0699  * the watchpoint. In case of an exact match, it returns 0.
0700  */
0701 static u32 get_distance_from_watchpoint(unsigned long addr, u32 val,
0702                     struct arch_hw_breakpoint_ctrl *ctrl)
0703 {
0704     u32 wp_low, wp_high;
0705     u32 lens, lene;
0706 
0707     lens = __ffs(ctrl->len);
0708     lene = __fls(ctrl->len);
0709 
0710     wp_low = val + lens;
0711     wp_high = val + lene;
0712     if (addr < wp_low)
0713         return wp_low - addr;
0714     else if (addr > wp_high)
0715         return addr - wp_high;
0716     else
0717         return 0;
0718 }
0719 
0720 static int watchpoint_fault_on_uaccess(struct pt_regs *regs,
0721                        struct arch_hw_breakpoint *info)
0722 {
0723     return !user_mode(regs) && info->ctrl.privilege == ARM_BREAKPOINT_USER;
0724 }
0725 
0726 static void watchpoint_handler(unsigned long addr, unsigned int fsr,
0727                    struct pt_regs *regs)
0728 {
0729     int i, access, closest_match = 0;
0730     u32 min_dist = -1, dist;
0731     u32 val, ctrl_reg;
0732     struct perf_event *wp, **slots;
0733     struct arch_hw_breakpoint *info;
0734     struct arch_hw_breakpoint_ctrl ctrl;
0735 
0736     slots = this_cpu_ptr(wp_on_reg);
0737 
0738     /*
0739      * Find all watchpoints that match the reported address. If no exact
0740      * match is found. Attribute the hit to the closest watchpoint.
0741      */
0742     rcu_read_lock();
0743     for (i = 0; i < core_num_wrps; ++i) {
0744         wp = slots[i];
0745         if (wp == NULL)
0746             continue;
0747 
0748         /*
0749          * The DFAR is an unknown value on debug architectures prior
0750          * to 7.1. Since we only allow a single watchpoint on these
0751          * older CPUs, we can set the trigger to the lowest possible
0752          * faulting address.
0753          */
0754         if (debug_arch < ARM_DEBUG_ARCH_V7_1) {
0755             BUG_ON(i > 0);
0756             info = counter_arch_bp(wp);
0757             info->trigger = wp->attr.bp_addr;
0758         } else {
0759             /* Check that the access type matches. */
0760             if (debug_exception_updates_fsr()) {
0761                 access = (fsr & ARM_FSR_ACCESS_MASK) ?
0762                       HW_BREAKPOINT_W : HW_BREAKPOINT_R;
0763                 if (!(access & hw_breakpoint_type(wp)))
0764                     continue;
0765             }
0766 
0767             val = read_wb_reg(ARM_BASE_WVR + i);
0768             ctrl_reg = read_wb_reg(ARM_BASE_WCR + i);
0769             decode_ctrl_reg(ctrl_reg, &ctrl);
0770             dist = get_distance_from_watchpoint(addr, val, &ctrl);
0771             if (dist < min_dist) {
0772                 min_dist = dist;
0773                 closest_match = i;
0774             }
0775             /* Is this an exact match? */
0776             if (dist != 0)
0777                 continue;
0778 
0779             /* We have a winner. */
0780             info = counter_arch_bp(wp);
0781             info->trigger = addr;
0782         }
0783 
0784         pr_debug("watchpoint fired: address = 0x%x\n", info->trigger);
0785 
0786         /*
0787          * If we triggered a user watchpoint from a uaccess routine,
0788          * then handle the stepping ourselves since userspace really
0789          * can't help us with this.
0790          */
0791         if (watchpoint_fault_on_uaccess(regs, info))
0792             goto step;
0793 
0794         perf_bp_event(wp, regs);
0795 
0796         /*
0797          * Defer stepping to the overflow handler if one is installed.
0798          * Otherwise, insert a temporary mismatch breakpoint so that
0799          * we can single-step over the watchpoint trigger.
0800          */
0801         if (!is_default_overflow_handler(wp))
0802             continue;
0803 step:
0804         enable_single_step(wp, instruction_pointer(regs));
0805     }
0806 
0807     if (min_dist > 0 && min_dist != -1) {
0808         /* No exact match found. */
0809         wp = slots[closest_match];
0810         info = counter_arch_bp(wp);
0811         info->trigger = addr;
0812         pr_debug("watchpoint fired: address = 0x%x\n", info->trigger);
0813         perf_bp_event(wp, regs);
0814         if (is_default_overflow_handler(wp))
0815             enable_single_step(wp, instruction_pointer(regs));
0816     }
0817 
0818     rcu_read_unlock();
0819 }
0820 
0821 static void watchpoint_single_step_handler(unsigned long pc)
0822 {
0823     int i;
0824     struct perf_event *wp, **slots;
0825     struct arch_hw_breakpoint *info;
0826 
0827     slots = this_cpu_ptr(wp_on_reg);
0828 
0829     for (i = 0; i < core_num_wrps; ++i) {
0830         rcu_read_lock();
0831 
0832         wp = slots[i];
0833 
0834         if (wp == NULL)
0835             goto unlock;
0836 
0837         info = counter_arch_bp(wp);
0838         if (!info->step_ctrl.enabled)
0839             goto unlock;
0840 
0841         /*
0842          * Restore the original watchpoint if we've completed the
0843          * single-step.
0844          */
0845         if (info->trigger != pc)
0846             disable_single_step(wp);
0847 
0848 unlock:
0849         rcu_read_unlock();
0850     }
0851 }
0852 
0853 static void breakpoint_handler(unsigned long unknown, struct pt_regs *regs)
0854 {
0855     int i;
0856     u32 ctrl_reg, val, addr;
0857     struct perf_event *bp, **slots;
0858     struct arch_hw_breakpoint *info;
0859     struct arch_hw_breakpoint_ctrl ctrl;
0860 
0861     slots = this_cpu_ptr(bp_on_reg);
0862 
0863     /* The exception entry code places the amended lr in the PC. */
0864     addr = regs->ARM_pc;
0865 
0866     /* Check the currently installed breakpoints first. */
0867     for (i = 0; i < core_num_brps; ++i) {
0868         rcu_read_lock();
0869 
0870         bp = slots[i];
0871 
0872         if (bp == NULL)
0873             goto unlock;
0874 
0875         info = counter_arch_bp(bp);
0876 
0877         /* Check if the breakpoint value matches. */
0878         val = read_wb_reg(ARM_BASE_BVR + i);
0879         if (val != (addr & ~0x3))
0880             goto mismatch;
0881 
0882         /* Possible match, check the byte address select to confirm. */
0883         ctrl_reg = read_wb_reg(ARM_BASE_BCR + i);
0884         decode_ctrl_reg(ctrl_reg, &ctrl);
0885         if ((1 << (addr & 0x3)) & ctrl.len) {
0886             info->trigger = addr;
0887             pr_debug("breakpoint fired: address = 0x%x\n", addr);
0888             perf_bp_event(bp, regs);
0889             if (is_default_overflow_handler(bp))
0890                 enable_single_step(bp, addr);
0891             goto unlock;
0892         }
0893 
0894 mismatch:
0895         /* If we're stepping a breakpoint, it can now be restored. */
0896         if (info->step_ctrl.enabled)
0897             disable_single_step(bp);
0898 unlock:
0899         rcu_read_unlock();
0900     }
0901 
0902     /* Handle any pending watchpoint single-step breakpoints. */
0903     watchpoint_single_step_handler(addr);
0904 }
0905 
0906 /*
0907  * Called from either the Data Abort Handler [watchpoint] or the
0908  * Prefetch Abort Handler [breakpoint] with interrupts disabled.
0909  */
0910 static int hw_breakpoint_pending(unsigned long addr, unsigned int fsr,
0911                  struct pt_regs *regs)
0912 {
0913     int ret = 0;
0914     u32 dscr;
0915 
0916     preempt_disable();
0917 
0918     if (interrupts_enabled(regs))
0919         local_irq_enable();
0920 
0921     /* We only handle watchpoints and hardware breakpoints. */
0922     ARM_DBG_READ(c0, c1, 0, dscr);
0923 
0924     /* Perform perf callbacks. */
0925     switch (ARM_DSCR_MOE(dscr)) {
0926     case ARM_ENTRY_BREAKPOINT:
0927         breakpoint_handler(addr, regs);
0928         break;
0929     case ARM_ENTRY_ASYNC_WATCHPOINT:
0930         WARN(1, "Asynchronous watchpoint exception taken. Debugging results may be unreliable\n");
0931         fallthrough;
0932     case ARM_ENTRY_SYNC_WATCHPOINT:
0933         watchpoint_handler(addr, fsr, regs);
0934         break;
0935     default:
0936         ret = 1; /* Unhandled fault. */
0937     }
0938 
0939     preempt_enable();
0940 
0941     return ret;
0942 }
0943 
0944 #ifdef CONFIG_ARM_ERRATA_764319
0945 static int oslsr_fault;
0946 
0947 static int debug_oslsr_trap(struct pt_regs *regs, unsigned int instr)
0948 {
0949     oslsr_fault = 1;
0950     instruction_pointer(regs) += 4;
0951     return 0;
0952 }
0953 
0954 static struct undef_hook debug_oslsr_hook = {
0955     .instr_mask  = 0xffffffff,
0956     .instr_val = 0xee115e91,
0957     .fn = debug_oslsr_trap,
0958 };
0959 #endif
0960 
0961 /*
0962  * One-time initialisation.
0963  */
0964 static cpumask_t debug_err_mask;
0965 
0966 static int debug_reg_trap(struct pt_regs *regs, unsigned int instr)
0967 {
0968     int cpu = smp_processor_id();
0969 
0970     pr_warn("Debug register access (0x%x) caused undefined instruction on CPU %d\n",
0971         instr, cpu);
0972 
0973     /* Set the error flag for this CPU and skip the faulting instruction. */
0974     cpumask_set_cpu(cpu, &debug_err_mask);
0975     instruction_pointer(regs) += 4;
0976     return 0;
0977 }
0978 
0979 static struct undef_hook debug_reg_hook = {
0980     .instr_mask = 0x0fe80f10,
0981     .instr_val  = 0x0e000e10,
0982     .fn     = debug_reg_trap,
0983 };
0984 
0985 /* Does this core support OS Save and Restore? */
0986 static bool core_has_os_save_restore(void)
0987 {
0988     u32 oslsr;
0989 
0990     switch (get_debug_arch()) {
0991     case ARM_DEBUG_ARCH_V7_1:
0992         return true;
0993     case ARM_DEBUG_ARCH_V7_ECP14:
0994 #ifdef CONFIG_ARM_ERRATA_764319
0995         oslsr_fault = 0;
0996         register_undef_hook(&debug_oslsr_hook);
0997         ARM_DBG_READ(c1, c1, 4, oslsr);
0998         unregister_undef_hook(&debug_oslsr_hook);
0999         if (oslsr_fault)
1000             return false;
1001 #else
1002         ARM_DBG_READ(c1, c1, 4, oslsr);
1003 #endif
1004         if (oslsr & ARM_OSLSR_OSLM0)
1005             return true;
1006         fallthrough;
1007     default:
1008         return false;
1009     }
1010 }
1011 
1012 static void reset_ctrl_regs(unsigned int cpu)
1013 {
1014     int i, raw_num_brps, err = 0;
1015     u32 val;
1016 
1017     /*
1018      * v7 debug contains save and restore registers so that debug state
1019      * can be maintained across low-power modes without leaving the debug
1020      * logic powered up. It is IMPLEMENTATION DEFINED whether we can access
1021      * the debug registers out of reset, so we must unlock the OS Lock
1022      * Access Register to avoid taking undefined instruction exceptions
1023      * later on.
1024      */
1025     switch (debug_arch) {
1026     case ARM_DEBUG_ARCH_V6:
1027     case ARM_DEBUG_ARCH_V6_1:
1028         /* ARMv6 cores clear the registers out of reset. */
1029         goto out_mdbgen;
1030     case ARM_DEBUG_ARCH_V7_ECP14:
1031         /*
1032          * Ensure sticky power-down is clear (i.e. debug logic is
1033          * powered up).
1034          */
1035         ARM_DBG_READ(c1, c5, 4, val);
1036         if ((val & 0x1) == 0)
1037             err = -EPERM;
1038 
1039         if (!has_ossr)
1040             goto clear_vcr;
1041         break;
1042     case ARM_DEBUG_ARCH_V7_1:
1043         /*
1044          * Ensure the OS double lock is clear.
1045          */
1046         ARM_DBG_READ(c1, c3, 4, val);
1047         if ((val & 0x1) == 1)
1048             err = -EPERM;
1049         break;
1050     }
1051 
1052     if (err) {
1053         pr_warn_once("CPU %d debug is powered down!\n", cpu);
1054         cpumask_or(&debug_err_mask, &debug_err_mask, cpumask_of(cpu));
1055         return;
1056     }
1057 
1058     /*
1059      * Unconditionally clear the OS lock by writing a value
1060      * other than CS_LAR_KEY to the access register.
1061      */
1062     ARM_DBG_WRITE(c1, c0, 4, ~CORESIGHT_UNLOCK);
1063     isb();
1064 
1065     /*
1066      * Clear any configured vector-catch events before
1067      * enabling monitor mode.
1068      */
1069 clear_vcr:
1070     ARM_DBG_WRITE(c0, c7, 0, 0);
1071     isb();
1072 
1073     if (cpumask_intersects(&debug_err_mask, cpumask_of(cpu))) {
1074         pr_warn_once("CPU %d failed to disable vector catch\n", cpu);
1075         return;
1076     }
1077 
1078     /*
1079      * The control/value register pairs are UNKNOWN out of reset so
1080      * clear them to avoid spurious debug events.
1081      */
1082     raw_num_brps = get_num_brp_resources();
1083     for (i = 0; i < raw_num_brps; ++i) {
1084         write_wb_reg(ARM_BASE_BCR + i, 0UL);
1085         write_wb_reg(ARM_BASE_BVR + i, 0UL);
1086     }
1087 
1088     for (i = 0; i < core_num_wrps; ++i) {
1089         write_wb_reg(ARM_BASE_WCR + i, 0UL);
1090         write_wb_reg(ARM_BASE_WVR + i, 0UL);
1091     }
1092 
1093     if (cpumask_intersects(&debug_err_mask, cpumask_of(cpu))) {
1094         pr_warn_once("CPU %d failed to clear debug register pairs\n", cpu);
1095         return;
1096     }
1097 
1098     /*
1099      * Have a crack at enabling monitor mode. We don't actually need
1100      * it yet, but reporting an error early is useful if it fails.
1101      */
1102 out_mdbgen:
1103     if (enable_monitor_mode())
1104         cpumask_or(&debug_err_mask, &debug_err_mask, cpumask_of(cpu));
1105 }
1106 
1107 static int dbg_reset_online(unsigned int cpu)
1108 {
1109     local_irq_disable();
1110     reset_ctrl_regs(cpu);
1111     local_irq_enable();
1112     return 0;
1113 }
1114 
1115 #ifdef CONFIG_CPU_PM
1116 static int dbg_cpu_pm_notify(struct notifier_block *self, unsigned long action,
1117                  void *v)
1118 {
1119     if (action == CPU_PM_EXIT)
1120         reset_ctrl_regs(smp_processor_id());
1121 
1122     return NOTIFY_OK;
1123 }
1124 
1125 static struct notifier_block dbg_cpu_pm_nb = {
1126     .notifier_call = dbg_cpu_pm_notify,
1127 };
1128 
1129 static void __init pm_init(void)
1130 {
1131     cpu_pm_register_notifier(&dbg_cpu_pm_nb);
1132 }
1133 #else
1134 static inline void pm_init(void)
1135 {
1136 }
1137 #endif
1138 
1139 static int __init arch_hw_breakpoint_init(void)
1140 {
1141     int ret;
1142 
1143     debug_arch = get_debug_arch();
1144 
1145     if (!debug_arch_supported()) {
1146         pr_info("debug architecture 0x%x unsupported.\n", debug_arch);
1147         return 0;
1148     }
1149 
1150     /*
1151      * Scorpion CPUs (at least those in APQ8060) seem to set DBGPRSR.SPD
1152      * whenever a WFI is issued, even if the core is not powered down, in
1153      * violation of the architecture.  When DBGPRSR.SPD is set, accesses to
1154      * breakpoint and watchpoint registers are treated as undefined, so
1155      * this results in boot time and runtime failures when these are
1156      * accessed and we unexpectedly take a trap.
1157      *
1158      * It's not clear if/how this can be worked around, so we blacklist
1159      * Scorpion CPUs to avoid these issues.
1160     */
1161     if (read_cpuid_part() == ARM_CPU_PART_SCORPION) {
1162         pr_info("Scorpion CPU detected. Hardware breakpoints and watchpoints disabled\n");
1163         return 0;
1164     }
1165 
1166     has_ossr = core_has_os_save_restore();
1167 
1168     /* Determine how many BRPs/WRPs are available. */
1169     core_num_brps = get_num_brps();
1170     core_num_wrps = get_num_wrps();
1171 
1172     /*
1173      * We need to tread carefully here because DBGSWENABLE may be
1174      * driven low on this core and there isn't an architected way to
1175      * determine that.
1176      */
1177     cpus_read_lock();
1178     register_undef_hook(&debug_reg_hook);
1179 
1180     /*
1181      * Register CPU notifier which resets the breakpoint resources. We
1182      * assume that a halting debugger will leave the world in a nice state
1183      * for us.
1184      */
1185     ret = cpuhp_setup_state_cpuslocked(CPUHP_AP_ONLINE_DYN,
1186                        "arm/hw_breakpoint:online",
1187                        dbg_reset_online, NULL);
1188     unregister_undef_hook(&debug_reg_hook);
1189     if (WARN_ON(ret < 0) || !cpumask_empty(&debug_err_mask)) {
1190         core_num_brps = 0;
1191         core_num_wrps = 0;
1192         if (ret > 0)
1193             cpuhp_remove_state_nocalls_cpuslocked(ret);
1194         cpus_read_unlock();
1195         return 0;
1196     }
1197 
1198     pr_info("found %d " "%s" "breakpoint and %d watchpoint registers.\n",
1199         core_num_brps, core_has_mismatch_brps() ? "(+1 reserved) " :
1200         "", core_num_wrps);
1201 
1202     /* Work out the maximum supported watchpoint length. */
1203     max_watchpoint_len = get_max_wp_len();
1204     pr_info("maximum watchpoint size is %u bytes.\n",
1205             max_watchpoint_len);
1206 
1207     /* Register debug fault handler. */
1208     hook_fault_code(FAULT_CODE_DEBUG, hw_breakpoint_pending, SIGTRAP,
1209             TRAP_HWBKPT, "watchpoint debug exception");
1210     hook_ifault_code(FAULT_CODE_DEBUG, hw_breakpoint_pending, SIGTRAP,
1211             TRAP_HWBKPT, "breakpoint debug exception");
1212     cpus_read_unlock();
1213 
1214     /* Register PM notifiers. */
1215     pm_init();
1216     return 0;
1217 }
1218 arch_initcall(arch_hw_breakpoint_init);
1219 
1220 void hw_breakpoint_pmu_read(struct perf_event *bp)
1221 {
1222 }
1223 
1224 /*
1225  * Dummy function to register with die_notifier.
1226  */
1227 int hw_breakpoint_exceptions_notify(struct notifier_block *unused,
1228                     unsigned long val, void *data)
1229 {
1230     return NOTIFY_DONE;
1231 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!