kernel/events/hw_breakpoint.c

0001 // SPDX-License-Identifier: GPL-2.0+
0002 /*
0003  * Copyright (C) 2007 Alan Stern
0004  * Copyright (C) IBM Corporation, 2009
0005  * Copyright (C) 2009, Frederic Weisbecker <fweisbec@gmail.com>
0006  *
0007  * Thanks to Ingo Molnar for his many suggestions.
0008  *
0009  * Authors: Alan Stern <stern@rowland.harvard.edu>
0010  *          K.Prasad <prasad@linux.vnet.ibm.com>
0011  *          Frederic Weisbecker <fweisbec@gmail.com>
0012  */
0013
0014 /*
0015  * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility,
0016  * using the CPU's debug registers.
0017  * This file contains the arch-independent routines.
0018  */
0019
0020 #include <linux/irqflags.h>
0021 #include <linux/kallsyms.h>
0022 #include <linux/notifier.h>
0023 #include <linux/kprobes.h>
0024 #include <linux/kdebug.h>
0025 #include <linux/kernel.h>
0026 #include <linux/module.h>
0027 #include <linux/percpu.h>
0028 #include <linux/sched.h>
0029 #include <linux/init.h>
0030 #include <linux/slab.h>
0031 #include <linux/list.h>
0032 #include <linux/cpu.h>
0033 #include <linux/smp.h>
0034 #include <linux/bug.h>
0035
0036 #include <linux/hw_breakpoint.h>
0037 /*
0038  * Constraints data
0039  */
0040 struct bp_cpuinfo {
0041     /* Number of pinned cpu breakpoints in a cpu */
0042     unsigned int    cpu_pinned;
0043     /* tsk_pinned[n] is the number of tasks having n+1 breakpoints */
0044     unsigned int    *tsk_pinned;
0045     /* Number of non-pinned cpu/task breakpoints in a cpu */
0046     unsigned int    flexible; /* XXX: placeholder, see fetch_this_slot() */
0047 };
0048
0049 static DEFINE_PER_CPU(struct bp_cpuinfo, bp_cpuinfo[TYPE_MAX]);
0050 static int nr_slots[TYPE_MAX];
0051
0052 static struct bp_cpuinfo *get_bp_info(int cpu, enum bp_type_idx type)
0053 {
0054     return per_cpu_ptr(bp_cpuinfo + type, cpu);
0055 }
0056
0057 /* Keep track of the breakpoints attached to tasks */
0058 static LIST_HEAD(bp_task_head);
0059
0060 static int constraints_initialized;
0061
0062 /* Gather the number of total pinned and un-pinned bp in a cpuset */
0063 struct bp_busy_slots {
0064     unsigned int pinned;
0065     unsigned int flexible;
0066 };
0067
0068 /* Serialize accesses to the above constraints */
0069 static DEFINE_MUTEX(nr_bp_mutex);
0070
0071 __weak int hw_breakpoint_weight(struct perf_event *bp)
0072 {
0073     return 1;
0074 }
0075
0076 static inline enum bp_type_idx find_slot_idx(u64 bp_type)
0077 {
0078     if (bp_type & HW_BREAKPOINT_RW)
0079         return TYPE_DATA;
0080
0081     return TYPE_INST;
0082 }
0083
0084 /*
0085  * Report the maximum number of pinned breakpoints a task
0086  * have in this cpu
0087  */
0088 static unsigned int max_task_bp_pinned(int cpu, enum bp_type_idx type)
0089 {
0090     unsigned int *tsk_pinned = get_bp_info(cpu, type)->tsk_pinned;
0091     int i;
0092
0093     for (i = nr_slots[type] - 1; i >= 0; i--) {
0094         if (tsk_pinned[i] > 0)
0095             return i + 1;
0096     }
0097
0098     return 0;
0099 }
0100
0101 /*
0102  * Count the number of breakpoints of the same type and same task.
0103  * The given event must be not on the list.
0104  */
0105 static int task_bp_pinned(int cpu, struct perf_event *bp, enum bp_type_idx type)
0106 {
0107     struct task_struct *tsk = bp->hw.target;
0108     struct perf_event *iter;
0109     int count = 0;
0110
0111     list_for_each_entry(iter, &bp_task_head, hw.bp_list) {
0112         if (iter->hw.target == tsk &&
0113             find_slot_idx(iter->attr.bp_type) == type &&
0114             (iter->cpu < 0 || cpu == iter->cpu))
0115             count += hw_breakpoint_weight(iter);
0116     }
0117
0118     return count;
0119 }
0120
0121 static const struct cpumask *cpumask_of_bp(struct perf_event *bp)
0122 {
0123     if (bp->cpu >= 0)
0124         return cpumask_of(bp->cpu);
0125     return cpu_possible_mask;
0126 }
0127
0128 /*
0129  * Report the number of pinned/un-pinned breakpoints we have in
0130  * a given cpu (cpu > -1) or in all of them (cpu = -1).
0131  */
0132 static void
0133 fetch_bp_busy_slots(struct bp_busy_slots *slots, struct perf_event *bp,
0134             enum bp_type_idx type)
0135 {
0136     const struct cpumask *cpumask = cpumask_of_bp(bp);
0137     int cpu;
0138
0139     for_each_cpu(cpu, cpumask) {
0140         struct bp_cpuinfo *info = get_bp_info(cpu, type);
0141         int nr;
0142
0143         nr = info->cpu_pinned;
0144         if (!bp->hw.target)
0145             nr += max_task_bp_pinned(cpu, type);
0146         else
0147             nr += task_bp_pinned(cpu, bp, type);
0148
0149         if (nr > slots->pinned)
0150             slots->pinned = nr;
0151
0152         nr = info->flexible;
0153         if (nr > slots->flexible)
0154             slots->flexible = nr;
0155     }
0156 }
0157
0158 /*
0159  * For now, continue to consider flexible as pinned, until we can
0160  * ensure no flexible event can ever be scheduled before a pinned event
0161  * in a same cpu.
0162  */
0163 static void
0164 fetch_this_slot(struct bp_busy_slots *slots, int weight)
0165 {
0166     slots->pinned += weight;
0167 }
0168
0169 /*
0170  * Add a pinned breakpoint for the given task in our constraint table
0171  */
0172 static void toggle_bp_task_slot(struct perf_event *bp, int cpu,
0173                 enum bp_type_idx type, int weight)
0174 {
0175     unsigned int *tsk_pinned = get_bp_info(cpu, type)->tsk_pinned;
0176     int old_idx, new_idx;
0177
0178     old_idx = task_bp_pinned(cpu, bp, type) - 1;
0179     new_idx = old_idx + weight;
0180
0181     if (old_idx >= 0)
0182         tsk_pinned[old_idx]--;
0183     if (new_idx >= 0)
0184         tsk_pinned[new_idx]++;
0185 }
0186
0187 /*
0188  * Add/remove the given breakpoint in our constraint table
0189  */
0190 static void
0191 toggle_bp_slot(struct perf_event *bp, bool enable, enum bp_type_idx type,
0192            int weight)
0193 {
0194     const struct cpumask *cpumask = cpumask_of_bp(bp);
0195     int cpu;
0196
0197     if (!enable)
0198         weight = -weight;
0199
0200     /* Pinned counter cpu profiling */
0201     if (!bp->hw.target) {
0202         get_bp_info(bp->cpu, type)->cpu_pinned += weight;
0203         return;
0204     }
0205
0206     /* Pinned counter task profiling */
0207     for_each_cpu(cpu, cpumask)
0208         toggle_bp_task_slot(bp, cpu, type, weight);
0209
0210     if (enable)
0211         list_add_tail(&bp->hw.bp_list, &bp_task_head);
0212     else
0213         list_del(&bp->hw.bp_list);
0214 }
0215
0216 __weak int arch_reserve_bp_slot(struct perf_event *bp)
0217 {
0218     return 0;
0219 }
0220
0221 __weak void arch_release_bp_slot(struct perf_event *bp)
0222 {
0223 }
0224
0225 /*
0226  * Function to perform processor-specific cleanup during unregistration
0227  */
0228 __weak void arch_unregister_hw_breakpoint(struct perf_event *bp)
0229 {
0230     /*
0231      * A weak stub function here for those archs that don't define
0232      * it inside arch/.../kernel/hw_breakpoint.c
0233      */
0234 }
0235
0236 /*
0237  * Constraints to check before allowing this new breakpoint counter:
0238  *
0239  *  == Non-pinned counter == (Considered as pinned for now)
0240  *
0241  *   - If attached to a single cpu, check:
0242  *
0243  *       (per_cpu(info->flexible, cpu) || (per_cpu(info->cpu_pinned, cpu)
0244  *           + max(per_cpu(info->tsk_pinned, cpu)))) < HBP_NUM
0245  *
0246  *       -> If there are already non-pinned counters in this cpu, it means
0247  *          there is already a free slot for them.
0248  *          Otherwise, we check that the maximum number of per task
0249  *          breakpoints (for this cpu) plus the number of per cpu breakpoint
0250  *          (for this cpu) doesn't cover every registers.
0251  *
0252  *   - If attached to every cpus, check:
0253  *
0254  *       (per_cpu(info->flexible, *) || (max(per_cpu(info->cpu_pinned, *))
0255  *           + max(per_cpu(info->tsk_pinned, *)))) < HBP_NUM
0256  *
0257  *       -> This is roughly the same, except we check the number of per cpu
0258  *          bp for every cpu and we keep the max one. Same for the per tasks
0259  *          breakpoints.
0260  *
0261  *
0262  * == Pinned counter ==
0263  *
0264  *   - If attached to a single cpu, check:
0265  *
0266  *       ((per_cpu(info->flexible, cpu) > 1) + per_cpu(info->cpu_pinned, cpu)
0267  *            + max(per_cpu(info->tsk_pinned, cpu))) < HBP_NUM
0268  *
0269  *       -> Same checks as before. But now the info->flexible, if any, must keep
0270  *          one register at least (or they will never be fed).
0271  *
0272  *   - If attached to every cpus, check:
0273  *
0274  *       ((per_cpu(info->flexible, *) > 1) + max(per_cpu(info->cpu_pinned, *))
0275  *            + max(per_cpu(info->tsk_pinned, *))) < HBP_NUM
0276  */
0277 static int __reserve_bp_slot(struct perf_event *bp, u64 bp_type)
0278 {
0279     struct bp_busy_slots slots = {0};
0280     enum bp_type_idx type;
0281     int weight;
0282     int ret;
0283
0284     /* We couldn't initialize breakpoint constraints on boot */
0285     if (!constraints_initialized)
0286         return -ENOMEM;
0287
0288     /* Basic checks */
0289     if (bp_type == HW_BREAKPOINT_EMPTY ||
0290         bp_type == HW_BREAKPOINT_INVALID)
0291         return -EINVAL;
0292
0293     type = find_slot_idx(bp_type);
0294     weight = hw_breakpoint_weight(bp);
0295
0296     fetch_bp_busy_slots(&slots, bp, type);
0297     /*
0298      * Simulate the addition of this breakpoint to the constraints
0299      * and see the result.
0300      */
0301     fetch_this_slot(&slots, weight);
0302
0303     /* Flexible counters need to keep at least one slot */
0304     if (slots.pinned + (!!slots.flexible) > nr_slots[type])
0305         return -ENOSPC;
0306
0307     ret = arch_reserve_bp_slot(bp);
0308     if (ret)
0309         return ret;
0310
0311     toggle_bp_slot(bp, true, type, weight);
0312
0313     return 0;
0314 }
0315
0316 int reserve_bp_slot(struct perf_event *bp)
0317 {
0318     int ret;
0319
0320     mutex_lock(&nr_bp_mutex);
0321
0322     ret = __reserve_bp_slot(bp, bp->attr.bp_type);
0323
0324     mutex_unlock(&nr_bp_mutex);
0325
0326     return ret;
0327 }
0328
0329 static void __release_bp_slot(struct perf_event *bp, u64 bp_type)
0330 {
0331     enum bp_type_idx type;
0332     int weight;
0333
0334     arch_release_bp_slot(bp);
0335
0336     type = find_slot_idx(bp_type);
0337     weight = hw_breakpoint_weight(bp);
0338     toggle_bp_slot(bp, false, type, weight);
0339 }
0340
0341 void release_bp_slot(struct perf_event *bp)
0342 {
0343     mutex_lock(&nr_bp_mutex);
0344
0345     arch_unregister_hw_breakpoint(bp);
0346     __release_bp_slot(bp, bp->attr.bp_type);
0347
0348     mutex_unlock(&nr_bp_mutex);
0349 }
0350
0351 static int __modify_bp_slot(struct perf_event *bp, u64 old_type, u64 new_type)
0352 {
0353     int err;
0354
0355     __release_bp_slot(bp, old_type);
0356
0357     err = __reserve_bp_slot(bp, new_type);
0358     if (err) {
0359         /*
0360          * Reserve the old_type slot back in case
0361          * there's no space for the new type.
0362          *
0363          * This must succeed, because we just released
0364          * the old_type slot in the __release_bp_slot
0365          * call above. If not, something is broken.
0366          */
0367         WARN_ON(__reserve_bp_slot(bp, old_type));
0368     }
0369
0370     return err;
0371 }
0372
0373 static int modify_bp_slot(struct perf_event *bp, u64 old_type, u64 new_type)
0374 {
0375     int ret;
0376
0377     mutex_lock(&nr_bp_mutex);
0378     ret = __modify_bp_slot(bp, old_type, new_type);
0379     mutex_unlock(&nr_bp_mutex);
0380     return ret;
0381 }
0382
0383 /*
0384  * Allow the kernel debugger to reserve breakpoint slots without
0385  * taking a lock using the dbg_* variant of for the reserve and
0386  * release breakpoint slots.
0387  */
0388 int dbg_reserve_bp_slot(struct perf_event *bp)
0389 {
0390     if (mutex_is_locked(&nr_bp_mutex))
0391         return -1;
0392
0393     return __reserve_bp_slot(bp, bp->attr.bp_type);
0394 }
0395
0396 int dbg_release_bp_slot(struct perf_event *bp)
0397 {
0398     if (mutex_is_locked(&nr_bp_mutex))
0399         return -1;
0400
0401     __release_bp_slot(bp, bp->attr.bp_type);
0402
0403     return 0;
0404 }
0405
0406 static int hw_breakpoint_parse(struct perf_event *bp,
0407                    const struct perf_event_attr *attr,
0408                    struct arch_hw_breakpoint *hw)
0409 {
0410     int err;
0411
0412     err = hw_breakpoint_arch_parse(bp, attr, hw);
0413     if (err)
0414         return err;
0415
0416     if (arch_check_bp_in_kernelspace(hw)) {
0417         if (attr->exclude_kernel)
0418             return -EINVAL;
0419         /*
0420          * Don't let unprivileged users set a breakpoint in the trap
0421          * path to avoid trap recursion attacks.
0422          */
0423         if (!capable(CAP_SYS_ADMIN))
0424             return -EPERM;
0425     }
0426
0427     return 0;
0428 }
0429
0430 int register_perf_hw_breakpoint(struct perf_event *bp)
0431 {
0432     struct arch_hw_breakpoint hw = { };
0433     int err;
0434
0435     err = reserve_bp_slot(bp);
0436     if (err)
0437         return err;
0438
0439     err = hw_breakpoint_parse(bp, &bp->attr, &hw);
0440     if (err) {
0441         release_bp_slot(bp);
0442         return err;
0443     }
0444
0445     bp->hw.info = hw;
0446
0447     return 0;
0448 }
0449
0450 /**
0451  * register_user_hw_breakpoint - register a hardware breakpoint for user space
0452  * @attr: breakpoint attributes
0453  * @triggered: callback to trigger when we hit the breakpoint
0454  * @context: context data could be used in the triggered callback
0455  * @tsk: pointer to 'task_struct' of the process to which the address belongs
0456  */
0457 struct perf_event *
0458 register_user_hw_breakpoint(struct perf_event_attr *attr,
0459                 perf_overflow_handler_t triggered,
0460                 void *context,
0461                 struct task_struct *tsk)
0462 {
0463     return perf_event_create_kernel_counter(attr, -1, tsk, triggered,
0464                         context);
0465 }
0466 EXPORT_SYMBOL_GPL(register_user_hw_breakpoint);
0467
0468 static void hw_breakpoint_copy_attr(struct perf_event_attr *to,
0469                     struct perf_event_attr *from)
0470 {
0471     to->bp_addr = from->bp_addr;
0472     to->bp_type = from->bp_type;
0473     to->bp_len  = from->bp_len;
0474     to->disabled = from->disabled;
0475 }
0476
0477 int
0478 modify_user_hw_breakpoint_check(struct perf_event *bp, struct perf_event_attr *attr,
0479                     bool check)
0480 {
0481     struct arch_hw_breakpoint hw = { };
0482     int err;
0483
0484     err = hw_breakpoint_parse(bp, attr, &hw);
0485     if (err)
0486         return err;
0487
0488     if (check) {
0489         struct perf_event_attr old_attr;
0490
0491         old_attr = bp->attr;
0492         hw_breakpoint_copy_attr(&old_attr, attr);
0493         if (memcmp(&old_attr, attr, sizeof(*attr)))
0494             return -EINVAL;
0495     }
0496
0497     if (bp->attr.bp_type != attr->bp_type) {
0498         err = modify_bp_slot(bp, bp->attr.bp_type, attr->bp_type);
0499         if (err)
0500             return err;
0501     }
0502
0503     hw_breakpoint_copy_attr(&bp->attr, attr);
0504     bp->hw.info = hw;
0505
0506     return 0;
0507 }
0508
0509 /**
0510  * modify_user_hw_breakpoint - modify a user-space hardware breakpoint
0511  * @bp: the breakpoint structure to modify
0512  * @attr: new breakpoint attributes
0513  */
0514 int modify_user_hw_breakpoint(struct perf_event *bp, struct perf_event_attr *attr)
0515 {
0516     int err;
0517
0518     /*
0519      * modify_user_hw_breakpoint can be invoked with IRQs disabled and hence it
0520      * will not be possible to raise IPIs that invoke __perf_event_disable.
0521      * So call the function directly after making sure we are targeting the
0522      * current task.
0523      */
0524     if (irqs_disabled() && bp->ctx && bp->ctx->task == current)
0525         perf_event_disable_local(bp);
0526     else
0527         perf_event_disable(bp);
0528
0529     err = modify_user_hw_breakpoint_check(bp, attr, false);
0530
0531     if (!bp->attr.disabled)
0532         perf_event_enable(bp);
0533
0534     return err;
0535 }
0536 EXPORT_SYMBOL_GPL(modify_user_hw_breakpoint);
0537
0538 /**
0539  * unregister_hw_breakpoint - unregister a user-space hardware breakpoint
0540  * @bp: the breakpoint structure to unregister
0541  */
0542 void unregister_hw_breakpoint(struct perf_event *bp)
0543 {
0544     if (!bp)
0545         return;
0546     perf_event_release_kernel(bp);
0547 }
0548 EXPORT_SYMBOL_GPL(unregister_hw_breakpoint);
0549
0550 /**
0551  * register_wide_hw_breakpoint - register a wide breakpoint in the kernel
0552  * @attr: breakpoint attributes
0553  * @triggered: callback to trigger when we hit the breakpoint
0554  * @context: context data could be used in the triggered callback
0555  *
0556  * @return a set of per_cpu pointers to perf events
0557  */
0558 struct perf_event * __percpu *
0559 register_wide_hw_breakpoint(struct perf_event_attr *attr,
0560                 perf_overflow_handler_t triggered,
0561                 void *context)
0562 {
0563     struct perf_event * __percpu *cpu_events, *bp;
0564     long err = 0;
0565     int cpu;
0566
0567     cpu_events = alloc_percpu(typeof(*cpu_events));
0568     if (!cpu_events)
0569         return (void __percpu __force *)ERR_PTR(-ENOMEM);
0570
0571     cpus_read_lock();
0572     for_each_online_cpu(cpu) {
0573         bp = perf_event_create_kernel_counter(attr, cpu, NULL,
0574                               triggered, context);
0575         if (IS_ERR(bp)) {
0576             err = PTR_ERR(bp);
0577             break;
0578         }
0579
0580         per_cpu(*cpu_events, cpu) = bp;
0581     }
0582     cpus_read_unlock();
0583
0584     if (likely(!err))
0585         return cpu_events;
0586
0587     unregister_wide_hw_breakpoint(cpu_events);
0588     return (void __percpu __force *)ERR_PTR(err);
0589 }
0590 EXPORT_SYMBOL_GPL(register_wide_hw_breakpoint);
0591
0592 /**
0593  * unregister_wide_hw_breakpoint - unregister a wide breakpoint in the kernel
0594  * @cpu_events: the per cpu set of events to unregister
0595  */
0596 void unregister_wide_hw_breakpoint(struct perf_event * __percpu *cpu_events)
0597 {
0598     int cpu;
0599
0600     for_each_possible_cpu(cpu)
0601         unregister_hw_breakpoint(per_cpu(*cpu_events, cpu));
0602
0603     free_percpu(cpu_events);
0604 }
0605 EXPORT_SYMBOL_GPL(unregister_wide_hw_breakpoint);
0606
0607 static struct notifier_block hw_breakpoint_exceptions_nb = {
0608     .notifier_call = hw_breakpoint_exceptions_notify,
0609     /* we need to be notified first */
0610     .priority = 0x7fffffff
0611 };
0612
0613 static void bp_perf_event_destroy(struct perf_event *event)
0614 {
0615     release_bp_slot(event);
0616 }
0617
0618 static int hw_breakpoint_event_init(struct perf_event *bp)
0619 {
0620     int err;
0621
0622     if (bp->attr.type != PERF_TYPE_BREAKPOINT)
0623         return -ENOENT;
0624
0625     /*
0626      * no branch sampling for breakpoint events
0627      */
0628     if (has_branch_stack(bp))
0629         return -EOPNOTSUPP;
0630
0631     err = register_perf_hw_breakpoint(bp);
0632     if (err)
0633         return err;
0634
0635     bp->destroy = bp_perf_event_destroy;
0636
0637     return 0;
0638 }
0639
0640 static int hw_breakpoint_add(struct perf_event *bp, int flags)
0641 {
0642     if (!(flags & PERF_EF_START))
0643         bp->hw.state = PERF_HES_STOPPED;
0644
0645     if (is_sampling_event(bp)) {
0646         bp->hw.last_period = bp->hw.sample_period;
0647         perf_swevent_set_period(bp);
0648     }
0649
0650     return arch_install_hw_breakpoint(bp);
0651 }
0652
0653 static void hw_breakpoint_del(struct perf_event *bp, int flags)
0654 {
0655     arch_uninstall_hw_breakpoint(bp);
0656 }
0657
0658 static void hw_breakpoint_start(struct perf_event *bp, int flags)
0659 {
0660     bp->hw.state = 0;
0661 }
0662
0663 static void hw_breakpoint_stop(struct perf_event *bp, int flags)
0664 {
0665     bp->hw.state = PERF_HES_STOPPED;
0666 }
0667
0668 static struct pmu perf_breakpoint = {
0669     .task_ctx_nr    = perf_sw_context, /* could eventually get its own */
0670
0671     .event_init = hw_breakpoint_event_init,
0672     .add        = hw_breakpoint_add,
0673     .del        = hw_breakpoint_del,
0674     .start      = hw_breakpoint_start,
0675     .stop       = hw_breakpoint_stop,
0676     .read       = hw_breakpoint_pmu_read,
0677 };
0678
0679 int __init init_hw_breakpoint(void)
0680 {
0681     int cpu, err_cpu;
0682     int i;
0683
0684     for (i = 0; i < TYPE_MAX; i++)
0685         nr_slots[i] = hw_breakpoint_slots(i);
0686
0687     for_each_possible_cpu(cpu) {
0688         for (i = 0; i < TYPE_MAX; i++) {
0689             struct bp_cpuinfo *info = get_bp_info(cpu, i);
0690
0691             info->tsk_pinned = kcalloc(nr_slots[i], sizeof(int),
0692                             GFP_KERNEL);
0693             if (!info->tsk_pinned)
0694                 goto err_alloc;
0695         }
0696     }
0697
0698     constraints_initialized = 1;
0699
0700     perf_pmu_register(&perf_breakpoint, "breakpoint", PERF_TYPE_BREAKPOINT);
0701
0702     return register_die_notifier(&hw_breakpoint_exceptions_nb);
0703
0704  err_alloc:
0705     for_each_possible_cpu(err_cpu) {
0706         for (i = 0; i < TYPE_MAX; i++)
0707             kfree(get_bp_info(err_cpu, i)->tsk_pinned);
0708         if (err_cpu == cpu)
0709             break;
0710     }
0711
0712     return -ENOMEM;
0713 }
0714
0715