Back to home page

LXR

 
 

    


0001 /*
0002  * Performance events callchain code, extracted from core.c:
0003  *
0004  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
0005  *  Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
0006  *  Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra
0007  *  Copyright    2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
0008  *
0009  * For licensing details see kernel-base/COPYING
0010  */
0011 
0012 #include <linux/perf_event.h>
0013 #include <linux/slab.h>
0014 #include "internal.h"
0015 
0016 struct callchain_cpus_entries {
0017     struct rcu_head         rcu_head;
0018     struct perf_callchain_entry *cpu_entries[0];
0019 };
0020 
0021 int sysctl_perf_event_max_stack __read_mostly = PERF_MAX_STACK_DEPTH;
0022 int sysctl_perf_event_max_contexts_per_stack __read_mostly = PERF_MAX_CONTEXTS_PER_STACK;
0023 
0024 static inline size_t perf_callchain_entry__sizeof(void)
0025 {
0026     return (sizeof(struct perf_callchain_entry) +
0027         sizeof(__u64) * (sysctl_perf_event_max_stack +
0028                  sysctl_perf_event_max_contexts_per_stack));
0029 }
0030 
0031 static DEFINE_PER_CPU(int, callchain_recursion[PERF_NR_CONTEXTS]);
0032 static atomic_t nr_callchain_events;
0033 static DEFINE_MUTEX(callchain_mutex);
0034 static struct callchain_cpus_entries *callchain_cpus_entries;
0035 
0036 
0037 __weak void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry,
0038                   struct pt_regs *regs)
0039 {
0040 }
0041 
0042 __weak void perf_callchain_user(struct perf_callchain_entry_ctx *entry,
0043                 struct pt_regs *regs)
0044 {
0045 }
0046 
0047 static void release_callchain_buffers_rcu(struct rcu_head *head)
0048 {
0049     struct callchain_cpus_entries *entries;
0050     int cpu;
0051 
0052     entries = container_of(head, struct callchain_cpus_entries, rcu_head);
0053 
0054     for_each_possible_cpu(cpu)
0055         kfree(entries->cpu_entries[cpu]);
0056 
0057     kfree(entries);
0058 }
0059 
0060 static void release_callchain_buffers(void)
0061 {
0062     struct callchain_cpus_entries *entries;
0063 
0064     entries = callchain_cpus_entries;
0065     RCU_INIT_POINTER(callchain_cpus_entries, NULL);
0066     call_rcu(&entries->rcu_head, release_callchain_buffers_rcu);
0067 }
0068 
0069 static int alloc_callchain_buffers(void)
0070 {
0071     int cpu;
0072     int size;
0073     struct callchain_cpus_entries *entries;
0074 
0075     /*
0076      * We can't use the percpu allocation API for data that can be
0077      * accessed from NMI. Use a temporary manual per cpu allocation
0078      * until that gets sorted out.
0079      */
0080     size = offsetof(struct callchain_cpus_entries, cpu_entries[nr_cpu_ids]);
0081 
0082     entries = kzalloc(size, GFP_KERNEL);
0083     if (!entries)
0084         return -ENOMEM;
0085 
0086     size = perf_callchain_entry__sizeof() * PERF_NR_CONTEXTS;
0087 
0088     for_each_possible_cpu(cpu) {
0089         entries->cpu_entries[cpu] = kmalloc_node(size, GFP_KERNEL,
0090                              cpu_to_node(cpu));
0091         if (!entries->cpu_entries[cpu])
0092             goto fail;
0093     }
0094 
0095     rcu_assign_pointer(callchain_cpus_entries, entries);
0096 
0097     return 0;
0098 
0099 fail:
0100     for_each_possible_cpu(cpu)
0101         kfree(entries->cpu_entries[cpu]);
0102     kfree(entries);
0103 
0104     return -ENOMEM;
0105 }
0106 
0107 int get_callchain_buffers(int event_max_stack)
0108 {
0109     int err = 0;
0110     int count;
0111 
0112     mutex_lock(&callchain_mutex);
0113 
0114     count = atomic_inc_return(&nr_callchain_events);
0115     if (WARN_ON_ONCE(count < 1)) {
0116         err = -EINVAL;
0117         goto exit;
0118     }
0119 
0120     if (count > 1) {
0121         /* If the allocation failed, give up */
0122         if (!callchain_cpus_entries)
0123             err = -ENOMEM;
0124         /*
0125          * If requesting per event more than the global cap,
0126          * return a different error to help userspace figure
0127          * this out.
0128          *
0129          * And also do it here so that we have &callchain_mutex held.
0130          */
0131         if (event_max_stack > sysctl_perf_event_max_stack)
0132             err = -EOVERFLOW;
0133         goto exit;
0134     }
0135 
0136     err = alloc_callchain_buffers();
0137 exit:
0138     if (err)
0139         atomic_dec(&nr_callchain_events);
0140 
0141     mutex_unlock(&callchain_mutex);
0142 
0143     return err;
0144 }
0145 
0146 void put_callchain_buffers(void)
0147 {
0148     if (atomic_dec_and_mutex_lock(&nr_callchain_events, &callchain_mutex)) {
0149         release_callchain_buffers();
0150         mutex_unlock(&callchain_mutex);
0151     }
0152 }
0153 
0154 static struct perf_callchain_entry *get_callchain_entry(int *rctx)
0155 {
0156     int cpu;
0157     struct callchain_cpus_entries *entries;
0158 
0159     *rctx = get_recursion_context(this_cpu_ptr(callchain_recursion));
0160     if (*rctx == -1)
0161         return NULL;
0162 
0163     entries = rcu_dereference(callchain_cpus_entries);
0164     if (!entries)
0165         return NULL;
0166 
0167     cpu = smp_processor_id();
0168 
0169     return (((void *)entries->cpu_entries[cpu]) +
0170         (*rctx * perf_callchain_entry__sizeof()));
0171 }
0172 
0173 static void
0174 put_callchain_entry(int rctx)
0175 {
0176     put_recursion_context(this_cpu_ptr(callchain_recursion), rctx);
0177 }
0178 
0179 struct perf_callchain_entry *
0180 perf_callchain(struct perf_event *event, struct pt_regs *regs)
0181 {
0182     bool kernel = !event->attr.exclude_callchain_kernel;
0183     bool user   = !event->attr.exclude_callchain_user;
0184     /* Disallow cross-task user callchains. */
0185     bool crosstask = event->ctx->task && event->ctx->task != current;
0186     const u32 max_stack = event->attr.sample_max_stack;
0187 
0188     if (!kernel && !user)
0189         return NULL;
0190 
0191     return get_perf_callchain(regs, 0, kernel, user, max_stack, crosstask, true);
0192 }
0193 
0194 struct perf_callchain_entry *
0195 get_perf_callchain(struct pt_regs *regs, u32 init_nr, bool kernel, bool user,
0196            u32 max_stack, bool crosstask, bool add_mark)
0197 {
0198     struct perf_callchain_entry *entry;
0199     struct perf_callchain_entry_ctx ctx;
0200     int rctx;
0201 
0202     entry = get_callchain_entry(&rctx);
0203     if (rctx == -1)
0204         return NULL;
0205 
0206     if (!entry)
0207         goto exit_put;
0208 
0209     ctx.entry     = entry;
0210     ctx.max_stack = max_stack;
0211     ctx.nr        = entry->nr = init_nr;
0212     ctx.contexts       = 0;
0213     ctx.contexts_maxed = false;
0214 
0215     if (kernel && !user_mode(regs)) {
0216         if (add_mark)
0217             perf_callchain_store_context(&ctx, PERF_CONTEXT_KERNEL);
0218         perf_callchain_kernel(&ctx, regs);
0219     }
0220 
0221     if (user) {
0222         if (!user_mode(regs)) {
0223             if  (current->mm)
0224                 regs = task_pt_regs(current);
0225             else
0226                 regs = NULL;
0227         }
0228 
0229         if (regs) {
0230             if (crosstask)
0231                 goto exit_put;
0232 
0233             if (add_mark)
0234                 perf_callchain_store_context(&ctx, PERF_CONTEXT_USER);
0235             perf_callchain_user(&ctx, regs);
0236         }
0237     }
0238 
0239 exit_put:
0240     put_callchain_entry(rctx);
0241 
0242     return entry;
0243 }
0244 
0245 /*
0246  * Used for sysctl_perf_event_max_stack and
0247  * sysctl_perf_event_max_contexts_per_stack.
0248  */
0249 int perf_event_max_stack_handler(struct ctl_table *table, int write,
0250                  void __user *buffer, size_t *lenp, loff_t *ppos)
0251 {
0252     int *value = table->data;
0253     int new_value = *value, ret;
0254     struct ctl_table new_table = *table;
0255 
0256     new_table.data = &new_value;
0257     ret = proc_dointvec_minmax(&new_table, write, buffer, lenp, ppos);
0258     if (ret || !write)
0259         return ret;
0260 
0261     mutex_lock(&callchain_mutex);
0262     if (atomic_read(&nr_callchain_events))
0263         ret = -EBUSY;
0264     else
0265         *value = new_value;
0266 
0267     mutex_unlock(&callchain_mutex);
0268 
0269     return ret;
0270 }