Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * uprobes-based tracing events
0004  *
0005  * Copyright (C) IBM Corporation, 2010-2012
0006  * Author:  Srikar Dronamraju <srikar@linux.vnet.ibm.com>
0007  */
0008 #define pr_fmt(fmt) "trace_uprobe: " fmt
0009 
0010 #include <linux/bpf-cgroup.h>
0011 #include <linux/security.h>
0012 #include <linux/ctype.h>
0013 #include <linux/module.h>
0014 #include <linux/uaccess.h>
0015 #include <linux/uprobes.h>
0016 #include <linux/namei.h>
0017 #include <linux/string.h>
0018 #include <linux/rculist.h>
0019 #include <linux/filter.h>
0020 
0021 #include "trace_dynevent.h"
0022 #include "trace_probe.h"
0023 #include "trace_probe_tmpl.h"
0024 
0025 #define UPROBE_EVENT_SYSTEM "uprobes"
0026 
0027 struct uprobe_trace_entry_head {
0028     struct trace_entry  ent;
0029     unsigned long       vaddr[];
0030 };
0031 
0032 #define SIZEOF_TRACE_ENTRY(is_return)           \
0033     (sizeof(struct uprobe_trace_entry_head) +   \
0034      sizeof(unsigned long) * (is_return ? 2 : 1))
0035 
0036 #define DATAOF_TRACE_ENTRY(entry, is_return)        \
0037     ((void*)(entry) + SIZEOF_TRACE_ENTRY(is_return))
0038 
0039 static int trace_uprobe_create(const char *raw_command);
0040 static int trace_uprobe_show(struct seq_file *m, struct dyn_event *ev);
0041 static int trace_uprobe_release(struct dyn_event *ev);
0042 static bool trace_uprobe_is_busy(struct dyn_event *ev);
0043 static bool trace_uprobe_match(const char *system, const char *event,
0044             int argc, const char **argv, struct dyn_event *ev);
0045 
0046 static struct dyn_event_operations trace_uprobe_ops = {
0047     .create = trace_uprobe_create,
0048     .show = trace_uprobe_show,
0049     .is_busy = trace_uprobe_is_busy,
0050     .free = trace_uprobe_release,
0051     .match = trace_uprobe_match,
0052 };
0053 
0054 /*
0055  * uprobe event core functions
0056  */
0057 struct trace_uprobe {
0058     struct dyn_event        devent;
0059     struct uprobe_consumer      consumer;
0060     struct path         path;
0061     struct inode            *inode;
0062     char                *filename;
0063     unsigned long           offset;
0064     unsigned long           ref_ctr_offset;
0065     unsigned long           nhit;
0066     struct trace_probe      tp;
0067 };
0068 
0069 static bool is_trace_uprobe(struct dyn_event *ev)
0070 {
0071     return ev->ops == &trace_uprobe_ops;
0072 }
0073 
0074 static struct trace_uprobe *to_trace_uprobe(struct dyn_event *ev)
0075 {
0076     return container_of(ev, struct trace_uprobe, devent);
0077 }
0078 
0079 /**
0080  * for_each_trace_uprobe - iterate over the trace_uprobe list
0081  * @pos:    the struct trace_uprobe * for each entry
0082  * @dpos:   the struct dyn_event * to use as a loop cursor
0083  */
0084 #define for_each_trace_uprobe(pos, dpos)    \
0085     for_each_dyn_event(dpos)        \
0086         if (is_trace_uprobe(dpos) && (pos = to_trace_uprobe(dpos)))
0087 
0088 static int register_uprobe_event(struct trace_uprobe *tu);
0089 static int unregister_uprobe_event(struct trace_uprobe *tu);
0090 
0091 struct uprobe_dispatch_data {
0092     struct trace_uprobe *tu;
0093     unsigned long       bp_addr;
0094 };
0095 
0096 static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs);
0097 static int uretprobe_dispatcher(struct uprobe_consumer *con,
0098                 unsigned long func, struct pt_regs *regs);
0099 
0100 #ifdef CONFIG_STACK_GROWSUP
0101 static unsigned long adjust_stack_addr(unsigned long addr, unsigned int n)
0102 {
0103     return addr - (n * sizeof(long));
0104 }
0105 #else
0106 static unsigned long adjust_stack_addr(unsigned long addr, unsigned int n)
0107 {
0108     return addr + (n * sizeof(long));
0109 }
0110 #endif
0111 
0112 static unsigned long get_user_stack_nth(struct pt_regs *regs, unsigned int n)
0113 {
0114     unsigned long ret;
0115     unsigned long addr = user_stack_pointer(regs);
0116 
0117     addr = adjust_stack_addr(addr, n);
0118 
0119     if (copy_from_user(&ret, (void __force __user *) addr, sizeof(ret)))
0120         return 0;
0121 
0122     return ret;
0123 }
0124 
0125 /*
0126  * Uprobes-specific fetch functions
0127  */
0128 static nokprobe_inline int
0129 probe_mem_read(void *dest, void *src, size_t size)
0130 {
0131     void __user *vaddr = (void __force __user *)src;
0132 
0133     return copy_from_user(dest, vaddr, size) ? -EFAULT : 0;
0134 }
0135 
0136 static nokprobe_inline int
0137 probe_mem_read_user(void *dest, void *src, size_t size)
0138 {
0139     return probe_mem_read(dest, src, size);
0140 }
0141 
0142 /*
0143  * Fetch a null-terminated string. Caller MUST set *(u32 *)dest with max
0144  * length and relative data location.
0145  */
0146 static nokprobe_inline int
0147 fetch_store_string(unsigned long addr, void *dest, void *base)
0148 {
0149     long ret;
0150     u32 loc = *(u32 *)dest;
0151     int maxlen  = get_loc_len(loc);
0152     u8 *dst = get_loc_data(dest, base);
0153     void __user *src = (void __force __user *) addr;
0154 
0155     if (unlikely(!maxlen))
0156         return -ENOMEM;
0157 
0158     if (addr == FETCH_TOKEN_COMM)
0159         ret = strlcpy(dst, current->comm, maxlen);
0160     else
0161         ret = strncpy_from_user(dst, src, maxlen);
0162     if (ret >= 0) {
0163         if (ret == maxlen)
0164             dst[ret - 1] = '\0';
0165         else
0166             /*
0167              * Include the terminating null byte. In this case it
0168              * was copied by strncpy_from_user but not accounted
0169              * for in ret.
0170              */
0171             ret++;
0172         *(u32 *)dest = make_data_loc(ret, (void *)dst - base);
0173     }
0174 
0175     return ret;
0176 }
0177 
0178 static nokprobe_inline int
0179 fetch_store_string_user(unsigned long addr, void *dest, void *base)
0180 {
0181     return fetch_store_string(addr, dest, base);
0182 }
0183 
0184 /* Return the length of string -- including null terminal byte */
0185 static nokprobe_inline int
0186 fetch_store_strlen(unsigned long addr)
0187 {
0188     int len;
0189     void __user *vaddr = (void __force __user *) addr;
0190 
0191     if (addr == FETCH_TOKEN_COMM)
0192         len = strlen(current->comm) + 1;
0193     else
0194         len = strnlen_user(vaddr, MAX_STRING_SIZE);
0195 
0196     return (len > MAX_STRING_SIZE) ? 0 : len;
0197 }
0198 
0199 static nokprobe_inline int
0200 fetch_store_strlen_user(unsigned long addr)
0201 {
0202     return fetch_store_strlen(addr);
0203 }
0204 
0205 static unsigned long translate_user_vaddr(unsigned long file_offset)
0206 {
0207     unsigned long base_addr;
0208     struct uprobe_dispatch_data *udd;
0209 
0210     udd = (void *) current->utask->vaddr;
0211 
0212     base_addr = udd->bp_addr - udd->tu->offset;
0213     return base_addr + file_offset;
0214 }
0215 
0216 /* Note that we don't verify it, since the code does not come from user space */
0217 static int
0218 process_fetch_insn(struct fetch_insn *code, void *rec, void *dest,
0219            void *base)
0220 {
0221     struct pt_regs *regs = rec;
0222     unsigned long val;
0223 
0224     /* 1st stage: get value from context */
0225     switch (code->op) {
0226     case FETCH_OP_REG:
0227         val = regs_get_register(regs, code->param);
0228         break;
0229     case FETCH_OP_STACK:
0230         val = get_user_stack_nth(regs, code->param);
0231         break;
0232     case FETCH_OP_STACKP:
0233         val = user_stack_pointer(regs);
0234         break;
0235     case FETCH_OP_RETVAL:
0236         val = regs_return_value(regs);
0237         break;
0238     case FETCH_OP_IMM:
0239         val = code->immediate;
0240         break;
0241     case FETCH_OP_COMM:
0242         val = FETCH_TOKEN_COMM;
0243         break;
0244     case FETCH_OP_DATA:
0245         val = (unsigned long)code->data;
0246         break;
0247     case FETCH_OP_FOFFS:
0248         val = translate_user_vaddr(code->immediate);
0249         break;
0250     default:
0251         return -EILSEQ;
0252     }
0253     code++;
0254 
0255     return process_fetch_insn_bottom(code, val, dest, base);
0256 }
0257 NOKPROBE_SYMBOL(process_fetch_insn)
0258 
0259 static inline void init_trace_uprobe_filter(struct trace_uprobe_filter *filter)
0260 {
0261     rwlock_init(&filter->rwlock);
0262     filter->nr_systemwide = 0;
0263     INIT_LIST_HEAD(&filter->perf_events);
0264 }
0265 
0266 static inline bool uprobe_filter_is_empty(struct trace_uprobe_filter *filter)
0267 {
0268     return !filter->nr_systemwide && list_empty(&filter->perf_events);
0269 }
0270 
0271 static inline bool is_ret_probe(struct trace_uprobe *tu)
0272 {
0273     return tu->consumer.ret_handler != NULL;
0274 }
0275 
0276 static bool trace_uprobe_is_busy(struct dyn_event *ev)
0277 {
0278     struct trace_uprobe *tu = to_trace_uprobe(ev);
0279 
0280     return trace_probe_is_enabled(&tu->tp);
0281 }
0282 
0283 static bool trace_uprobe_match_command_head(struct trace_uprobe *tu,
0284                         int argc, const char **argv)
0285 {
0286     char buf[MAX_ARGSTR_LEN + 1];
0287     int len;
0288 
0289     if (!argc)
0290         return true;
0291 
0292     len = strlen(tu->filename);
0293     if (strncmp(tu->filename, argv[0], len) || argv[0][len] != ':')
0294         return false;
0295 
0296     if (tu->ref_ctr_offset == 0)
0297         snprintf(buf, sizeof(buf), "0x%0*lx",
0298                 (int)(sizeof(void *) * 2), tu->offset);
0299     else
0300         snprintf(buf, sizeof(buf), "0x%0*lx(0x%lx)",
0301                 (int)(sizeof(void *) * 2), tu->offset,
0302                 tu->ref_ctr_offset);
0303     if (strcmp(buf, &argv[0][len + 1]))
0304         return false;
0305 
0306     argc--; argv++;
0307 
0308     return trace_probe_match_command_args(&tu->tp, argc, argv);
0309 }
0310 
0311 static bool trace_uprobe_match(const char *system, const char *event,
0312             int argc, const char **argv, struct dyn_event *ev)
0313 {
0314     struct trace_uprobe *tu = to_trace_uprobe(ev);
0315 
0316     return (event[0] == '\0' ||
0317         strcmp(trace_probe_name(&tu->tp), event) == 0) &&
0318        (!system || strcmp(trace_probe_group_name(&tu->tp), system) == 0) &&
0319        trace_uprobe_match_command_head(tu, argc, argv);
0320 }
0321 
0322 static nokprobe_inline struct trace_uprobe *
0323 trace_uprobe_primary_from_call(struct trace_event_call *call)
0324 {
0325     struct trace_probe *tp;
0326 
0327     tp = trace_probe_primary_from_call(call);
0328     if (WARN_ON_ONCE(!tp))
0329         return NULL;
0330 
0331     return container_of(tp, struct trace_uprobe, tp);
0332 }
0333 
0334 /*
0335  * Allocate new trace_uprobe and initialize it (including uprobes).
0336  */
0337 static struct trace_uprobe *
0338 alloc_trace_uprobe(const char *group, const char *event, int nargs, bool is_ret)
0339 {
0340     struct trace_uprobe *tu;
0341     int ret;
0342 
0343     tu = kzalloc(struct_size(tu, tp.args, nargs), GFP_KERNEL);
0344     if (!tu)
0345         return ERR_PTR(-ENOMEM);
0346 
0347     ret = trace_probe_init(&tu->tp, event, group, true);
0348     if (ret < 0)
0349         goto error;
0350 
0351     dyn_event_init(&tu->devent, &trace_uprobe_ops);
0352     tu->consumer.handler = uprobe_dispatcher;
0353     if (is_ret)
0354         tu->consumer.ret_handler = uretprobe_dispatcher;
0355     init_trace_uprobe_filter(tu->tp.event->filter);
0356     return tu;
0357 
0358 error:
0359     kfree(tu);
0360 
0361     return ERR_PTR(ret);
0362 }
0363 
0364 static void free_trace_uprobe(struct trace_uprobe *tu)
0365 {
0366     if (!tu)
0367         return;
0368 
0369     path_put(&tu->path);
0370     trace_probe_cleanup(&tu->tp);
0371     kfree(tu->filename);
0372     kfree(tu);
0373 }
0374 
0375 static struct trace_uprobe *find_probe_event(const char *event, const char *group)
0376 {
0377     struct dyn_event *pos;
0378     struct trace_uprobe *tu;
0379 
0380     for_each_trace_uprobe(tu, pos)
0381         if (strcmp(trace_probe_name(&tu->tp), event) == 0 &&
0382             strcmp(trace_probe_group_name(&tu->tp), group) == 0)
0383             return tu;
0384 
0385     return NULL;
0386 }
0387 
0388 /* Unregister a trace_uprobe and probe_event */
0389 static int unregister_trace_uprobe(struct trace_uprobe *tu)
0390 {
0391     int ret;
0392 
0393     if (trace_probe_has_sibling(&tu->tp))
0394         goto unreg;
0395 
0396     /* If there's a reference to the dynamic event */
0397     if (trace_event_dyn_busy(trace_probe_event_call(&tu->tp)))
0398         return -EBUSY;
0399 
0400     ret = unregister_uprobe_event(tu);
0401     if (ret)
0402         return ret;
0403 
0404 unreg:
0405     dyn_event_remove(&tu->devent);
0406     trace_probe_unlink(&tu->tp);
0407     free_trace_uprobe(tu);
0408     return 0;
0409 }
0410 
0411 static bool trace_uprobe_has_same_uprobe(struct trace_uprobe *orig,
0412                      struct trace_uprobe *comp)
0413 {
0414     struct trace_probe_event *tpe = orig->tp.event;
0415     struct inode *comp_inode = d_real_inode(comp->path.dentry);
0416     int i;
0417 
0418     list_for_each_entry(orig, &tpe->probes, tp.list) {
0419         if (comp_inode != d_real_inode(orig->path.dentry) ||
0420             comp->offset != orig->offset)
0421             continue;
0422 
0423         /*
0424          * trace_probe_compare_arg_type() ensured that nr_args and
0425          * each argument name and type are same. Let's compare comm.
0426          */
0427         for (i = 0; i < orig->tp.nr_args; i++) {
0428             if (strcmp(orig->tp.args[i].comm,
0429                    comp->tp.args[i].comm))
0430                 break;
0431         }
0432 
0433         if (i == orig->tp.nr_args)
0434             return true;
0435     }
0436 
0437     return false;
0438 }
0439 
0440 static int append_trace_uprobe(struct trace_uprobe *tu, struct trace_uprobe *to)
0441 {
0442     int ret;
0443 
0444     ret = trace_probe_compare_arg_type(&tu->tp, &to->tp);
0445     if (ret) {
0446         /* Note that argument starts index = 2 */
0447         trace_probe_log_set_index(ret + 1);
0448         trace_probe_log_err(0, DIFF_ARG_TYPE);
0449         return -EEXIST;
0450     }
0451     if (trace_uprobe_has_same_uprobe(to, tu)) {
0452         trace_probe_log_set_index(0);
0453         trace_probe_log_err(0, SAME_PROBE);
0454         return -EEXIST;
0455     }
0456 
0457     /* Append to existing event */
0458     ret = trace_probe_append(&tu->tp, &to->tp);
0459     if (!ret)
0460         dyn_event_add(&tu->devent, trace_probe_event_call(&tu->tp));
0461 
0462     return ret;
0463 }
0464 
0465 /*
0466  * Uprobe with multiple reference counter is not allowed. i.e.
0467  * If inode and offset matches, reference counter offset *must*
0468  * match as well. Though, there is one exception: If user is
0469  * replacing old trace_uprobe with new one(same group/event),
0470  * then we allow same uprobe with new reference counter as far
0471  * as the new one does not conflict with any other existing
0472  * ones.
0473  */
0474 static int validate_ref_ctr_offset(struct trace_uprobe *new)
0475 {
0476     struct dyn_event *pos;
0477     struct trace_uprobe *tmp;
0478     struct inode *new_inode = d_real_inode(new->path.dentry);
0479 
0480     for_each_trace_uprobe(tmp, pos) {
0481         if (new_inode == d_real_inode(tmp->path.dentry) &&
0482             new->offset == tmp->offset &&
0483             new->ref_ctr_offset != tmp->ref_ctr_offset) {
0484             pr_warn("Reference counter offset mismatch.");
0485             return -EINVAL;
0486         }
0487     }
0488     return 0;
0489 }
0490 
0491 /* Register a trace_uprobe and probe_event */
0492 static int register_trace_uprobe(struct trace_uprobe *tu)
0493 {
0494     struct trace_uprobe *old_tu;
0495     int ret;
0496 
0497     mutex_lock(&event_mutex);
0498 
0499     ret = validate_ref_ctr_offset(tu);
0500     if (ret)
0501         goto end;
0502 
0503     /* register as an event */
0504     old_tu = find_probe_event(trace_probe_name(&tu->tp),
0505                   trace_probe_group_name(&tu->tp));
0506     if (old_tu) {
0507         if (is_ret_probe(tu) != is_ret_probe(old_tu)) {
0508             trace_probe_log_set_index(0);
0509             trace_probe_log_err(0, DIFF_PROBE_TYPE);
0510             ret = -EEXIST;
0511         } else {
0512             ret = append_trace_uprobe(tu, old_tu);
0513         }
0514         goto end;
0515     }
0516 
0517     ret = register_uprobe_event(tu);
0518     if (ret) {
0519         if (ret == -EEXIST) {
0520             trace_probe_log_set_index(0);
0521             trace_probe_log_err(0, EVENT_EXIST);
0522         } else
0523             pr_warn("Failed to register probe event(%d)\n", ret);
0524         goto end;
0525     }
0526 
0527     dyn_event_add(&tu->devent, trace_probe_event_call(&tu->tp));
0528 
0529 end:
0530     mutex_unlock(&event_mutex);
0531 
0532     return ret;
0533 }
0534 
0535 /*
0536  * Argument syntax:
0537  *  - Add uprobe: p|r[:[GRP/][EVENT]] PATH:OFFSET[%return][(REF)] [FETCHARGS]
0538  */
0539 static int __trace_uprobe_create(int argc, const char **argv)
0540 {
0541     struct trace_uprobe *tu;
0542     const char *event = NULL, *group = UPROBE_EVENT_SYSTEM;
0543     char *arg, *filename, *rctr, *rctr_end, *tmp;
0544     char buf[MAX_EVENT_NAME_LEN];
0545     char gbuf[MAX_EVENT_NAME_LEN];
0546     enum probe_print_type ptype;
0547     struct path path;
0548     unsigned long offset, ref_ctr_offset;
0549     bool is_return = false;
0550     int i, ret;
0551 
0552     ref_ctr_offset = 0;
0553 
0554     switch (argv[0][0]) {
0555     case 'r':
0556         is_return = true;
0557         break;
0558     case 'p':
0559         break;
0560     default:
0561         return -ECANCELED;
0562     }
0563 
0564     if (argc < 2)
0565         return -ECANCELED;
0566 
0567     if (argv[0][1] == ':')
0568         event = &argv[0][2];
0569 
0570     if (!strchr(argv[1], '/'))
0571         return -ECANCELED;
0572 
0573     filename = kstrdup(argv[1], GFP_KERNEL);
0574     if (!filename)
0575         return -ENOMEM;
0576 
0577     /* Find the last occurrence, in case the path contains ':' too. */
0578     arg = strrchr(filename, ':');
0579     if (!arg || !isdigit(arg[1])) {
0580         kfree(filename);
0581         return -ECANCELED;
0582     }
0583 
0584     trace_probe_log_init("trace_uprobe", argc, argv);
0585     trace_probe_log_set_index(1);   /* filename is the 2nd argument */
0586 
0587     *arg++ = '\0';
0588     ret = kern_path(filename, LOOKUP_FOLLOW, &path);
0589     if (ret) {
0590         trace_probe_log_err(0, FILE_NOT_FOUND);
0591         kfree(filename);
0592         trace_probe_log_clear();
0593         return ret;
0594     }
0595     if (!d_is_reg(path.dentry)) {
0596         trace_probe_log_err(0, NO_REGULAR_FILE);
0597         ret = -EINVAL;
0598         goto fail_address_parse;
0599     }
0600 
0601     /* Parse reference counter offset if specified. */
0602     rctr = strchr(arg, '(');
0603     if (rctr) {
0604         rctr_end = strchr(rctr, ')');
0605         if (!rctr_end) {
0606             ret = -EINVAL;
0607             rctr_end = rctr + strlen(rctr);
0608             trace_probe_log_err(rctr_end - filename,
0609                         REFCNT_OPEN_BRACE);
0610             goto fail_address_parse;
0611         } else if (rctr_end[1] != '\0') {
0612             ret = -EINVAL;
0613             trace_probe_log_err(rctr_end + 1 - filename,
0614                         BAD_REFCNT_SUFFIX);
0615             goto fail_address_parse;
0616         }
0617 
0618         *rctr++ = '\0';
0619         *rctr_end = '\0';
0620         ret = kstrtoul(rctr, 0, &ref_ctr_offset);
0621         if (ret) {
0622             trace_probe_log_err(rctr - filename, BAD_REFCNT);
0623             goto fail_address_parse;
0624         }
0625     }
0626 
0627     /* Check if there is %return suffix */
0628     tmp = strchr(arg, '%');
0629     if (tmp) {
0630         if (!strcmp(tmp, "%return")) {
0631             *tmp = '\0';
0632             is_return = true;
0633         } else {
0634             trace_probe_log_err(tmp - filename, BAD_ADDR_SUFFIX);
0635             ret = -EINVAL;
0636             goto fail_address_parse;
0637         }
0638     }
0639 
0640     /* Parse uprobe offset. */
0641     ret = kstrtoul(arg, 0, &offset);
0642     if (ret) {
0643         trace_probe_log_err(arg - filename, BAD_UPROBE_OFFS);
0644         goto fail_address_parse;
0645     }
0646 
0647     /* setup a probe */
0648     trace_probe_log_set_index(0);
0649     if (event) {
0650         ret = traceprobe_parse_event_name(&event, &group, gbuf,
0651                           event - argv[0]);
0652         if (ret)
0653             goto fail_address_parse;
0654     }
0655 
0656     if (!event) {
0657         char *tail;
0658         char *ptr;
0659 
0660         tail = kstrdup(kbasename(filename), GFP_KERNEL);
0661         if (!tail) {
0662             ret = -ENOMEM;
0663             goto fail_address_parse;
0664         }
0665 
0666         ptr = strpbrk(tail, ".-_");
0667         if (ptr)
0668             *ptr = '\0';
0669 
0670         snprintf(buf, MAX_EVENT_NAME_LEN, "%c_%s_0x%lx", 'p', tail, offset);
0671         event = buf;
0672         kfree(tail);
0673     }
0674 
0675     argc -= 2;
0676     argv += 2;
0677 
0678     tu = alloc_trace_uprobe(group, event, argc, is_return);
0679     if (IS_ERR(tu)) {
0680         ret = PTR_ERR(tu);
0681         /* This must return -ENOMEM otherwise there is a bug */
0682         WARN_ON_ONCE(ret != -ENOMEM);
0683         goto fail_address_parse;
0684     }
0685     tu->offset = offset;
0686     tu->ref_ctr_offset = ref_ctr_offset;
0687     tu->path = path;
0688     tu->filename = filename;
0689 
0690     /* parse arguments */
0691     for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) {
0692         trace_probe_log_set_index(i + 2);
0693         ret = traceprobe_parse_probe_arg(&tu->tp, i, argv[i],
0694                     is_return ? TPARG_FL_RETURN : 0);
0695         if (ret)
0696             goto error;
0697     }
0698 
0699     ptype = is_ret_probe(tu) ? PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL;
0700     ret = traceprobe_set_print_fmt(&tu->tp, ptype);
0701     if (ret < 0)
0702         goto error;
0703 
0704     ret = register_trace_uprobe(tu);
0705     if (!ret)
0706         goto out;
0707 
0708 error:
0709     free_trace_uprobe(tu);
0710 out:
0711     trace_probe_log_clear();
0712     return ret;
0713 
0714 fail_address_parse:
0715     trace_probe_log_clear();
0716     path_put(&path);
0717     kfree(filename);
0718 
0719     return ret;
0720 }
0721 
0722 int trace_uprobe_create(const char *raw_command)
0723 {
0724     return trace_probe_create(raw_command, __trace_uprobe_create);
0725 }
0726 
0727 static int create_or_delete_trace_uprobe(const char *raw_command)
0728 {
0729     int ret;
0730 
0731     if (raw_command[0] == '-')
0732         return dyn_event_release(raw_command, &trace_uprobe_ops);
0733 
0734     ret = trace_uprobe_create(raw_command);
0735     return ret == -ECANCELED ? -EINVAL : ret;
0736 }
0737 
0738 static int trace_uprobe_release(struct dyn_event *ev)
0739 {
0740     struct trace_uprobe *tu = to_trace_uprobe(ev);
0741 
0742     return unregister_trace_uprobe(tu);
0743 }
0744 
0745 /* Probes listing interfaces */
0746 static int trace_uprobe_show(struct seq_file *m, struct dyn_event *ev)
0747 {
0748     struct trace_uprobe *tu = to_trace_uprobe(ev);
0749     char c = is_ret_probe(tu) ? 'r' : 'p';
0750     int i;
0751 
0752     seq_printf(m, "%c:%s/%s %s:0x%0*lx", c, trace_probe_group_name(&tu->tp),
0753             trace_probe_name(&tu->tp), tu->filename,
0754             (int)(sizeof(void *) * 2), tu->offset);
0755 
0756     if (tu->ref_ctr_offset)
0757         seq_printf(m, "(0x%lx)", tu->ref_ctr_offset);
0758 
0759     for (i = 0; i < tu->tp.nr_args; i++)
0760         seq_printf(m, " %s=%s", tu->tp.args[i].name, tu->tp.args[i].comm);
0761 
0762     seq_putc(m, '\n');
0763     return 0;
0764 }
0765 
0766 static int probes_seq_show(struct seq_file *m, void *v)
0767 {
0768     struct dyn_event *ev = v;
0769 
0770     if (!is_trace_uprobe(ev))
0771         return 0;
0772 
0773     return trace_uprobe_show(m, ev);
0774 }
0775 
0776 static const struct seq_operations probes_seq_op = {
0777     .start  = dyn_event_seq_start,
0778     .next   = dyn_event_seq_next,
0779     .stop   = dyn_event_seq_stop,
0780     .show   = probes_seq_show
0781 };
0782 
0783 static int probes_open(struct inode *inode, struct file *file)
0784 {
0785     int ret;
0786 
0787     ret = security_locked_down(LOCKDOWN_TRACEFS);
0788     if (ret)
0789         return ret;
0790 
0791     if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
0792         ret = dyn_events_release_all(&trace_uprobe_ops);
0793         if (ret)
0794             return ret;
0795     }
0796 
0797     return seq_open(file, &probes_seq_op);
0798 }
0799 
0800 static ssize_t probes_write(struct file *file, const char __user *buffer,
0801                 size_t count, loff_t *ppos)
0802 {
0803     return trace_parse_run_command(file, buffer, count, ppos,
0804                     create_or_delete_trace_uprobe);
0805 }
0806 
0807 static const struct file_operations uprobe_events_ops = {
0808     .owner      = THIS_MODULE,
0809     .open       = probes_open,
0810     .read       = seq_read,
0811     .llseek     = seq_lseek,
0812     .release    = seq_release,
0813     .write      = probes_write,
0814 };
0815 
0816 /* Probes profiling interfaces */
0817 static int probes_profile_seq_show(struct seq_file *m, void *v)
0818 {
0819     struct dyn_event *ev = v;
0820     struct trace_uprobe *tu;
0821 
0822     if (!is_trace_uprobe(ev))
0823         return 0;
0824 
0825     tu = to_trace_uprobe(ev);
0826     seq_printf(m, "  %s %-44s %15lu\n", tu->filename,
0827             trace_probe_name(&tu->tp), tu->nhit);
0828     return 0;
0829 }
0830 
0831 static const struct seq_operations profile_seq_op = {
0832     .start  = dyn_event_seq_start,
0833     .next   = dyn_event_seq_next,
0834     .stop   = dyn_event_seq_stop,
0835     .show   = probes_profile_seq_show
0836 };
0837 
0838 static int profile_open(struct inode *inode, struct file *file)
0839 {
0840     int ret;
0841 
0842     ret = security_locked_down(LOCKDOWN_TRACEFS);
0843     if (ret)
0844         return ret;
0845 
0846     return seq_open(file, &profile_seq_op);
0847 }
0848 
0849 static const struct file_operations uprobe_profile_ops = {
0850     .owner      = THIS_MODULE,
0851     .open       = profile_open,
0852     .read       = seq_read,
0853     .llseek     = seq_lseek,
0854     .release    = seq_release,
0855 };
0856 
0857 struct uprobe_cpu_buffer {
0858     struct mutex mutex;
0859     void *buf;
0860 };
0861 static struct uprobe_cpu_buffer __percpu *uprobe_cpu_buffer;
0862 static int uprobe_buffer_refcnt;
0863 
0864 static int uprobe_buffer_init(void)
0865 {
0866     int cpu, err_cpu;
0867 
0868     uprobe_cpu_buffer = alloc_percpu(struct uprobe_cpu_buffer);
0869     if (uprobe_cpu_buffer == NULL)
0870         return -ENOMEM;
0871 
0872     for_each_possible_cpu(cpu) {
0873         struct page *p = alloc_pages_node(cpu_to_node(cpu),
0874                           GFP_KERNEL, 0);
0875         if (p == NULL) {
0876             err_cpu = cpu;
0877             goto err;
0878         }
0879         per_cpu_ptr(uprobe_cpu_buffer, cpu)->buf = page_address(p);
0880         mutex_init(&per_cpu_ptr(uprobe_cpu_buffer, cpu)->mutex);
0881     }
0882 
0883     return 0;
0884 
0885 err:
0886     for_each_possible_cpu(cpu) {
0887         if (cpu == err_cpu)
0888             break;
0889         free_page((unsigned long)per_cpu_ptr(uprobe_cpu_buffer, cpu)->buf);
0890     }
0891 
0892     free_percpu(uprobe_cpu_buffer);
0893     return -ENOMEM;
0894 }
0895 
0896 static int uprobe_buffer_enable(void)
0897 {
0898     int ret = 0;
0899 
0900     BUG_ON(!mutex_is_locked(&event_mutex));
0901 
0902     if (uprobe_buffer_refcnt++ == 0) {
0903         ret = uprobe_buffer_init();
0904         if (ret < 0)
0905             uprobe_buffer_refcnt--;
0906     }
0907 
0908     return ret;
0909 }
0910 
0911 static void uprobe_buffer_disable(void)
0912 {
0913     int cpu;
0914 
0915     BUG_ON(!mutex_is_locked(&event_mutex));
0916 
0917     if (--uprobe_buffer_refcnt == 0) {
0918         for_each_possible_cpu(cpu)
0919             free_page((unsigned long)per_cpu_ptr(uprobe_cpu_buffer,
0920                                  cpu)->buf);
0921 
0922         free_percpu(uprobe_cpu_buffer);
0923         uprobe_cpu_buffer = NULL;
0924     }
0925 }
0926 
0927 static struct uprobe_cpu_buffer *uprobe_buffer_get(void)
0928 {
0929     struct uprobe_cpu_buffer *ucb;
0930     int cpu;
0931 
0932     cpu = raw_smp_processor_id();
0933     ucb = per_cpu_ptr(uprobe_cpu_buffer, cpu);
0934 
0935     /*
0936      * Use per-cpu buffers for fastest access, but we might migrate
0937      * so the mutex makes sure we have sole access to it.
0938      */
0939     mutex_lock(&ucb->mutex);
0940 
0941     return ucb;
0942 }
0943 
0944 static void uprobe_buffer_put(struct uprobe_cpu_buffer *ucb)
0945 {
0946     mutex_unlock(&ucb->mutex);
0947 }
0948 
0949 static void __uprobe_trace_func(struct trace_uprobe *tu,
0950                 unsigned long func, struct pt_regs *regs,
0951                 struct uprobe_cpu_buffer *ucb, int dsize,
0952                 struct trace_event_file *trace_file)
0953 {
0954     struct uprobe_trace_entry_head *entry;
0955     struct trace_event_buffer fbuffer;
0956     void *data;
0957     int size, esize;
0958     struct trace_event_call *call = trace_probe_event_call(&tu->tp);
0959 
0960     WARN_ON(call != trace_file->event_call);
0961 
0962     if (WARN_ON_ONCE(tu->tp.size + dsize > PAGE_SIZE))
0963         return;
0964 
0965     if (trace_trigger_soft_disabled(trace_file))
0966         return;
0967 
0968     esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));
0969     size = esize + tu->tp.size + dsize;
0970     entry = trace_event_buffer_reserve(&fbuffer, trace_file, size);
0971     if (!entry)
0972         return;
0973 
0974     if (is_ret_probe(tu)) {
0975         entry->vaddr[0] = func;
0976         entry->vaddr[1] = instruction_pointer(regs);
0977         data = DATAOF_TRACE_ENTRY(entry, true);
0978     } else {
0979         entry->vaddr[0] = instruction_pointer(regs);
0980         data = DATAOF_TRACE_ENTRY(entry, false);
0981     }
0982 
0983     memcpy(data, ucb->buf, tu->tp.size + dsize);
0984 
0985     trace_event_buffer_commit(&fbuffer);
0986 }
0987 
0988 /* uprobe handler */
0989 static int uprobe_trace_func(struct trace_uprobe *tu, struct pt_regs *regs,
0990                  struct uprobe_cpu_buffer *ucb, int dsize)
0991 {
0992     struct event_file_link *link;
0993 
0994     if (is_ret_probe(tu))
0995         return 0;
0996 
0997     rcu_read_lock();
0998     trace_probe_for_each_link_rcu(link, &tu->tp)
0999         __uprobe_trace_func(tu, 0, regs, ucb, dsize, link->file);
1000     rcu_read_unlock();
1001 
1002     return 0;
1003 }
1004 
1005 static void uretprobe_trace_func(struct trace_uprobe *tu, unsigned long func,
1006                  struct pt_regs *regs,
1007                  struct uprobe_cpu_buffer *ucb, int dsize)
1008 {
1009     struct event_file_link *link;
1010 
1011     rcu_read_lock();
1012     trace_probe_for_each_link_rcu(link, &tu->tp)
1013         __uprobe_trace_func(tu, func, regs, ucb, dsize, link->file);
1014     rcu_read_unlock();
1015 }
1016 
1017 /* Event entry printers */
1018 static enum print_line_t
1019 print_uprobe_event(struct trace_iterator *iter, int flags, struct trace_event *event)
1020 {
1021     struct uprobe_trace_entry_head *entry;
1022     struct trace_seq *s = &iter->seq;
1023     struct trace_uprobe *tu;
1024     u8 *data;
1025 
1026     entry = (struct uprobe_trace_entry_head *)iter->ent;
1027     tu = trace_uprobe_primary_from_call(
1028         container_of(event, struct trace_event_call, event));
1029     if (unlikely(!tu))
1030         goto out;
1031 
1032     if (is_ret_probe(tu)) {
1033         trace_seq_printf(s, "%s: (0x%lx <- 0x%lx)",
1034                  trace_probe_name(&tu->tp),
1035                  entry->vaddr[1], entry->vaddr[0]);
1036         data = DATAOF_TRACE_ENTRY(entry, true);
1037     } else {
1038         trace_seq_printf(s, "%s: (0x%lx)",
1039                  trace_probe_name(&tu->tp),
1040                  entry->vaddr[0]);
1041         data = DATAOF_TRACE_ENTRY(entry, false);
1042     }
1043 
1044     if (print_probe_args(s, tu->tp.args, tu->tp.nr_args, data, entry) < 0)
1045         goto out;
1046 
1047     trace_seq_putc(s, '\n');
1048 
1049  out:
1050     return trace_handle_return(s);
1051 }
1052 
1053 typedef bool (*filter_func_t)(struct uprobe_consumer *self,
1054                 enum uprobe_filter_ctx ctx,
1055                 struct mm_struct *mm);
1056 
1057 static int trace_uprobe_enable(struct trace_uprobe *tu, filter_func_t filter)
1058 {
1059     int ret;
1060 
1061     tu->consumer.filter = filter;
1062     tu->inode = d_real_inode(tu->path.dentry);
1063 
1064     if (tu->ref_ctr_offset)
1065         ret = uprobe_register_refctr(tu->inode, tu->offset,
1066                 tu->ref_ctr_offset, &tu->consumer);
1067     else
1068         ret = uprobe_register(tu->inode, tu->offset, &tu->consumer);
1069 
1070     if (ret)
1071         tu->inode = NULL;
1072 
1073     return ret;
1074 }
1075 
1076 static void __probe_event_disable(struct trace_probe *tp)
1077 {
1078     struct trace_uprobe *tu;
1079 
1080     tu = container_of(tp, struct trace_uprobe, tp);
1081     WARN_ON(!uprobe_filter_is_empty(tu->tp.event->filter));
1082 
1083     list_for_each_entry(tu, trace_probe_probe_list(tp), tp.list) {
1084         if (!tu->inode)
1085             continue;
1086 
1087         uprobe_unregister(tu->inode, tu->offset, &tu->consumer);
1088         tu->inode = NULL;
1089     }
1090 }
1091 
1092 static int probe_event_enable(struct trace_event_call *call,
1093             struct trace_event_file *file, filter_func_t filter)
1094 {
1095     struct trace_probe *tp;
1096     struct trace_uprobe *tu;
1097     bool enabled;
1098     int ret;
1099 
1100     tp = trace_probe_primary_from_call(call);
1101     if (WARN_ON_ONCE(!tp))
1102         return -ENODEV;
1103     enabled = trace_probe_is_enabled(tp);
1104 
1105     /* This may also change "enabled" state */
1106     if (file) {
1107         if (trace_probe_test_flag(tp, TP_FLAG_PROFILE))
1108             return -EINTR;
1109 
1110         ret = trace_probe_add_file(tp, file);
1111         if (ret < 0)
1112             return ret;
1113     } else {
1114         if (trace_probe_test_flag(tp, TP_FLAG_TRACE))
1115             return -EINTR;
1116 
1117         trace_probe_set_flag(tp, TP_FLAG_PROFILE);
1118     }
1119 
1120     tu = container_of(tp, struct trace_uprobe, tp);
1121     WARN_ON(!uprobe_filter_is_empty(tu->tp.event->filter));
1122 
1123     if (enabled)
1124         return 0;
1125 
1126     ret = uprobe_buffer_enable();
1127     if (ret)
1128         goto err_flags;
1129 
1130     list_for_each_entry(tu, trace_probe_probe_list(tp), tp.list) {
1131         ret = trace_uprobe_enable(tu, filter);
1132         if (ret) {
1133             __probe_event_disable(tp);
1134             goto err_buffer;
1135         }
1136     }
1137 
1138     return 0;
1139 
1140  err_buffer:
1141     uprobe_buffer_disable();
1142 
1143  err_flags:
1144     if (file)
1145         trace_probe_remove_file(tp, file);
1146     else
1147         trace_probe_clear_flag(tp, TP_FLAG_PROFILE);
1148 
1149     return ret;
1150 }
1151 
1152 static void probe_event_disable(struct trace_event_call *call,
1153                 struct trace_event_file *file)
1154 {
1155     struct trace_probe *tp;
1156 
1157     tp = trace_probe_primary_from_call(call);
1158     if (WARN_ON_ONCE(!tp))
1159         return;
1160 
1161     if (!trace_probe_is_enabled(tp))
1162         return;
1163 
1164     if (file) {
1165         if (trace_probe_remove_file(tp, file) < 0)
1166             return;
1167 
1168         if (trace_probe_is_enabled(tp))
1169             return;
1170     } else
1171         trace_probe_clear_flag(tp, TP_FLAG_PROFILE);
1172 
1173     __probe_event_disable(tp);
1174     uprobe_buffer_disable();
1175 }
1176 
1177 static int uprobe_event_define_fields(struct trace_event_call *event_call)
1178 {
1179     int ret, size;
1180     struct uprobe_trace_entry_head field;
1181     struct trace_uprobe *tu;
1182 
1183     tu = trace_uprobe_primary_from_call(event_call);
1184     if (unlikely(!tu))
1185         return -ENODEV;
1186 
1187     if (is_ret_probe(tu)) {
1188         DEFINE_FIELD(unsigned long, vaddr[0], FIELD_STRING_FUNC, 0);
1189         DEFINE_FIELD(unsigned long, vaddr[1], FIELD_STRING_RETIP, 0);
1190         size = SIZEOF_TRACE_ENTRY(true);
1191     } else {
1192         DEFINE_FIELD(unsigned long, vaddr[0], FIELD_STRING_IP, 0);
1193         size = SIZEOF_TRACE_ENTRY(false);
1194     }
1195 
1196     return traceprobe_define_arg_fields(event_call, size, &tu->tp);
1197 }
1198 
1199 #ifdef CONFIG_PERF_EVENTS
1200 static bool
1201 __uprobe_perf_filter(struct trace_uprobe_filter *filter, struct mm_struct *mm)
1202 {
1203     struct perf_event *event;
1204 
1205     if (filter->nr_systemwide)
1206         return true;
1207 
1208     list_for_each_entry(event, &filter->perf_events, hw.tp_list) {
1209         if (event->hw.target->mm == mm)
1210             return true;
1211     }
1212 
1213     return false;
1214 }
1215 
1216 static inline bool
1217 trace_uprobe_filter_event(struct trace_uprobe_filter *filter,
1218               struct perf_event *event)
1219 {
1220     return __uprobe_perf_filter(filter, event->hw.target->mm);
1221 }
1222 
1223 static bool trace_uprobe_filter_remove(struct trace_uprobe_filter *filter,
1224                        struct perf_event *event)
1225 {
1226     bool done;
1227 
1228     write_lock(&filter->rwlock);
1229     if (event->hw.target) {
1230         list_del(&event->hw.tp_list);
1231         done = filter->nr_systemwide ||
1232             (event->hw.target->flags & PF_EXITING) ||
1233             trace_uprobe_filter_event(filter, event);
1234     } else {
1235         filter->nr_systemwide--;
1236         done = filter->nr_systemwide;
1237     }
1238     write_unlock(&filter->rwlock);
1239 
1240     return done;
1241 }
1242 
1243 /* This returns true if the filter always covers target mm */
1244 static bool trace_uprobe_filter_add(struct trace_uprobe_filter *filter,
1245                     struct perf_event *event)
1246 {
1247     bool done;
1248 
1249     write_lock(&filter->rwlock);
1250     if (event->hw.target) {
1251         /*
1252          * event->parent != NULL means copy_process(), we can avoid
1253          * uprobe_apply(). current->mm must be probed and we can rely
1254          * on dup_mmap() which preserves the already installed bp's.
1255          *
1256          * attr.enable_on_exec means that exec/mmap will install the
1257          * breakpoints we need.
1258          */
1259         done = filter->nr_systemwide ||
1260             event->parent || event->attr.enable_on_exec ||
1261             trace_uprobe_filter_event(filter, event);
1262         list_add(&event->hw.tp_list, &filter->perf_events);
1263     } else {
1264         done = filter->nr_systemwide;
1265         filter->nr_systemwide++;
1266     }
1267     write_unlock(&filter->rwlock);
1268 
1269     return done;
1270 }
1271 
1272 static int uprobe_perf_close(struct trace_event_call *call,
1273                  struct perf_event *event)
1274 {
1275     struct trace_probe *tp;
1276     struct trace_uprobe *tu;
1277     int ret = 0;
1278 
1279     tp = trace_probe_primary_from_call(call);
1280     if (WARN_ON_ONCE(!tp))
1281         return -ENODEV;
1282 
1283     tu = container_of(tp, struct trace_uprobe, tp);
1284     if (trace_uprobe_filter_remove(tu->tp.event->filter, event))
1285         return 0;
1286 
1287     list_for_each_entry(tu, trace_probe_probe_list(tp), tp.list) {
1288         ret = uprobe_apply(tu->inode, tu->offset, &tu->consumer, false);
1289         if (ret)
1290             break;
1291     }
1292 
1293     return ret;
1294 }
1295 
1296 static int uprobe_perf_open(struct trace_event_call *call,
1297                 struct perf_event *event)
1298 {
1299     struct trace_probe *tp;
1300     struct trace_uprobe *tu;
1301     int err = 0;
1302 
1303     tp = trace_probe_primary_from_call(call);
1304     if (WARN_ON_ONCE(!tp))
1305         return -ENODEV;
1306 
1307     tu = container_of(tp, struct trace_uprobe, tp);
1308     if (trace_uprobe_filter_add(tu->tp.event->filter, event))
1309         return 0;
1310 
1311     list_for_each_entry(tu, trace_probe_probe_list(tp), tp.list) {
1312         err = uprobe_apply(tu->inode, tu->offset, &tu->consumer, true);
1313         if (err) {
1314             uprobe_perf_close(call, event);
1315             break;
1316         }
1317     }
1318 
1319     return err;
1320 }
1321 
1322 static bool uprobe_perf_filter(struct uprobe_consumer *uc,
1323                 enum uprobe_filter_ctx ctx, struct mm_struct *mm)
1324 {
1325     struct trace_uprobe_filter *filter;
1326     struct trace_uprobe *tu;
1327     int ret;
1328 
1329     tu = container_of(uc, struct trace_uprobe, consumer);
1330     filter = tu->tp.event->filter;
1331 
1332     read_lock(&filter->rwlock);
1333     ret = __uprobe_perf_filter(filter, mm);
1334     read_unlock(&filter->rwlock);
1335 
1336     return ret;
1337 }
1338 
1339 static void __uprobe_perf_func(struct trace_uprobe *tu,
1340                    unsigned long func, struct pt_regs *regs,
1341                    struct uprobe_cpu_buffer *ucb, int dsize)
1342 {
1343     struct trace_event_call *call = trace_probe_event_call(&tu->tp);
1344     struct uprobe_trace_entry_head *entry;
1345     struct hlist_head *head;
1346     void *data;
1347     int size, esize;
1348     int rctx;
1349 
1350 #ifdef CONFIG_BPF_EVENTS
1351     if (bpf_prog_array_valid(call)) {
1352         u32 ret;
1353 
1354         ret = bpf_prog_run_array_sleepable(call->prog_array, regs, bpf_prog_run);
1355         if (!ret)
1356             return;
1357     }
1358 #endif /* CONFIG_BPF_EVENTS */
1359 
1360     esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));
1361 
1362     size = esize + tu->tp.size + dsize;
1363     size = ALIGN(size + sizeof(u32), sizeof(u64)) - sizeof(u32);
1364     if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE, "profile buffer not large enough"))
1365         return;
1366 
1367     preempt_disable();
1368     head = this_cpu_ptr(call->perf_events);
1369     if (hlist_empty(head))
1370         goto out;
1371 
1372     entry = perf_trace_buf_alloc(size, NULL, &rctx);
1373     if (!entry)
1374         goto out;
1375 
1376     if (is_ret_probe(tu)) {
1377         entry->vaddr[0] = func;
1378         entry->vaddr[1] = instruction_pointer(regs);
1379         data = DATAOF_TRACE_ENTRY(entry, true);
1380     } else {
1381         entry->vaddr[0] = instruction_pointer(regs);
1382         data = DATAOF_TRACE_ENTRY(entry, false);
1383     }
1384 
1385     memcpy(data, ucb->buf, tu->tp.size + dsize);
1386 
1387     if (size - esize > tu->tp.size + dsize) {
1388         int len = tu->tp.size + dsize;
1389 
1390         memset(data + len, 0, size - esize - len);
1391     }
1392 
1393     perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
1394                   head, NULL);
1395  out:
1396     preempt_enable();
1397 }
1398 
1399 /* uprobe profile handler */
1400 static int uprobe_perf_func(struct trace_uprobe *tu, struct pt_regs *regs,
1401                 struct uprobe_cpu_buffer *ucb, int dsize)
1402 {
1403     if (!uprobe_perf_filter(&tu->consumer, 0, current->mm))
1404         return UPROBE_HANDLER_REMOVE;
1405 
1406     if (!is_ret_probe(tu))
1407         __uprobe_perf_func(tu, 0, regs, ucb, dsize);
1408     return 0;
1409 }
1410 
1411 static void uretprobe_perf_func(struct trace_uprobe *tu, unsigned long func,
1412                 struct pt_regs *regs,
1413                 struct uprobe_cpu_buffer *ucb, int dsize)
1414 {
1415     __uprobe_perf_func(tu, func, regs, ucb, dsize);
1416 }
1417 
1418 int bpf_get_uprobe_info(const struct perf_event *event, u32 *fd_type,
1419             const char **filename, u64 *probe_offset,
1420             bool perf_type_tracepoint)
1421 {
1422     const char *pevent = trace_event_name(event->tp_event);
1423     const char *group = event->tp_event->class->system;
1424     struct trace_uprobe *tu;
1425 
1426     if (perf_type_tracepoint)
1427         tu = find_probe_event(pevent, group);
1428     else
1429         tu = trace_uprobe_primary_from_call(event->tp_event);
1430     if (!tu)
1431         return -EINVAL;
1432 
1433     *fd_type = is_ret_probe(tu) ? BPF_FD_TYPE_URETPROBE
1434                     : BPF_FD_TYPE_UPROBE;
1435     *filename = tu->filename;
1436     *probe_offset = tu->offset;
1437     return 0;
1438 }
1439 #endif  /* CONFIG_PERF_EVENTS */
1440 
1441 static int
1442 trace_uprobe_register(struct trace_event_call *event, enum trace_reg type,
1443               void *data)
1444 {
1445     struct trace_event_file *file = data;
1446 
1447     switch (type) {
1448     case TRACE_REG_REGISTER:
1449         return probe_event_enable(event, file, NULL);
1450 
1451     case TRACE_REG_UNREGISTER:
1452         probe_event_disable(event, file);
1453         return 0;
1454 
1455 #ifdef CONFIG_PERF_EVENTS
1456     case TRACE_REG_PERF_REGISTER:
1457         return probe_event_enable(event, NULL, uprobe_perf_filter);
1458 
1459     case TRACE_REG_PERF_UNREGISTER:
1460         probe_event_disable(event, NULL);
1461         return 0;
1462 
1463     case TRACE_REG_PERF_OPEN:
1464         return uprobe_perf_open(event, data);
1465 
1466     case TRACE_REG_PERF_CLOSE:
1467         return uprobe_perf_close(event, data);
1468 
1469 #endif
1470     default:
1471         return 0;
1472     }
1473 }
1474 
1475 static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs)
1476 {
1477     struct trace_uprobe *tu;
1478     struct uprobe_dispatch_data udd;
1479     struct uprobe_cpu_buffer *ucb;
1480     int dsize, esize;
1481     int ret = 0;
1482 
1483 
1484     tu = container_of(con, struct trace_uprobe, consumer);
1485     tu->nhit++;
1486 
1487     udd.tu = tu;
1488     udd.bp_addr = instruction_pointer(regs);
1489 
1490     current->utask->vaddr = (unsigned long) &udd;
1491 
1492     if (WARN_ON_ONCE(!uprobe_cpu_buffer))
1493         return 0;
1494 
1495     dsize = __get_data_size(&tu->tp, regs);
1496     esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));
1497 
1498     ucb = uprobe_buffer_get();
1499     store_trace_args(ucb->buf, &tu->tp, regs, esize, dsize);
1500 
1501     if (trace_probe_test_flag(&tu->tp, TP_FLAG_TRACE))
1502         ret |= uprobe_trace_func(tu, regs, ucb, dsize);
1503 
1504 #ifdef CONFIG_PERF_EVENTS
1505     if (trace_probe_test_flag(&tu->tp, TP_FLAG_PROFILE))
1506         ret |= uprobe_perf_func(tu, regs, ucb, dsize);
1507 #endif
1508     uprobe_buffer_put(ucb);
1509     return ret;
1510 }
1511 
1512 static int uretprobe_dispatcher(struct uprobe_consumer *con,
1513                 unsigned long func, struct pt_regs *regs)
1514 {
1515     struct trace_uprobe *tu;
1516     struct uprobe_dispatch_data udd;
1517     struct uprobe_cpu_buffer *ucb;
1518     int dsize, esize;
1519 
1520     tu = container_of(con, struct trace_uprobe, consumer);
1521 
1522     udd.tu = tu;
1523     udd.bp_addr = func;
1524 
1525     current->utask->vaddr = (unsigned long) &udd;
1526 
1527     if (WARN_ON_ONCE(!uprobe_cpu_buffer))
1528         return 0;
1529 
1530     dsize = __get_data_size(&tu->tp, regs);
1531     esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));
1532 
1533     ucb = uprobe_buffer_get();
1534     store_trace_args(ucb->buf, &tu->tp, regs, esize, dsize);
1535 
1536     if (trace_probe_test_flag(&tu->tp, TP_FLAG_TRACE))
1537         uretprobe_trace_func(tu, func, regs, ucb, dsize);
1538 
1539 #ifdef CONFIG_PERF_EVENTS
1540     if (trace_probe_test_flag(&tu->tp, TP_FLAG_PROFILE))
1541         uretprobe_perf_func(tu, func, regs, ucb, dsize);
1542 #endif
1543     uprobe_buffer_put(ucb);
1544     return 0;
1545 }
1546 
1547 static struct trace_event_functions uprobe_funcs = {
1548     .trace      = print_uprobe_event
1549 };
1550 
1551 static struct trace_event_fields uprobe_fields_array[] = {
1552     { .type = TRACE_FUNCTION_TYPE,
1553       .define_fields = uprobe_event_define_fields },
1554     {}
1555 };
1556 
1557 static inline void init_trace_event_call(struct trace_uprobe *tu)
1558 {
1559     struct trace_event_call *call = trace_probe_event_call(&tu->tp);
1560     call->event.funcs = &uprobe_funcs;
1561     call->class->fields_array = uprobe_fields_array;
1562 
1563     call->flags = TRACE_EVENT_FL_UPROBE | TRACE_EVENT_FL_CAP_ANY;
1564     call->class->reg = trace_uprobe_register;
1565 }
1566 
1567 static int register_uprobe_event(struct trace_uprobe *tu)
1568 {
1569     init_trace_event_call(tu);
1570 
1571     return trace_probe_register_event_call(&tu->tp);
1572 }
1573 
1574 static int unregister_uprobe_event(struct trace_uprobe *tu)
1575 {
1576     return trace_probe_unregister_event_call(&tu->tp);
1577 }
1578 
1579 #ifdef CONFIG_PERF_EVENTS
1580 struct trace_event_call *
1581 create_local_trace_uprobe(char *name, unsigned long offs,
1582               unsigned long ref_ctr_offset, bool is_return)
1583 {
1584     enum probe_print_type ptype;
1585     struct trace_uprobe *tu;
1586     struct path path;
1587     int ret;
1588 
1589     ret = kern_path(name, LOOKUP_FOLLOW, &path);
1590     if (ret)
1591         return ERR_PTR(ret);
1592 
1593     if (!d_is_reg(path.dentry)) {
1594         path_put(&path);
1595         return ERR_PTR(-EINVAL);
1596     }
1597 
1598     /*
1599      * local trace_kprobes are not added to dyn_event, so they are never
1600      * searched in find_trace_kprobe(). Therefore, there is no concern of
1601      * duplicated name "DUMMY_EVENT" here.
1602      */
1603     tu = alloc_trace_uprobe(UPROBE_EVENT_SYSTEM, "DUMMY_EVENT", 0,
1604                 is_return);
1605 
1606     if (IS_ERR(tu)) {
1607         pr_info("Failed to allocate trace_uprobe.(%d)\n",
1608             (int)PTR_ERR(tu));
1609         path_put(&path);
1610         return ERR_CAST(tu);
1611     }
1612 
1613     tu->offset = offs;
1614     tu->path = path;
1615     tu->ref_ctr_offset = ref_ctr_offset;
1616     tu->filename = kstrdup(name, GFP_KERNEL);
1617     if (!tu->filename) {
1618         ret = -ENOMEM;
1619         goto error;
1620     }
1621 
1622     init_trace_event_call(tu);
1623 
1624     ptype = is_ret_probe(tu) ? PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL;
1625     if (traceprobe_set_print_fmt(&tu->tp, ptype) < 0) {
1626         ret = -ENOMEM;
1627         goto error;
1628     }
1629 
1630     return trace_probe_event_call(&tu->tp);
1631 error:
1632     free_trace_uprobe(tu);
1633     return ERR_PTR(ret);
1634 }
1635 
1636 void destroy_local_trace_uprobe(struct trace_event_call *event_call)
1637 {
1638     struct trace_uprobe *tu;
1639 
1640     tu = trace_uprobe_primary_from_call(event_call);
1641 
1642     free_trace_uprobe(tu);
1643 }
1644 #endif /* CONFIG_PERF_EVENTS */
1645 
1646 /* Make a trace interface for controlling probe points */
1647 static __init int init_uprobe_trace(void)
1648 {
1649     int ret;
1650 
1651     ret = dyn_event_register(&trace_uprobe_ops);
1652     if (ret)
1653         return ret;
1654 
1655     ret = tracing_init_dentry();
1656     if (ret)
1657         return 0;
1658 
1659     trace_create_file("uprobe_events", TRACE_MODE_WRITE, NULL,
1660                     NULL, &uprobe_events_ops);
1661     /* Profile interface */
1662     trace_create_file("uprobe_profile", TRACE_MODE_READ, NULL,
1663                     NULL, &uprobe_profile_ops);
1664     return 0;
1665 }
1666 
1667 fs_initcall(init_uprobe_trace);
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!