kernel/trace/fprobe.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * fprobe - Simple ftrace probe wrapper for function entry.
0004  */
0005 #define pr_fmt(fmt) "fprobe: " fmt
0006
0007 #include <linux/err.h>
0008 #include <linux/fprobe.h>
0009 #include <linux/kallsyms.h>
0010 #include <linux/kprobes.h>
0011 #include <linux/rethook.h>
0012 #include <linux/slab.h>
0013 #include <linux/sort.h>
0014
0015 #include "trace.h"
0016
0017 struct fprobe_rethook_node {
0018     struct rethook_node node;
0019     unsigned long entry_ip;
0020 };
0021
0022 static void fprobe_handler(unsigned long ip, unsigned long parent_ip,
0023                struct ftrace_ops *ops, struct ftrace_regs *fregs)
0024 {
0025     struct fprobe_rethook_node *fpr;
0026     struct rethook_node *rh;
0027     struct fprobe *fp;
0028     int bit;
0029
0030     fp = container_of(ops, struct fprobe, ops);
0031     if (fprobe_disabled(fp))
0032         return;
0033
0034     bit = ftrace_test_recursion_trylock(ip, parent_ip);
0035     if (bit < 0) {
0036         fp->nmissed++;
0037         return;
0038     }
0039
0040     if (fp->entry_handler)
0041         fp->entry_handler(fp, ip, ftrace_get_regs(fregs));
0042
0043     if (fp->exit_handler) {
0044         rh = rethook_try_get(fp->rethook);
0045         if (!rh) {
0046             fp->nmissed++;
0047             goto out;
0048         }
0049         fpr = container_of(rh, struct fprobe_rethook_node, node);
0050         fpr->entry_ip = ip;
0051         rethook_hook(rh, ftrace_get_regs(fregs), true);
0052     }
0053
0054 out:
0055     ftrace_test_recursion_unlock(bit);
0056 }
0057 NOKPROBE_SYMBOL(fprobe_handler);
0058
0059 static void fprobe_kprobe_handler(unsigned long ip, unsigned long parent_ip,
0060                   struct ftrace_ops *ops, struct ftrace_regs *fregs)
0061 {
0062     struct fprobe *fp = container_of(ops, struct fprobe, ops);
0063
0064     if (unlikely(kprobe_running())) {
0065         fp->nmissed++;
0066         return;
0067     }
0068     kprobe_busy_begin();
0069     fprobe_handler(ip, parent_ip, ops, fregs);
0070     kprobe_busy_end();
0071 }
0072
0073 static void fprobe_exit_handler(struct rethook_node *rh, void *data,
0074                 struct pt_regs *regs)
0075 {
0076     struct fprobe *fp = (struct fprobe *)data;
0077     struct fprobe_rethook_node *fpr;
0078
0079     if (!fp || fprobe_disabled(fp))
0080         return;
0081
0082     fpr = container_of(rh, struct fprobe_rethook_node, node);
0083
0084     fp->exit_handler(fp, fpr->entry_ip, regs);
0085 }
0086 NOKPROBE_SYMBOL(fprobe_exit_handler);
0087
0088 static int symbols_cmp(const void *a, const void *b)
0089 {
0090     const char **str_a = (const char **) a;
0091     const char **str_b = (const char **) b;
0092
0093     return strcmp(*str_a, *str_b);
0094 }
0095
0096 /* Convert ftrace location address from symbols */
0097 static unsigned long *get_ftrace_locations(const char **syms, int num)
0098 {
0099     unsigned long *addrs;
0100
0101     /* Convert symbols to symbol address */
0102     addrs = kcalloc(num, sizeof(*addrs), GFP_KERNEL);
0103     if (!addrs)
0104         return ERR_PTR(-ENOMEM);
0105
0106     /* ftrace_lookup_symbols expects sorted symbols */
0107     sort(syms, num, sizeof(*syms), symbols_cmp, NULL);
0108
0109     if (!ftrace_lookup_symbols(syms, num, addrs))
0110         return addrs;
0111
0112     kfree(addrs);
0113     return ERR_PTR(-ENOENT);
0114 }
0115
0116 static void fprobe_init(struct fprobe *fp)
0117 {
0118     fp->nmissed = 0;
0119     if (fprobe_shared_with_kprobes(fp))
0120         fp->ops.func = fprobe_kprobe_handler;
0121     else
0122         fp->ops.func = fprobe_handler;
0123     fp->ops.flags |= FTRACE_OPS_FL_SAVE_REGS;
0124 }
0125
0126 static int fprobe_init_rethook(struct fprobe *fp, int num)
0127 {
0128     int i, size;
0129
0130     if (num < 0)
0131         return -EINVAL;
0132
0133     if (!fp->exit_handler) {
0134         fp->rethook = NULL;
0135         return 0;
0136     }
0137
0138     /* Initialize rethook if needed */
0139     size = num * num_possible_cpus() * 2;
0140     if (size < 0)
0141         return -E2BIG;
0142
0143     fp->rethook = rethook_alloc((void *)fp, fprobe_exit_handler);
0144     for (i = 0; i < size; i++) {
0145         struct fprobe_rethook_node *node;
0146
0147         node = kzalloc(sizeof(*node), GFP_KERNEL);
0148         if (!node) {
0149             rethook_free(fp->rethook);
0150             fp->rethook = NULL;
0151             return -ENOMEM;
0152         }
0153         rethook_add_node(fp->rethook, &node->node);
0154     }
0155     return 0;
0156 }
0157
0158 static void fprobe_fail_cleanup(struct fprobe *fp)
0159 {
0160     if (fp->rethook) {
0161         /* Don't need to cleanup rethook->handler because this is not used. */
0162         rethook_free(fp->rethook);
0163         fp->rethook = NULL;
0164     }
0165     ftrace_free_filter(&fp->ops);
0166 }
0167
0168 /**
0169  * register_fprobe() - Register fprobe to ftrace by pattern.
0170  * @fp: A fprobe data structure to be registered.
0171  * @filter: A wildcard pattern of probed symbols.
0172  * @notfilter: A wildcard pattern of NOT probed symbols.
0173  *
0174  * Register @fp to ftrace for enabling the probe on the symbols matched to @filter.
0175  * If @notfilter is not NULL, the symbols matched the @notfilter are not probed.
0176  *
0177  * Return 0 if @fp is registered successfully, -errno if not.
0178  */
0179 int register_fprobe(struct fprobe *fp, const char *filter, const char *notfilter)
0180 {
0181     struct ftrace_hash *hash;
0182     unsigned char *str;
0183     int ret, len;
0184
0185     if (!fp || !filter)
0186         return -EINVAL;
0187
0188     fprobe_init(fp);
0189
0190     len = strlen(filter);
0191     str = kstrdup(filter, GFP_KERNEL);
0192     ret = ftrace_set_filter(&fp->ops, str, len, 0);
0193     kfree(str);
0194     if (ret)
0195         return ret;
0196
0197     if (notfilter) {
0198         len = strlen(notfilter);
0199         str = kstrdup(notfilter, GFP_KERNEL);
0200         ret = ftrace_set_notrace(&fp->ops, str, len, 0);
0201         kfree(str);
0202         if (ret)
0203             goto out;
0204     }
0205
0206     /* TODO:
0207      * correctly calculate the total number of filtered symbols
0208      * from both filter and notfilter.
0209      */
0210     hash = rcu_access_pointer(fp->ops.local_hash.filter_hash);
0211     if (WARN_ON_ONCE(!hash))
0212         goto out;
0213
0214     ret = fprobe_init_rethook(fp, (int)hash->count);
0215     if (!ret)
0216         ret = register_ftrace_function(&fp->ops);
0217
0218 out:
0219     if (ret)
0220         fprobe_fail_cleanup(fp);
0221     return ret;
0222 }
0223 EXPORT_SYMBOL_GPL(register_fprobe);
0224
0225 /**
0226  * register_fprobe_ips() - Register fprobe to ftrace by address.
0227  * @fp: A fprobe data structure to be registered.
0228  * @addrs: An array of target ftrace location addresses.
0229  * @num: The number of entries of @addrs.
0230  *
0231  * Register @fp to ftrace for enabling the probe on the address given by @addrs.
0232  * The @addrs must be the addresses of ftrace location address, which may be
0233  * the symbol address + arch-dependent offset.
0234  * If you unsure what this mean, please use other registration functions.
0235  *
0236  * Return 0 if @fp is registered successfully, -errno if not.
0237  */
0238 int register_fprobe_ips(struct fprobe *fp, unsigned long *addrs, int num)
0239 {
0240     int ret;
0241
0242     if (!fp || !addrs || num <= 0)
0243         return -EINVAL;
0244
0245     fprobe_init(fp);
0246
0247     ret = ftrace_set_filter_ips(&fp->ops, addrs, num, 0, 0);
0248     if (ret)
0249         return ret;
0250
0251     ret = fprobe_init_rethook(fp, num);
0252     if (!ret)
0253         ret = register_ftrace_function(&fp->ops);
0254
0255     if (ret)
0256         fprobe_fail_cleanup(fp);
0257     return ret;
0258 }
0259 EXPORT_SYMBOL_GPL(register_fprobe_ips);
0260
0261 /**
0262  * register_fprobe_syms() - Register fprobe to ftrace by symbols.
0263  * @fp: A fprobe data structure to be registered.
0264  * @syms: An array of target symbols.
0265  * @num: The number of entries of @syms.
0266  *
0267  * Register @fp to the symbols given by @syms array. This will be useful if
0268  * you are sure the symbols exist in the kernel.
0269  *
0270  * Return 0 if @fp is registered successfully, -errno if not.
0271  */
0272 int register_fprobe_syms(struct fprobe *fp, const char **syms, int num)
0273 {
0274     unsigned long *addrs;
0275     int ret;
0276
0277     if (!fp || !syms || num <= 0)
0278         return -EINVAL;
0279
0280     addrs = get_ftrace_locations(syms, num);
0281     if (IS_ERR(addrs))
0282         return PTR_ERR(addrs);
0283
0284     ret = register_fprobe_ips(fp, addrs, num);
0285
0286     kfree(addrs);
0287
0288     return ret;
0289 }
0290 EXPORT_SYMBOL_GPL(register_fprobe_syms);
0291
0292 /**
0293  * unregister_fprobe() - Unregister fprobe from ftrace
0294  * @fp: A fprobe data structure to be unregistered.
0295  *
0296  * Unregister fprobe (and remove ftrace hooks from the function entries).
0297  *
0298  * Return 0 if @fp is unregistered successfully, -errno if not.
0299  */
0300 int unregister_fprobe(struct fprobe *fp)
0301 {
0302     int ret;
0303
0304     if (!fp || fp->ops.func != fprobe_handler)
0305         return -EINVAL;
0306
0307     /*
0308      * rethook_free() starts disabling the rethook, but the rethook handlers
0309      * may be running on other processors at this point. To make sure that all
0310      * current running handlers are finished, call unregister_ftrace_function()
0311      * after this.
0312      */
0313     if (fp->rethook)
0314         rethook_free(fp->rethook);
0315
0316     ret = unregister_ftrace_function(&fp->ops);
0317     if (ret < 0)
0318         return ret;
0319
0320     ftrace_free_filter(&fp->ops);
0321
0322     return ret;
0323 }
0324 EXPORT_SYMBOL_GPL(unregister_fprobe);