kernel/trace/rethook.c

0001 // SPDX-License-Identifier: GPL-2.0
0002
0003 #define pr_fmt(fmt) "rethook: " fmt
0004
0005 #include <linux/bug.h>
0006 #include <linux/kallsyms.h>
0007 #include <linux/kprobes.h>
0008 #include <linux/preempt.h>
0009 #include <linux/rethook.h>
0010 #include <linux/slab.h>
0011 #include <linux/sort.h>
0012
0013 /* Return hook list (shadow stack by list) */
0014
0015 /*
0016  * This function is called from delayed_put_task_struct() when a task is
0017  * dead and cleaned up to recycle any kretprobe instances associated with
0018  * this task. These left over instances represent probed functions that
0019  * have been called but will never return.
0020  */
0021 void rethook_flush_task(struct task_struct *tk)
0022 {
0023     struct rethook_node *rhn;
0024     struct llist_node *node;
0025
0026     node = __llist_del_all(&tk->rethooks);
0027     while (node) {
0028         rhn = container_of(node, struct rethook_node, llist);
0029         node = node->next;
0030         preempt_disable();
0031         rethook_recycle(rhn);
0032         preempt_enable();
0033     }
0034 }
0035
0036 static void rethook_free_rcu(struct rcu_head *head)
0037 {
0038     struct rethook *rh = container_of(head, struct rethook, rcu);
0039     struct rethook_node *rhn;
0040     struct freelist_node *node;
0041     int count = 1;
0042
0043     node = rh->pool.head;
0044     while (node) {
0045         rhn = container_of(node, struct rethook_node, freelist);
0046         node = node->next;
0047         kfree(rhn);
0048         count++;
0049     }
0050
0051     /* The rh->ref is the number of pooled node + 1 */
0052     if (refcount_sub_and_test(count, &rh->ref))
0053         kfree(rh);
0054 }
0055
0056 /**
0057  * rethook_free() - Free struct rethook.
0058  * @rh: the struct rethook to be freed.
0059  *
0060  * Free the rethook. Before calling this function, user must ensure the
0061  * @rh::data is cleaned if needed (or, the handler can access it after
0062  * calling this function.) This function will set the @rh to be freed
0063  * after all rethook_node are freed (not soon). And the caller must
0064  * not touch @rh after calling this.
0065  */
0066 void rethook_free(struct rethook *rh)
0067 {
0068     WRITE_ONCE(rh->handler, NULL);
0069
0070     call_rcu(&rh->rcu, rethook_free_rcu);
0071 }
0072
0073 /**
0074  * rethook_alloc() - Allocate struct rethook.
0075  * @data: a data to pass the @handler when hooking the return.
0076  * @handler: the return hook callback function.
0077  *
0078  * Allocate and initialize a new rethook with @data and @handler.
0079  * Return NULL if memory allocation fails or @handler is NULL.
0080  * Note that @handler == NULL means this rethook is going to be freed.
0081  */
0082 struct rethook *rethook_alloc(void *data, rethook_handler_t handler)
0083 {
0084     struct rethook *rh = kzalloc(sizeof(struct rethook), GFP_KERNEL);
0085
0086     if (!rh || !handler)
0087         return NULL;
0088
0089     rh->data = data;
0090     rh->handler = handler;
0091     rh->pool.head = NULL;
0092     refcount_set(&rh->ref, 1);
0093
0094     return rh;
0095 }
0096
0097 /**
0098  * rethook_add_node() - Add a new node to the rethook.
0099  * @rh: the struct rethook.
0100  * @node: the struct rethook_node to be added.
0101  *
0102  * Add @node to @rh. User must allocate @node (as a part of user's
0103  * data structure.) The @node fields are initialized in this function.
0104  */
0105 void rethook_add_node(struct rethook *rh, struct rethook_node *node)
0106 {
0107     node->rethook = rh;
0108     freelist_add(&node->freelist, &rh->pool);
0109     refcount_inc(&rh->ref);
0110 }
0111
0112 static void free_rethook_node_rcu(struct rcu_head *head)
0113 {
0114     struct rethook_node *node = container_of(head, struct rethook_node, rcu);
0115
0116     if (refcount_dec_and_test(&node->rethook->ref))
0117         kfree(node->rethook);
0118     kfree(node);
0119 }
0120
0121 /**
0122  * rethook_recycle() - return the node to rethook.
0123  * @node: The struct rethook_node to be returned.
0124  *
0125  * Return back the @node to @node::rethook. If the @node::rethook is already
0126  * marked as freed, this will free the @node.
0127  */
0128 void rethook_recycle(struct rethook_node *node)
0129 {
0130     lockdep_assert_preemption_disabled();
0131
0132     if (likely(READ_ONCE(node->rethook->handler)))
0133         freelist_add(&node->freelist, &node->rethook->pool);
0134     else
0135         call_rcu(&node->rcu, free_rethook_node_rcu);
0136 }
0137 NOKPROBE_SYMBOL(rethook_recycle);
0138
0139 /**
0140  * rethook_try_get() - get an unused rethook node.
0141  * @rh: The struct rethook which pools the nodes.
0142  *
0143  * Get an unused rethook node from @rh. If the node pool is empty, this
0144  * will return NULL. Caller must disable preemption.
0145  */
0146 struct rethook_node *rethook_try_get(struct rethook *rh)
0147 {
0148     rethook_handler_t handler = READ_ONCE(rh->handler);
0149     struct freelist_node *fn;
0150
0151     lockdep_assert_preemption_disabled();
0152
0153     /* Check whether @rh is going to be freed. */
0154     if (unlikely(!handler))
0155         return NULL;
0156
0157     /*
0158      * This expects the caller will set up a rethook on a function entry.
0159      * When the function returns, the rethook will eventually be reclaimed
0160      * or released in the rethook_recycle() with call_rcu().
0161      * This means the caller must be run in the RCU-availabe context.
0162      */
0163     if (unlikely(!rcu_is_watching()))
0164         return NULL;
0165
0166     fn = freelist_try_get(&rh->pool);
0167     if (!fn)
0168         return NULL;
0169
0170     return container_of(fn, struct rethook_node, freelist);
0171 }
0172 NOKPROBE_SYMBOL(rethook_try_get);
0173
0174 /**
0175  * rethook_hook() - Hook the current function return.
0176  * @node: The struct rethook node to hook the function return.
0177  * @regs: The struct pt_regs for the function entry.
0178  * @mcount: True if this is called from mcount(ftrace) context.
0179  *
0180  * Hook the current running function return. This must be called when the
0181  * function entry (or at least @regs must be the registers of the function
0182  * entry.) @mcount is used for identifying the context. If this is called
0183  * from ftrace (mcount) callback, @mcount must be set true. If this is called
0184  * from the real function entry (e.g. kprobes) @mcount must be set false.
0185  * This is because the way to hook the function return depends on the context.
0186  */
0187 void rethook_hook(struct rethook_node *node, struct pt_regs *regs, bool mcount)
0188 {
0189     arch_rethook_prepare(node, regs, mcount);
0190     __llist_add(&node->llist, &current->rethooks);
0191 }
0192 NOKPROBE_SYMBOL(rethook_hook);
0193
0194 /* This assumes the 'tsk' is the current task or is not running. */
0195 static unsigned long __rethook_find_ret_addr(struct task_struct *tsk,
0196                          struct llist_node **cur)
0197 {
0198     struct rethook_node *rh = NULL;
0199     struct llist_node *node = *cur;
0200
0201     if (!node)
0202         node = tsk->rethooks.first;
0203     else
0204         node = node->next;
0205
0206     while (node) {
0207         rh = container_of(node, struct rethook_node, llist);
0208         if (rh->ret_addr != (unsigned long)arch_rethook_trampoline) {
0209             *cur = node;
0210             return rh->ret_addr;
0211         }
0212         node = node->next;
0213     }
0214     return 0;
0215 }
0216 NOKPROBE_SYMBOL(__rethook_find_ret_addr);
0217
0218 /**
0219  * rethook_find_ret_addr -- Find correct return address modified by rethook
0220  * @tsk: Target task
0221  * @frame: A frame pointer
0222  * @cur: a storage of the loop cursor llist_node pointer for next call
0223  *
0224  * Find the correct return address modified by a rethook on @tsk in unsigned
0225  * long type.
0226  * The @tsk must be 'current' or a task which is not running. @frame is a hint
0227  * to get the currect return address - which is compared with the
0228  * rethook::frame field. The @cur is a loop cursor for searching the
0229  * kretprobe return addresses on the @tsk. The '*@cur' should be NULL at the
0230  * first call, but '@cur' itself must NOT NULL.
0231  *
0232  * Returns found address value or zero if not found.
0233  */
0234 unsigned long rethook_find_ret_addr(struct task_struct *tsk, unsigned long frame,
0235                     struct llist_node **cur)
0236 {
0237     struct rethook_node *rhn = NULL;
0238     unsigned long ret;
0239
0240     if (WARN_ON_ONCE(!cur))
0241         return 0;
0242
0243     if (WARN_ON_ONCE(tsk != current && task_is_running(tsk)))
0244         return 0;
0245
0246     do {
0247         ret = __rethook_find_ret_addr(tsk, cur);
0248         if (!ret)
0249             break;
0250         rhn = container_of(*cur, struct rethook_node, llist);
0251     } while (rhn->frame != frame);
0252
0253     return ret;
0254 }
0255 NOKPROBE_SYMBOL(rethook_find_ret_addr);
0256
0257 void __weak arch_rethook_fixup_return(struct pt_regs *regs,
0258                       unsigned long correct_ret_addr)
0259 {
0260     /*
0261      * Do nothing by default. If the architecture which uses a
0262      * frame pointer to record real return address on the stack,
0263      * it should fill this function to fixup the return address
0264      * so that stacktrace works from the rethook handler.
0265      */
0266 }
0267
0268 /* This function will be called from each arch-defined trampoline. */
0269 unsigned long rethook_trampoline_handler(struct pt_regs *regs,
0270                      unsigned long frame)
0271 {
0272     struct llist_node *first, *node = NULL;
0273     unsigned long correct_ret_addr;
0274     rethook_handler_t handler;
0275     struct rethook_node *rhn;
0276
0277     correct_ret_addr = __rethook_find_ret_addr(current, &node);
0278     if (!correct_ret_addr) {
0279         pr_err("rethook: Return address not found! Maybe there is a bug in the kernel\n");
0280         BUG_ON(1);
0281     }
0282
0283     instruction_pointer_set(regs, correct_ret_addr);
0284
0285     /*
0286      * These loops must be protected from rethook_free_rcu() because those
0287      * are accessing 'rhn->rethook'.
0288      */
0289     preempt_disable();
0290
0291     /*
0292      * Run the handler on the shadow stack. Do not unlink the list here because
0293      * stackdump inside the handlers needs to decode it.
0294      */
0295     first = current->rethooks.first;
0296     while (first) {
0297         rhn = container_of(first, struct rethook_node, llist);
0298         if (WARN_ON_ONCE(rhn->frame != frame))
0299             break;
0300         handler = READ_ONCE(rhn->rethook->handler);
0301         if (handler)
0302             handler(rhn, rhn->rethook->data, regs);
0303
0304         if (first == node)
0305             break;
0306         first = first->next;
0307     }
0308
0309     /* Fixup registers for returning to correct address. */
0310     arch_rethook_fixup_return(regs, correct_ret_addr);
0311
0312     /* Unlink used shadow stack */
0313     first = current->rethooks.first;
0314     current->rethooks.first = node->next;
0315     node->next = NULL;
0316
0317     while (first) {
0318         rhn = container_of(first, struct rethook_node, llist);
0319         first = first->next;
0320         rethook_recycle(rhn);
0321     }
0322     preempt_enable();
0323
0324     return correct_ret_addr;
0325 }
0326 NOKPROBE_SYMBOL(rethook_trampoline_handler);