kernel/livepatch/transition.c

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  * transition.c - Kernel Live Patching transition functions
0004  *
0005  * Copyright (C) 2015-2016 Josh Poimboeuf <jpoimboe@redhat.com>
0006  */
0007
0008 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
0009
0010 #include <linux/cpu.h>
0011 #include <linux/stacktrace.h>
0012 #include "core.h"
0013 #include "patch.h"
0014 #include "transition.h"
0015
0016 #define MAX_STACK_ENTRIES  100
0017 #define STACK_ERR_BUF_SIZE 128
0018
0019 #define SIGNALS_TIMEOUT 15
0020
0021 struct klp_patch *klp_transition_patch;
0022
0023 static int klp_target_state = KLP_UNDEFINED;
0024
0025 static unsigned int klp_signals_cnt;
0026
0027 /*
0028  * This work can be performed periodically to finish patching or unpatching any
0029  * "straggler" tasks which failed to transition in the first attempt.
0030  */
0031 static void klp_transition_work_fn(struct work_struct *work)
0032 {
0033     mutex_lock(&klp_mutex);
0034
0035     if (klp_transition_patch)
0036         klp_try_complete_transition();
0037
0038     mutex_unlock(&klp_mutex);
0039 }
0040 static DECLARE_DELAYED_WORK(klp_transition_work, klp_transition_work_fn);
0041
0042 /*
0043  * This function is just a stub to implement a hard force
0044  * of synchronize_rcu(). This requires synchronizing
0045  * tasks even in userspace and idle.
0046  */
0047 static void klp_sync(struct work_struct *work)
0048 {
0049 }
0050
0051 /*
0052  * We allow to patch also functions where RCU is not watching,
0053  * e.g. before user_exit(). We can not rely on the RCU infrastructure
0054  * to do the synchronization. Instead hard force the sched synchronization.
0055  *
0056  * This approach allows to use RCU functions for manipulating func_stack
0057  * safely.
0058  */
0059 static void klp_synchronize_transition(void)
0060 {
0061     schedule_on_each_cpu(klp_sync);
0062 }
0063
0064 /*
0065  * The transition to the target patch state is complete.  Clean up the data
0066  * structures.
0067  */
0068 static void klp_complete_transition(void)
0069 {
0070     struct klp_object *obj;
0071     struct klp_func *func;
0072     struct task_struct *g, *task;
0073     unsigned int cpu;
0074
0075     pr_debug("'%s': completing %s transition\n",
0076          klp_transition_patch->mod->name,
0077          klp_target_state == KLP_PATCHED ? "patching" : "unpatching");
0078
0079     if (klp_transition_patch->replace && klp_target_state == KLP_PATCHED) {
0080         klp_unpatch_replaced_patches(klp_transition_patch);
0081         klp_discard_nops(klp_transition_patch);
0082     }
0083
0084     if (klp_target_state == KLP_UNPATCHED) {
0085         /*
0086          * All tasks have transitioned to KLP_UNPATCHED so we can now
0087          * remove the new functions from the func_stack.
0088          */
0089         klp_unpatch_objects(klp_transition_patch);
0090
0091         /*
0092          * Make sure klp_ftrace_handler() can no longer see functions
0093          * from this patch on the ops->func_stack.  Otherwise, after
0094          * func->transition gets cleared, the handler may choose a
0095          * removed function.
0096          */
0097         klp_synchronize_transition();
0098     }
0099
0100     klp_for_each_object(klp_transition_patch, obj)
0101         klp_for_each_func(obj, func)
0102             func->transition = false;
0103
0104     /* Prevent klp_ftrace_handler() from seeing KLP_UNDEFINED state */
0105     if (klp_target_state == KLP_PATCHED)
0106         klp_synchronize_transition();
0107
0108     read_lock(&tasklist_lock);
0109     for_each_process_thread(g, task) {
0110         WARN_ON_ONCE(test_tsk_thread_flag(task, TIF_PATCH_PENDING));
0111         task->patch_state = KLP_UNDEFINED;
0112     }
0113     read_unlock(&tasklist_lock);
0114
0115     for_each_possible_cpu(cpu) {
0116         task = idle_task(cpu);
0117         WARN_ON_ONCE(test_tsk_thread_flag(task, TIF_PATCH_PENDING));
0118         task->patch_state = KLP_UNDEFINED;
0119     }
0120
0121     klp_for_each_object(klp_transition_patch, obj) {
0122         if (!klp_is_object_loaded(obj))
0123             continue;
0124         if (klp_target_state == KLP_PATCHED)
0125             klp_post_patch_callback(obj);
0126         else if (klp_target_state == KLP_UNPATCHED)
0127             klp_post_unpatch_callback(obj);
0128     }
0129
0130     pr_notice("'%s': %s complete\n", klp_transition_patch->mod->name,
0131           klp_target_state == KLP_PATCHED ? "patching" : "unpatching");
0132
0133     klp_target_state = KLP_UNDEFINED;
0134     klp_transition_patch = NULL;
0135 }
0136
0137 /*
0138  * This is called in the error path, to cancel a transition before it has
0139  * started, i.e. klp_init_transition() has been called but
0140  * klp_start_transition() hasn't.  If the transition *has* been started,
0141  * klp_reverse_transition() should be used instead.
0142  */
0143 void klp_cancel_transition(void)
0144 {
0145     if (WARN_ON_ONCE(klp_target_state != KLP_PATCHED))
0146         return;
0147
0148     pr_debug("'%s': canceling patching transition, going to unpatch\n",
0149          klp_transition_patch->mod->name);
0150
0151     klp_target_state = KLP_UNPATCHED;
0152     klp_complete_transition();
0153 }
0154
0155 /*
0156  * Switch the patched state of the task to the set of functions in the target
0157  * patch state.
0158  *
0159  * NOTE: If task is not 'current', the caller must ensure the task is inactive.
0160  * Otherwise klp_ftrace_handler() might read the wrong 'patch_state' value.
0161  */
0162 void klp_update_patch_state(struct task_struct *task)
0163 {
0164     /*
0165      * A variant of synchronize_rcu() is used to allow patching functions
0166      * where RCU is not watching, see klp_synchronize_transition().
0167      */
0168     preempt_disable_notrace();
0169
0170     /*
0171      * This test_and_clear_tsk_thread_flag() call also serves as a read
0172      * barrier (smp_rmb) for two cases:
0173      *
0174      * 1) Enforce the order of the TIF_PATCH_PENDING read and the
0175      *    klp_target_state read.  The corresponding write barrier is in
0176      *    klp_init_transition().
0177      *
0178      * 2) Enforce the order of the TIF_PATCH_PENDING read and a future read
0179      *    of func->transition, if klp_ftrace_handler() is called later on
0180      *    the same CPU.  See __klp_disable_patch().
0181      */
0182     if (test_and_clear_tsk_thread_flag(task, TIF_PATCH_PENDING))
0183         task->patch_state = READ_ONCE(klp_target_state);
0184
0185     preempt_enable_notrace();
0186 }
0187
0188 /*
0189  * Determine whether the given stack trace includes any references to a
0190  * to-be-patched or to-be-unpatched function.
0191  */
0192 static int klp_check_stack_func(struct klp_func *func, unsigned long *entries,
0193                 unsigned int nr_entries)
0194 {
0195     unsigned long func_addr, func_size, address;
0196     struct klp_ops *ops;
0197     int i;
0198
0199     for (i = 0; i < nr_entries; i++) {
0200         address = entries[i];
0201
0202         if (klp_target_state == KLP_UNPATCHED) {
0203              /*
0204               * Check for the to-be-unpatched function
0205               * (the func itself).
0206               */
0207             func_addr = (unsigned long)func->new_func;
0208             func_size = func->new_size;
0209         } else {
0210             /*
0211              * Check for the to-be-patched function
0212              * (the previous func).
0213              */
0214             ops = klp_find_ops(func->old_func);
0215
0216             if (list_is_singular(&ops->func_stack)) {
0217                 /* original function */
0218                 func_addr = (unsigned long)func->old_func;
0219                 func_size = func->old_size;
0220             } else {
0221                 /* previously patched function */
0222                 struct klp_func *prev;
0223
0224                 prev = list_next_entry(func, stack_node);
0225                 func_addr = (unsigned long)prev->new_func;
0226                 func_size = prev->new_size;
0227             }
0228         }
0229
0230         if (address >= func_addr && address < func_addr + func_size)
0231             return -EAGAIN;
0232     }
0233
0234     return 0;
0235 }
0236
0237 /*
0238  * Determine whether it's safe to transition the task to the target patch state
0239  * by looking for any to-be-patched or to-be-unpatched functions on its stack.
0240  */
0241 static int klp_check_stack(struct task_struct *task, const char **oldname)
0242 {
0243     static unsigned long entries[MAX_STACK_ENTRIES];
0244     struct klp_object *obj;
0245     struct klp_func *func;
0246     int ret, nr_entries;
0247
0248     ret = stack_trace_save_tsk_reliable(task, entries, ARRAY_SIZE(entries));
0249     if (ret < 0)
0250         return -EINVAL;
0251     nr_entries = ret;
0252
0253     klp_for_each_object(klp_transition_patch, obj) {
0254         if (!obj->patched)
0255             continue;
0256         klp_for_each_func(obj, func) {
0257             ret = klp_check_stack_func(func, entries, nr_entries);
0258             if (ret) {
0259                 *oldname = func->old_name;
0260                 return -EADDRINUSE;
0261             }
0262         }
0263     }
0264
0265     return 0;
0266 }
0267
0268 static int klp_check_and_switch_task(struct task_struct *task, void *arg)
0269 {
0270     int ret;
0271
0272     if (task_curr(task) && task != current)
0273         return -EBUSY;
0274
0275     ret = klp_check_stack(task, arg);
0276     if (ret)
0277         return ret;
0278
0279     clear_tsk_thread_flag(task, TIF_PATCH_PENDING);
0280     task->patch_state = klp_target_state;
0281     return 0;
0282 }
0283
0284 /*
0285  * Try to safely switch a task to the target patch state.  If it's currently
0286  * running, or it's sleeping on a to-be-patched or to-be-unpatched function, or
0287  * if the stack is unreliable, return false.
0288  */
0289 static bool klp_try_switch_task(struct task_struct *task)
0290 {
0291     const char *old_name;
0292     int ret;
0293
0294     /* check if this task has already switched over */
0295     if (task->patch_state == klp_target_state)
0296         return true;
0297
0298     /*
0299      * For arches which don't have reliable stack traces, we have to rely
0300      * on other methods (e.g., switching tasks at kernel exit).
0301      */
0302     if (!klp_have_reliable_stack())
0303         return false;
0304
0305     /*
0306      * Now try to check the stack for any to-be-patched or to-be-unpatched
0307      * functions.  If all goes well, switch the task to the target patch
0308      * state.
0309      */
0310     ret = task_call_func(task, klp_check_and_switch_task, &old_name);
0311     switch (ret) {
0312     case 0:     /* success */
0313         break;
0314
0315     case -EBUSY:    /* klp_check_and_switch_task() */
0316         pr_debug("%s: %s:%d is running\n",
0317              __func__, task->comm, task->pid);
0318         break;
0319     case -EINVAL:   /* klp_check_and_switch_task() */
0320         pr_debug("%s: %s:%d has an unreliable stack\n",
0321              __func__, task->comm, task->pid);
0322         break;
0323     case -EADDRINUSE: /* klp_check_and_switch_task() */
0324         pr_debug("%s: %s:%d is sleeping on function %s\n",
0325              __func__, task->comm, task->pid, old_name);
0326         break;
0327
0328     default:
0329         pr_debug("%s: Unknown error code (%d) when trying to switch %s:%d\n",
0330              __func__, ret, task->comm, task->pid);
0331         break;
0332     }
0333
0334     return !ret;
0335 }
0336
0337 /*
0338  * Sends a fake signal to all non-kthread tasks with TIF_PATCH_PENDING set.
0339  * Kthreads with TIF_PATCH_PENDING set are woken up.
0340  */
0341 static void klp_send_signals(void)
0342 {
0343     struct task_struct *g, *task;
0344
0345     if (klp_signals_cnt == SIGNALS_TIMEOUT)
0346         pr_notice("signaling remaining tasks\n");
0347
0348     read_lock(&tasklist_lock);
0349     for_each_process_thread(g, task) {
0350         if (!klp_patch_pending(task))
0351             continue;
0352
0353         /*
0354          * There is a small race here. We could see TIF_PATCH_PENDING
0355          * set and decide to wake up a kthread or send a fake signal.
0356          * Meanwhile the task could migrate itself and the action
0357          * would be meaningless. It is not serious though.
0358          */
0359         if (task->flags & PF_KTHREAD) {
0360             /*
0361              * Wake up a kthread which sleeps interruptedly and
0362              * still has not been migrated.
0363              */
0364             wake_up_state(task, TASK_INTERRUPTIBLE);
0365         } else {
0366             /*
0367              * Send fake signal to all non-kthread tasks which are
0368              * still not migrated.
0369              */
0370             set_notify_signal(task);
0371         }
0372     }
0373     read_unlock(&tasklist_lock);
0374 }
0375
0376 /*
0377  * Try to switch all remaining tasks to the target patch state by walking the
0378  * stacks of sleeping tasks and looking for any to-be-patched or
0379  * to-be-unpatched functions.  If such functions are found, the task can't be
0380  * switched yet.
0381  *
0382  * If any tasks are still stuck in the initial patch state, schedule a retry.
0383  */
0384 void klp_try_complete_transition(void)
0385 {
0386     unsigned int cpu;
0387     struct task_struct *g, *task;
0388     struct klp_patch *patch;
0389     bool complete = true;
0390
0391     WARN_ON_ONCE(klp_target_state == KLP_UNDEFINED);
0392
0393     /*
0394      * Try to switch the tasks to the target patch state by walking their
0395      * stacks and looking for any to-be-patched or to-be-unpatched
0396      * functions.  If such functions are found on a stack, or if the stack
0397      * is deemed unreliable, the task can't be switched yet.
0398      *
0399      * Usually this will transition most (or all) of the tasks on a system
0400      * unless the patch includes changes to a very common function.
0401      */
0402     read_lock(&tasklist_lock);
0403     for_each_process_thread(g, task)
0404         if (!klp_try_switch_task(task))
0405             complete = false;
0406     read_unlock(&tasklist_lock);
0407
0408     /*
0409      * Ditto for the idle "swapper" tasks.
0410      */
0411     cpus_read_lock();
0412     for_each_possible_cpu(cpu) {
0413         task = idle_task(cpu);
0414         if (cpu_online(cpu)) {
0415             if (!klp_try_switch_task(task)) {
0416                 complete = false;
0417                 /* Make idle task go through the main loop. */
0418                 wake_up_if_idle(cpu);
0419             }
0420         } else if (task->patch_state != klp_target_state) {
0421             /* offline idle tasks can be switched immediately */
0422             clear_tsk_thread_flag(task, TIF_PATCH_PENDING);
0423             task->patch_state = klp_target_state;
0424         }
0425     }
0426     cpus_read_unlock();
0427
0428     if (!complete) {
0429         if (klp_signals_cnt && !(klp_signals_cnt % SIGNALS_TIMEOUT))
0430             klp_send_signals();
0431         klp_signals_cnt++;
0432
0433         /*
0434          * Some tasks weren't able to be switched over.  Try again
0435          * later and/or wait for other methods like kernel exit
0436          * switching.
0437          */
0438         schedule_delayed_work(&klp_transition_work,
0439                       round_jiffies_relative(HZ));
0440         return;
0441     }
0442
0443     /* we're done, now cleanup the data structures */
0444     patch = klp_transition_patch;
0445     klp_complete_transition();
0446
0447     /*
0448      * It would make more sense to free the unused patches in
0449      * klp_complete_transition() but it is called also
0450      * from klp_cancel_transition().
0451      */
0452     if (!patch->enabled)
0453         klp_free_patch_async(patch);
0454     else if (patch->replace)
0455         klp_free_replaced_patches_async(patch);
0456 }
0457
0458 /*
0459  * Start the transition to the specified target patch state so tasks can begin
0460  * switching to it.
0461  */
0462 void klp_start_transition(void)
0463 {
0464     struct task_struct *g, *task;
0465     unsigned int cpu;
0466
0467     WARN_ON_ONCE(klp_target_state == KLP_UNDEFINED);
0468
0469     pr_notice("'%s': starting %s transition\n",
0470           klp_transition_patch->mod->name,
0471           klp_target_state == KLP_PATCHED ? "patching" : "unpatching");
0472
0473     /*
0474      * Mark all normal tasks as needing a patch state update.  They'll
0475      * switch either in klp_try_complete_transition() or as they exit the
0476      * kernel.
0477      */
0478     read_lock(&tasklist_lock);
0479     for_each_process_thread(g, task)
0480         if (task->patch_state != klp_target_state)
0481             set_tsk_thread_flag(task, TIF_PATCH_PENDING);
0482     read_unlock(&tasklist_lock);
0483
0484     /*
0485      * Mark all idle tasks as needing a patch state update.  They'll switch
0486      * either in klp_try_complete_transition() or at the idle loop switch
0487      * point.
0488      */
0489     for_each_possible_cpu(cpu) {
0490         task = idle_task(cpu);
0491         if (task->patch_state != klp_target_state)
0492             set_tsk_thread_flag(task, TIF_PATCH_PENDING);
0493     }
0494
0495     klp_signals_cnt = 0;
0496 }
0497
0498 /*
0499  * Initialize the global target patch state and all tasks to the initial patch
0500  * state, and initialize all function transition states to true in preparation
0501  * for patching or unpatching.
0502  */
0503 void klp_init_transition(struct klp_patch *patch, int state)
0504 {
0505     struct task_struct *g, *task;
0506     unsigned int cpu;
0507     struct klp_object *obj;
0508     struct klp_func *func;
0509     int initial_state = !state;
0510
0511     WARN_ON_ONCE(klp_target_state != KLP_UNDEFINED);
0512
0513     klp_transition_patch = patch;
0514
0515     /*
0516      * Set the global target patch state which tasks will switch to.  This
0517      * has no effect until the TIF_PATCH_PENDING flags get set later.
0518      */
0519     klp_target_state = state;
0520
0521     pr_debug("'%s': initializing %s transition\n", patch->mod->name,
0522          klp_target_state == KLP_PATCHED ? "patching" : "unpatching");
0523
0524     /*
0525      * Initialize all tasks to the initial patch state to prepare them for
0526      * switching to the target state.
0527      */
0528     read_lock(&tasklist_lock);
0529     for_each_process_thread(g, task) {
0530         WARN_ON_ONCE(task->patch_state != KLP_UNDEFINED);
0531         task->patch_state = initial_state;
0532     }
0533     read_unlock(&tasklist_lock);
0534
0535     /*
0536      * Ditto for the idle "swapper" tasks.
0537      */
0538     for_each_possible_cpu(cpu) {
0539         task = idle_task(cpu);
0540         WARN_ON_ONCE(task->patch_state != KLP_UNDEFINED);
0541         task->patch_state = initial_state;
0542     }
0543
0544     /*
0545      * Enforce the order of the task->patch_state initializations and the
0546      * func->transition updates to ensure that klp_ftrace_handler() doesn't
0547      * see a func in transition with a task->patch_state of KLP_UNDEFINED.
0548      *
0549      * Also enforce the order of the klp_target_state write and future
0550      * TIF_PATCH_PENDING writes to ensure klp_update_patch_state() doesn't
0551      * set a task->patch_state to KLP_UNDEFINED.
0552      */
0553     smp_wmb();
0554
0555     /*
0556      * Set the func transition states so klp_ftrace_handler() will know to
0557      * switch to the transition logic.
0558      *
0559      * When patching, the funcs aren't yet in the func_stack and will be
0560      * made visible to the ftrace handler shortly by the calls to
0561      * klp_patch_object().
0562      *
0563      * When unpatching, the funcs are already in the func_stack and so are
0564      * already visible to the ftrace handler.
0565      */
0566     klp_for_each_object(patch, obj)
0567         klp_for_each_func(obj, func)
0568             func->transition = true;
0569 }
0570
0571 /*
0572  * This function can be called in the middle of an existing transition to
0573  * reverse the direction of the target patch state.  This can be done to
0574  * effectively cancel an existing enable or disable operation if there are any
0575  * tasks which are stuck in the initial patch state.
0576  */
0577 void klp_reverse_transition(void)
0578 {
0579     unsigned int cpu;
0580     struct task_struct *g, *task;
0581
0582     pr_debug("'%s': reversing transition from %s\n",
0583          klp_transition_patch->mod->name,
0584          klp_target_state == KLP_PATCHED ? "patching to unpatching" :
0585                            "unpatching to patching");
0586
0587     klp_transition_patch->enabled = !klp_transition_patch->enabled;
0588
0589     klp_target_state = !klp_target_state;
0590
0591     /*
0592      * Clear all TIF_PATCH_PENDING flags to prevent races caused by
0593      * klp_update_patch_state() running in parallel with
0594      * klp_start_transition().
0595      */
0596     read_lock(&tasklist_lock);
0597     for_each_process_thread(g, task)
0598         clear_tsk_thread_flag(task, TIF_PATCH_PENDING);
0599     read_unlock(&tasklist_lock);
0600
0601     for_each_possible_cpu(cpu)
0602         clear_tsk_thread_flag(idle_task(cpu), TIF_PATCH_PENDING);
0603
0604     /* Let any remaining calls to klp_update_patch_state() complete */
0605     klp_synchronize_transition();
0606
0607     klp_start_transition();
0608 }
0609
0610 /* Called from copy_process() during fork */
0611 void klp_copy_process(struct task_struct *child)
0612 {
0613     child->patch_state = current->patch_state;
0614
0615     /* TIF_PATCH_PENDING gets copied in setup_thread_stack() */
0616 }
0617
0618 /*
0619  * Drop TIF_PATCH_PENDING of all tasks on admin's request. This forces an
0620  * existing transition to finish.
0621  *
0622  * NOTE: klp_update_patch_state(task) requires the task to be inactive or
0623  * 'current'. This is not the case here and the consistency model could be
0624  * broken. Administrator, who is the only one to execute the
0625  * klp_force_transitions(), has to be aware of this.
0626  */
0627 void klp_force_transition(void)
0628 {
0629     struct klp_patch *patch;
0630     struct task_struct *g, *task;
0631     unsigned int cpu;
0632
0633     pr_warn("forcing remaining tasks to the patched state\n");
0634
0635     read_lock(&tasklist_lock);
0636     for_each_process_thread(g, task)
0637         klp_update_patch_state(task);
0638     read_unlock(&tasklist_lock);
0639
0640     for_each_possible_cpu(cpu)
0641         klp_update_patch_state(idle_task(cpu));
0642
0643     /* Set forced flag for patches being removed. */
0644     if (klp_target_state == KLP_UNPATCHED)
0645         klp_transition_patch->forced = true;
0646     else if (klp_transition_patch->replace) {
0647         klp_for_each_patch(patch) {
0648             if (patch != klp_transition_patch)
0649                 patch->forced = true;
0650         }
0651     }
0652 }