arm/kernel/ftrace.c

0001 /*
0002  * Dynamic function tracing support.
0003  *
0004  * Copyright (C) 2008 Abhishek Sagar <sagar.abhishek@gmail.com>
0005  * Copyright (C) 2010 Rabin Vincent <rabin@rab.in>
0006  *
0007  * For licencing details, see COPYING.
0008  *
0009  * Defines low-level handling of mcount calls when the kernel
0010  * is compiled with the -pg flag. When using dynamic ftrace, the
0011  * mcount call-sites get patched with NOP till they are enabled.
0012  * All code mutation routines here are called under stop_machine().
0013  */
0014
0015 #include <linux/ftrace.h>
0016 #include <linux/uaccess.h>
0017 #include <linux/module.h>
0018 #include <linux/stop_machine.h>
0019
0020 #include <asm/cacheflush.h>
0021 #include <asm/opcodes.h>
0022 #include <asm/ftrace.h>
0023 #include <asm/insn.h>
0024 #include <asm/set_memory.h>
0025 #include <asm/stacktrace.h>
0026 #include <asm/patch.h>
0027
0028 /*
0029  * The compiler emitted profiling hook consists of
0030  *
0031  *   PUSH    {LR}
0032  *   BL      __gnu_mcount_nc
0033  *
0034  * To turn this combined sequence into a NOP, we need to restore the value of
0035  * SP before the PUSH. Let's use an ADD rather than a POP into LR, as LR is not
0036  * modified anyway, and reloading LR from memory is highly likely to be less
0037  * efficient.
0038  */
0039 #ifdef CONFIG_THUMB2_KERNEL
0040 #define NOP     0xf10d0d04  /* add.w sp, sp, #4 */
0041 #else
0042 #define NOP     0xe28dd004  /* add   sp, sp, #4 */
0043 #endif
0044
0045 #ifdef CONFIG_DYNAMIC_FTRACE
0046
0047 static int __ftrace_modify_code(void *data)
0048 {
0049     int *command = data;
0050
0051     ftrace_modify_all_code(*command);
0052
0053     return 0;
0054 }
0055
0056 void arch_ftrace_update_code(int command)
0057 {
0058     stop_machine(__ftrace_modify_code, &command, NULL);
0059 }
0060
0061 static unsigned long ftrace_nop_replace(struct dyn_ftrace *rec)
0062 {
0063     return NOP;
0064 }
0065
0066 void ftrace_caller_from_init(void);
0067 void ftrace_regs_caller_from_init(void);
0068
0069 static unsigned long __ref adjust_address(struct dyn_ftrace *rec,
0070                       unsigned long addr)
0071 {
0072     if (!IS_ENABLED(CONFIG_DYNAMIC_FTRACE) ||
0073         system_state >= SYSTEM_FREEING_INITMEM ||
0074         likely(!is_kernel_inittext(rec->ip)))
0075         return addr;
0076     if (!IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS) ||
0077         addr == (unsigned long)&ftrace_caller)
0078         return (unsigned long)&ftrace_caller_from_init;
0079     return (unsigned long)&ftrace_regs_caller_from_init;
0080 }
0081
0082 void ftrace_arch_code_modify_prepare(void)
0083 {
0084 }
0085
0086 void ftrace_arch_code_modify_post_process(void)
0087 {
0088     /* Make sure any TLB misses during machine stop are cleared. */
0089     flush_tlb_all();
0090 }
0091
0092 static unsigned long ftrace_call_replace(unsigned long pc, unsigned long addr,
0093                      bool warn)
0094 {
0095     return arm_gen_branch_link(pc, addr, warn);
0096 }
0097
0098 static int ftrace_modify_code(unsigned long pc, unsigned long old,
0099                   unsigned long new, bool validate)
0100 {
0101     unsigned long replaced;
0102
0103     if (IS_ENABLED(CONFIG_THUMB2_KERNEL))
0104         old = __opcode_to_mem_thumb32(old);
0105     else
0106         old = __opcode_to_mem_arm(old);
0107
0108     if (validate) {
0109         if (copy_from_kernel_nofault(&replaced, (void *)pc,
0110                 MCOUNT_INSN_SIZE))
0111             return -EFAULT;
0112
0113         if (replaced != old)
0114             return -EINVAL;
0115     }
0116
0117     __patch_text((void *)pc, new);
0118
0119     return 0;
0120 }
0121
0122 int ftrace_update_ftrace_func(ftrace_func_t func)
0123 {
0124     unsigned long pc;
0125     unsigned long new;
0126     int ret;
0127
0128     pc = (unsigned long)&ftrace_call;
0129     new = ftrace_call_replace(pc, (unsigned long)func, true);
0130
0131     ret = ftrace_modify_code(pc, 0, new, false);
0132
0133 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
0134     if (!ret) {
0135         pc = (unsigned long)&ftrace_regs_call;
0136         new = ftrace_call_replace(pc, (unsigned long)func, true);
0137
0138         ret = ftrace_modify_code(pc, 0, new, false);
0139     }
0140 #endif
0141
0142     return ret;
0143 }
0144
0145 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
0146 {
0147     unsigned long new, old;
0148     unsigned long ip = rec->ip;
0149     unsigned long aaddr = adjust_address(rec, addr);
0150     struct module *mod = NULL;
0151
0152 #ifdef CONFIG_ARM_MODULE_PLTS
0153     mod = rec->arch.mod;
0154 #endif
0155
0156     old = ftrace_nop_replace(rec);
0157
0158     new = ftrace_call_replace(ip, aaddr, !mod);
0159 #ifdef CONFIG_ARM_MODULE_PLTS
0160     if (!new && mod) {
0161         aaddr = get_module_plt(mod, ip, aaddr);
0162         new = ftrace_call_replace(ip, aaddr, true);
0163     }
0164 #endif
0165
0166     return ftrace_modify_code(rec->ip, old, new, true);
0167 }
0168
0169 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
0170
0171 int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
0172                 unsigned long addr)
0173 {
0174     unsigned long new, old;
0175     unsigned long ip = rec->ip;
0176
0177     old = ftrace_call_replace(ip, adjust_address(rec, old_addr), true);
0178
0179     new = ftrace_call_replace(ip, adjust_address(rec, addr), true);
0180
0181     return ftrace_modify_code(rec->ip, old, new, true);
0182 }
0183
0184 #endif
0185
0186 int ftrace_make_nop(struct module *mod,
0187             struct dyn_ftrace *rec, unsigned long addr)
0188 {
0189     unsigned long aaddr = adjust_address(rec, addr);
0190     unsigned long ip = rec->ip;
0191     unsigned long old;
0192     unsigned long new;
0193     int ret;
0194
0195 #ifdef CONFIG_ARM_MODULE_PLTS
0196     /* mod is only supplied during module loading */
0197     if (!mod)
0198         mod = rec->arch.mod;
0199     else
0200         rec->arch.mod = mod;
0201 #endif
0202
0203     old = ftrace_call_replace(ip, aaddr,
0204                   !IS_ENABLED(CONFIG_ARM_MODULE_PLTS) || !mod);
0205 #ifdef CONFIG_ARM_MODULE_PLTS
0206     if (!old && mod) {
0207         aaddr = get_module_plt(mod, ip, aaddr);
0208         old = ftrace_call_replace(ip, aaddr, true);
0209     }
0210 #endif
0211
0212     new = ftrace_nop_replace(rec);
0213     /*
0214      * Locations in .init.text may call __gnu_mcount_mc via a linker
0215      * emitted veneer if they are too far away from its implementation, and
0216      * so validation may fail spuriously in such cases. Let's work around
0217      * this by omitting those from validation.
0218      */
0219     ret = ftrace_modify_code(ip, old, new, !is_kernel_inittext(ip));
0220
0221     return ret;
0222 }
0223 #endif /* CONFIG_DYNAMIC_FTRACE */
0224
0225 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
0226 asmlinkage
0227 void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr,
0228                unsigned long frame_pointer,
0229                unsigned long stack_pointer)
0230 {
0231     unsigned long return_hooker = (unsigned long) &return_to_handler;
0232     unsigned long old;
0233
0234     if (unlikely(atomic_read(&current->tracing_graph_pause)))
0235         return;
0236
0237     if (IS_ENABLED(CONFIG_UNWINDER_FRAME_POINTER)) {
0238         /* FP points one word below parent's top of stack */
0239         frame_pointer += 4;
0240     } else {
0241         struct stackframe frame = {
0242             .fp = frame_pointer,
0243             .sp = stack_pointer,
0244             .lr = self_addr,
0245             .pc = self_addr,
0246         };
0247         if (unwind_frame(&frame) < 0)
0248             return;
0249         if (frame.lr != self_addr)
0250             parent = frame.lr_addr;
0251         frame_pointer = frame.sp;
0252     }
0253
0254     old = *parent;
0255     *parent = return_hooker;
0256
0257     if (function_graph_enter(old, self_addr, frame_pointer, NULL))
0258         *parent = old;
0259 }
0260
0261 #ifdef CONFIG_DYNAMIC_FTRACE
0262 extern unsigned long ftrace_graph_call;
0263 extern unsigned long ftrace_graph_call_old;
0264 extern void ftrace_graph_caller_old(void);
0265 extern unsigned long ftrace_graph_regs_call;
0266 extern void ftrace_graph_regs_caller(void);
0267
0268 static int __ftrace_modify_caller(unsigned long *callsite,
0269                   void (*func) (void), bool enable)
0270 {
0271     unsigned long caller_fn = (unsigned long) func;
0272     unsigned long pc = (unsigned long) callsite;
0273     unsigned long branch = arm_gen_branch(pc, caller_fn);
0274     unsigned long nop = arm_gen_nop();
0275     unsigned long old = enable ? nop : branch;
0276     unsigned long new = enable ? branch : nop;
0277
0278     return ftrace_modify_code(pc, old, new, true);
0279 }
0280
0281 static int ftrace_modify_graph_caller(bool enable)
0282 {
0283     int ret;
0284
0285     ret = __ftrace_modify_caller(&ftrace_graph_call,
0286                      ftrace_graph_caller,
0287                      enable);
0288
0289 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
0290     if (!ret)
0291         ret = __ftrace_modify_caller(&ftrace_graph_regs_call,
0292                      ftrace_graph_regs_caller,
0293                      enable);
0294 #endif
0295
0296
0297     return ret;
0298 }
0299
0300 int ftrace_enable_ftrace_graph_caller(void)
0301 {
0302     return ftrace_modify_graph_caller(true);
0303 }
0304
0305 int ftrace_disable_ftrace_graph_caller(void)
0306 {
0307     return ftrace_modify_graph_caller(false);
0308 }
0309 #endif /* CONFIG_DYNAMIC_FTRACE */
0310 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */