arm/kernel/entry-ftrace.S

0001 /* SPDX-License-Identifier: GPL-2.0-only */
0002
0003 #include <asm/assembler.h>
0004 #include <asm/ftrace.h>
0005 #include <asm/unwind.h>
0006
0007 #include "entry-header.S"
0008
0009 /*
0010  * When compiling with -pg, gcc inserts a call to the mcount routine at the
0011  * start of every function.  In mcount, apart from the function's address (in
0012  * lr), we need to get hold of the function's caller's address.
0013  *
0014  * Newer GCCs (4.4+) solve this problem by using a version of mcount with call
0015  * sites like:
0016  *
0017  *  push    {lr}
0018  *  bl  __gnu_mcount_nc
0019  *
0020  * With these compilers, frame pointers are not necessary.
0021  *
0022  * mcount can be thought of as a function called in the middle of a subroutine
0023  * call.  As such, it needs to be transparent for both the caller and the
0024  * callee: the original lr needs to be restored when leaving mcount, and no
0025  * registers should be clobbered.
0026  *
0027  * When using dynamic ftrace, we patch out the mcount call by a "add sp, #4"
0028  * instead of the __gnu_mcount_nc call (see arch/arm/kernel/ftrace.c).
0029  */
0030
0031 .macro mcount_adjust_addr rd, rn
0032     bic \rd, \rn, #1        @ clear the Thumb bit if present
0033     sub \rd, \rd, #MCOUNT_INSN_SIZE
0034 .endm
0035
0036 .macro __mcount suffix
0037     mcount_enter
0038     ldr_va  r2, ftrace_trace_function
0039     badr    r0, .Lftrace_stub
0040     cmp r0, r2
0041     bne 1f
0042
0043 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
0044     ldr_va  r2, ftrace_graph_return
0045     cmp r0, r2
0046     bne ftrace_graph_caller\suffix
0047
0048     ldr_va  r2, ftrace_graph_entry
0049     mov_l   r0, ftrace_graph_entry_stub
0050     cmp r0, r2
0051     bne ftrace_graph_caller\suffix
0052 #endif
0053
0054     mcount_exit
0055
0056 1:  mcount_get_lr   r1          @ lr of instrumented func
0057     mcount_adjust_addr  r0, lr      @ instrumented function
0058     badr    lr, 2f
0059     mov pc, r2
0060 2:  mcount_exit
0061 .endm
0062
0063 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
0064
0065 .macro __ftrace_regs_caller
0066
0067     str lr, [sp, #-8]!  @ store LR as PC and make space for CPSR/OLD_R0,
0068                 @ OLD_R0 will overwrite previous LR
0069
0070     ldr lr, [sp, #8]    @ get previous LR
0071
0072     str r0, [sp, #8]    @ write r0 as OLD_R0 over previous LR
0073
0074     str lr, [sp, #-4]!  @ store previous LR as LR
0075
0076     add     lr, sp, #16 @ move in LR the value of SP as it was
0077                 @ before the push {lr} of the mcount mechanism
0078
0079     push    {r0-r11, ip, lr}
0080
0081     @ stack content at this point:
0082     @ 0  4          48   52       56            60   64    68       72
0083     @ R0 | R1 | ... | IP | SP + 4 | previous LR | LR | PSR | OLD_R0 |
0084
0085     mov r3, sp              @ struct pt_regs*
0086
0087     ldr_va  r2, function_trace_op       @ pointer to the current
0088                         @ function tracing op
0089
0090     ldr r1, [sp, #S_LR]         @ lr of instrumented func
0091
0092     ldr lr, [sp, #S_PC]         @ get LR
0093
0094     mcount_adjust_addr  r0, lr      @ instrumented function
0095
0096     .globl ftrace_regs_call
0097 ftrace_regs_call:
0098     bl  ftrace_stub
0099
0100 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
0101     .globl ftrace_graph_regs_call
0102 ftrace_graph_regs_call:
0103 ARM(    mov r0, r0  )
0104 THUMB(  nop.w       )
0105 #endif
0106
0107     @ pop saved regs
0108     pop {r0-r11, ip, lr}        @ restore r0 through r12
0109     ldr lr, [sp], #4            @ restore LR
0110     ldr pc, [sp], #12
0111 .endm
0112
0113 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
0114 .macro __ftrace_graph_regs_caller
0115
0116 #ifdef CONFIG_UNWINDER_FRAME_POINTER
0117     sub r0, fp, #4      @ lr of instrumented routine (parent)
0118 #else
0119     add r0, sp, #S_LR
0120 #endif
0121
0122     @ called from __ftrace_regs_caller
0123     ldr r1, [sp, #S_PC]     @ instrumented routine (func)
0124     mcount_adjust_addr  r1, r1
0125
0126     mov r2, fpreg       @ frame pointer
0127     add r3, sp, #PT_REGS_SIZE
0128     bl  prepare_ftrace_return
0129
0130     @ pop registers saved in ftrace_regs_caller
0131     pop {r0-r11, ip, lr}        @ restore r0 through r12
0132     ldr lr, [sp], #4            @ restore LR
0133     ldr pc, [sp], #12
0134
0135 .endm
0136 #endif
0137 #endif
0138
0139 .macro __ftrace_caller suffix
0140     mcount_enter
0141
0142     mcount_get_lr   r1          @ lr of instrumented func
0143     mcount_adjust_addr  r0, lr      @ instrumented function
0144
0145 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
0146     ldr_va  r2, function_trace_op       @ pointer to the current
0147                         @ function tracing op
0148     mov r3, #0              @ regs is NULL
0149 #endif
0150
0151     .globl ftrace_call\suffix
0152 ftrace_call\suffix:
0153     bl  ftrace_stub
0154
0155 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
0156     .globl ftrace_graph_call\suffix
0157 ftrace_graph_call\suffix:
0158 ARM(    mov r0, r0  )
0159 THUMB(  nop.w       )
0160 #endif
0161
0162     mcount_exit
0163 .endm
0164
0165 .macro __ftrace_graph_caller
0166 #ifdef CONFIG_UNWINDER_FRAME_POINTER
0167     sub r0, fp, #4      @ &lr of instrumented routine (&parent)
0168 #else
0169     add r0, sp, #20
0170 #endif
0171 #ifdef CONFIG_DYNAMIC_FTRACE
0172     @ called from __ftrace_caller, saved in mcount_enter
0173     ldr r1, [sp, #16]       @ instrumented routine (func)
0174     mcount_adjust_addr  r1, r1
0175 #else
0176     @ called from __mcount, untouched in lr
0177     mcount_adjust_addr  r1, lr  @ instrumented routine (func)
0178 #endif
0179     mov r2, fpreg       @ frame pointer
0180     add r3, sp, #24
0181     bl  prepare_ftrace_return
0182     mcount_exit
0183 .endm
0184
0185 /*
0186  * __gnu_mcount_nc
0187  */
0188
0189 .macro mcount_enter
0190 /*
0191  * This pad compensates for the push {lr} at the call site.  Note that we are
0192  * unable to unwind through a function which does not otherwise save its lr.
0193  */
0194  UNWIND(.pad    #4)
0195     stmdb   sp!, {r0-r3, lr}
0196  UNWIND(.save   {r0-r3, lr})
0197 .endm
0198
0199 .macro mcount_get_lr reg
0200     ldr \reg, [sp, #20]
0201 .endm
0202
0203 .macro mcount_exit
0204     ldmia   sp!, {r0-r3}
0205     ldr lr, [sp, #4]
0206     ldr pc, [sp], #8
0207 .endm
0208
0209 ENTRY(__gnu_mcount_nc)
0210 UNWIND(.fnstart)
0211 #ifdef CONFIG_DYNAMIC_FTRACE
0212     push    {lr}
0213     ldr lr, [sp, #4]
0214     ldr pc, [sp], #8
0215 #else
0216     __mcount
0217 #endif
0218 UNWIND(.fnend)
0219 ENDPROC(__gnu_mcount_nc)
0220
0221 #ifdef CONFIG_DYNAMIC_FTRACE
0222 ENTRY(ftrace_caller)
0223 UNWIND(.fnstart)
0224     __ftrace_caller
0225 UNWIND(.fnend)
0226 ENDPROC(ftrace_caller)
0227
0228 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
0229 ENTRY(ftrace_regs_caller)
0230 UNWIND(.fnstart)
0231     __ftrace_regs_caller
0232 UNWIND(.fnend)
0233 ENDPROC(ftrace_regs_caller)
0234 #endif
0235
0236 #endif
0237
0238 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
0239 ENTRY(ftrace_graph_caller)
0240 UNWIND(.fnstart)
0241     __ftrace_graph_caller
0242 UNWIND(.fnend)
0243 ENDPROC(ftrace_graph_caller)
0244
0245 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
0246 ENTRY(ftrace_graph_regs_caller)
0247 UNWIND(.fnstart)
0248     __ftrace_graph_regs_caller
0249 UNWIND(.fnend)
0250 ENDPROC(ftrace_graph_regs_caller)
0251 #endif
0252 #endif
0253
0254 .purgem mcount_enter
0255 .purgem mcount_get_lr
0256 .purgem mcount_exit
0257
0258 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
0259 ENTRY(return_to_handler)
0260     stmdb   sp!, {r0-r3}
0261     add r0, sp, #16     @ sp at exit of instrumented routine
0262     bl  ftrace_return_to_handler
0263     mov lr, r0          @ r0 has real ret addr
0264     ldmia   sp!, {r0-r3}
0265     ret lr
0266 ENDPROC(return_to_handler)
0267 #endif
0268
0269 ENTRY(ftrace_stub)
0270 .Lftrace_stub:
0271     ret lr
0272 ENDPROC(ftrace_stub)
0273
0274 #ifdef CONFIG_DYNAMIC_FTRACE
0275
0276     __INIT
0277
0278     .macro  init_tramp, dst:req
0279 ENTRY(\dst\()_from_init)
0280     ldr pc, =\dst
0281 ENDPROC(\dst\()_from_init)
0282     .endm
0283
0284     init_tramp  ftrace_caller
0285 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
0286     init_tramp  ftrace_regs_caller
0287 #endif
0288 #endif