kernel/trace/ftrace_mprofile.S

0001 /* SPDX-License-Identifier: GPL-2.0-or-later */
0002 /*
0003  * Split from ftrace_64.S
0004  */
0005
0006 #include <linux/magic.h>
0007 #include <asm/ppc_asm.h>
0008 #include <asm/asm-offsets.h>
0009 #include <asm/ftrace.h>
0010 #include <asm/ppc-opcode.h>
0011 #include <asm/export.h>
0012 #include <asm/thread_info.h>
0013 #include <asm/bug.h>
0014 #include <asm/ptrace.h>
0015
0016 /*
0017  *
0018  * ftrace_caller()/ftrace_regs_caller() is the function that replaces _mcount()
0019  * when ftrace is active.
0020  *
0021  * We arrive here after a function A calls function B, and we are the trace
0022  * function for B. When we enter r1 points to A's stack frame, B has not yet
0023  * had a chance to allocate one yet.
0024  *
0025  * Additionally r2 may point either to the TOC for A, or B, depending on
0026  * whether B did a TOC setup sequence before calling us.
0027  *
0028  * On entry the LR points back to the _mcount() call site, and r0 holds the
0029  * saved LR as it was on entry to B, ie. the original return address at the
0030  * call site in A.
0031  *
0032  * Our job is to save the register state into a struct pt_regs (on the stack)
0033  * and then arrange for the ftrace function to be called.
0034  */
0035 .macro  ftrace_regs_entry allregs
0036     /* Create our stack frame + pt_regs */
0037     PPC_STLU    r1,-SWITCH_FRAME_SIZE(r1)
0038
0039     /* Save all gprs to pt_regs */
0040     SAVE_GPR(0, r1)
0041     SAVE_GPRS(3, 10, r1)
0042
0043 #ifdef CONFIG_PPC64
0044     /* Save the original return address in A's stack frame */
0045     std r0, LRSAVE+SWITCH_FRAME_SIZE(r1)
0046     /* Ok to continue? */
0047     lbz r3, PACA_FTRACE_ENABLED(r13)
0048     cmpdi   r3, 0
0049     beq ftrace_no_trace
0050 #endif
0051
0052     .if \allregs == 1
0053     SAVE_GPR(2, r1)
0054     SAVE_GPRS(11, 31, r1)
0055     .else
0056 #ifdef CONFIG_LIVEPATCH_64
0057     SAVE_GPR(14, r1)
0058 #endif
0059     .endif
0060
0061     /* Save previous stack pointer (r1) */
0062     addi    r8, r1, SWITCH_FRAME_SIZE
0063     PPC_STL r8, GPR1(r1)
0064
0065     .if \allregs == 1
0066     /* Load special regs for save below */
0067     mfmsr   r8
0068     mfctr   r9
0069     mfxer   r10
0070     mfcr    r11
0071     .else
0072     /* Clear MSR to flag as ftrace_caller versus frace_regs_caller */
0073     li  r8, 0
0074     .endif
0075
0076     /* Get the _mcount() call site out of LR */
0077     mflr    r7
0078     /* Save it as pt_regs->nip */
0079     PPC_STL r7, _NIP(r1)
0080     /* Save the read LR in pt_regs->link */
0081     PPC_STL r0, _LINK(r1)
0082
0083 #ifdef CONFIG_PPC64
0084     /* Save callee's TOC in the ABI compliant location */
0085     std r2, STK_GOT(r1)
0086     ld  r2,PACATOC(r13) /* get kernel TOC in r2 */
0087
0088     addis   r3,r2,function_trace_op@toc@ha
0089     addi    r3,r3,function_trace_op@toc@l
0090     ld  r5,0(r3)
0091 #else
0092     lis r3,function_trace_op@ha
0093     lwz r5,function_trace_op@l(r3)
0094 #endif
0095
0096 #ifdef CONFIG_LIVEPATCH_64
0097     mr  r14, r7     /* remember old NIP */
0098 #endif
0099
0100     /* Calculate ip from nip-4 into r3 for call below */
0101     subi    r3, r7, MCOUNT_INSN_SIZE
0102
0103     /* Put the original return address in r4 as parent_ip */
0104     mr  r4, r0
0105
0106     /* Save special regs */
0107     PPC_STL r8, _MSR(r1)
0108     .if \allregs == 1
0109     PPC_STL r9, _CTR(r1)
0110     PPC_STL r10, _XER(r1)
0111     PPC_STL r11, _CCR(r1)
0112     .endif
0113
0114     /* Load &pt_regs in r6 for call below */
0115     addi    r6, r1, STACK_FRAME_OVERHEAD
0116 .endm
0117
0118 .macro  ftrace_regs_exit allregs
0119     /* Load ctr with the possibly modified NIP */
0120     PPC_LL  r3, _NIP(r1)
0121     mtctr   r3
0122
0123 #ifdef CONFIG_LIVEPATCH_64
0124     cmpd    r14, r3     /* has NIP been altered? */
0125 #endif
0126
0127     /* Restore gprs */
0128     .if \allregs == 1
0129     REST_GPRS(2, 31, r1)
0130     .else
0131     REST_GPRS(3, 10, r1)
0132 #ifdef CONFIG_LIVEPATCH_64
0133     REST_GPR(14, r1)
0134 #endif
0135     .endif
0136
0137     /* Restore possibly modified LR */
0138     PPC_LL  r0, _LINK(r1)
0139     mtlr    r0
0140
0141 #ifdef CONFIG_PPC64
0142     /* Restore callee's TOC */
0143     ld  r2, STK_GOT(r1)
0144 #endif
0145
0146     /* Pop our stack frame */
0147     addi r1, r1, SWITCH_FRAME_SIZE
0148
0149 #ifdef CONFIG_LIVEPATCH_64
0150         /* Based on the cmpd above, if the NIP was altered handle livepatch */
0151     bne-    livepatch_handler
0152 #endif
0153     bctr            /* jump after _mcount site */
0154 .endm
0155
0156 _GLOBAL(ftrace_regs_caller)
0157     ftrace_regs_entry 1
0158     /* ftrace_call(r3, r4, r5, r6) */
0159 .globl ftrace_regs_call
0160 ftrace_regs_call:
0161     bl  ftrace_stub
0162     nop
0163     ftrace_regs_exit 1
0164
0165 _GLOBAL(ftrace_caller)
0166     ftrace_regs_entry 0
0167     /* ftrace_call(r3, r4, r5, r6) */
0168 .globl ftrace_call
0169 ftrace_call:
0170     bl  ftrace_stub
0171     nop
0172     ftrace_regs_exit 0
0173
0174 _GLOBAL(ftrace_stub)
0175     blr
0176
0177 #ifdef CONFIG_PPC64
0178 ftrace_no_trace:
0179     mflr    r3
0180     mtctr   r3
0181     REST_GPR(3, r1)
0182     addi    r1, r1, SWITCH_FRAME_SIZE
0183     mtlr    r0
0184     bctr
0185 #endif
0186
0187 #ifdef CONFIG_LIVEPATCH_64
0188     /*
0189      * This function runs in the mcount context, between two functions. As
0190      * such it can only clobber registers which are volatile and used in
0191      * function linkage.
0192      *
0193      * We get here when a function A, calls another function B, but B has
0194      * been live patched with a new function C.
0195      *
0196      * On entry:
0197      *  - we have no stack frame and can not allocate one
0198      *  - LR points back to the original caller (in A)
0199      *  - CTR holds the new NIP in C
0200      *  - r0, r11 & r12 are free
0201      */
0202 livepatch_handler:
0203     ld  r12, PACA_THREAD_INFO(r13)
0204
0205     /* Allocate 3 x 8 bytes */
0206     ld  r11, TI_livepatch_sp(r12)
0207     addi    r11, r11, 24
0208     std r11, TI_livepatch_sp(r12)
0209
0210     /* Save toc & real LR on livepatch stack */
0211     std r2,  -24(r11)
0212     mflr    r12
0213     std r12, -16(r11)
0214
0215     /* Store stack end marker */
0216     lis     r12, STACK_END_MAGIC@h
0217     ori     r12, r12, STACK_END_MAGIC@l
0218     std r12, -8(r11)
0219
0220     /* Put ctr in r12 for global entry and branch there */
0221     mfctr   r12
0222     bctrl
0223
0224     /*
0225      * Now we are returning from the patched function to the original
0226      * caller A. We are free to use r11, r12 and we can use r2 until we
0227      * restore it.
0228      */
0229
0230     ld  r12, PACA_THREAD_INFO(r13)
0231
0232     ld  r11, TI_livepatch_sp(r12)
0233
0234     /* Check stack marker hasn't been trashed */
0235     lis     r2,  STACK_END_MAGIC@h
0236     ori     r2,  r2, STACK_END_MAGIC@l
0237     ld  r12, -8(r11)
0238 1:  tdne    r12, r2
0239     EMIT_BUG_ENTRY 1b, __FILE__, __LINE__ - 1, 0
0240
0241     /* Restore LR & toc from livepatch stack */
0242     ld  r12, -16(r11)
0243     mtlr    r12
0244     ld  r2,  -24(r11)
0245
0246     /* Pop livepatch stack frame */
0247     ld  r12, PACA_THREAD_INFO(r13)
0248     subi    r11, r11, 24
0249     std r11, TI_livepatch_sp(r12)
0250
0251     /* Return to original caller of live patched function */
0252     blr
0253 #endif /* CONFIG_LIVEPATCH */