x86/kernel/ftrace_32.S

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 /*
0003  *  Copyright (C) 2017  Steven Rostedt, VMware Inc.
0004  */
0005
0006 #include <linux/linkage.h>
0007 #include <asm/page_types.h>
0008 #include <asm/segment.h>
0009 #include <asm/export.h>
0010 #include <asm/ftrace.h>
0011 #include <asm/nospec-branch.h>
0012 #include <asm/frame.h>
0013 #include <asm/asm-offsets.h>
0014
0015 #ifdef CONFIG_FRAME_POINTER
0016 # define MCOUNT_FRAME           1   /* using frame = true  */
0017 #else
0018 # define MCOUNT_FRAME           0   /* using frame = false */
0019 #endif
0020
0021 SYM_FUNC_START(__fentry__)
0022     RET
0023 SYM_FUNC_END(__fentry__)
0024 EXPORT_SYMBOL(__fentry__)
0025
0026 SYM_CODE_START(ftrace_caller)
0027
0028 #ifdef CONFIG_FRAME_POINTER
0029     /*
0030      * Frame pointers are of ip followed by bp.
0031      * Since fentry is an immediate jump, we are left with
0032      * parent-ip, function-ip. We need to add a frame with
0033      * parent-ip followed by ebp.
0034      */
0035     pushl   4(%esp)             /* parent ip */
0036     pushl   %ebp
0037     movl    %esp, %ebp
0038     pushl   2*4(%esp)           /* function ip */
0039
0040     /* For mcount, the function ip is directly above */
0041     pushl   %ebp
0042     movl    %esp, %ebp
0043 #endif
0044     pushl   %eax
0045     pushl   %ecx
0046     pushl   %edx
0047     pushl   $0              /* Pass NULL as regs pointer */
0048
0049 #ifdef CONFIG_FRAME_POINTER
0050     /* Load parent ebp into edx */
0051     movl    4*4(%esp), %edx
0052 #else
0053     /* There's no frame pointer, load the appropriate stack addr instead */
0054     lea 4*4(%esp), %edx
0055 #endif
0056
0057     movl    (MCOUNT_FRAME+4)*4(%esp), %eax  /* load the rip */
0058     /* Get the parent ip */
0059     movl    4(%edx), %edx           /* edx has ebp */
0060
0061     movl    function_trace_op, %ecx
0062     subl    $MCOUNT_INSN_SIZE, %eax
0063
0064 .globl ftrace_call
0065 ftrace_call:
0066     call    ftrace_stub
0067
0068     addl    $4, %esp            /* skip NULL pointer */
0069     popl    %edx
0070     popl    %ecx
0071     popl    %eax
0072 #ifdef CONFIG_FRAME_POINTER
0073     popl    %ebp
0074     addl    $4,%esp             /* skip function ip */
0075     popl    %ebp                /* this is the orig bp */
0076     addl    $4, %esp            /* skip parent ip */
0077 #endif
0078 .Lftrace_ret:
0079 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
0080 .globl ftrace_graph_call
0081 ftrace_graph_call:
0082     jmp ftrace_stub
0083 #endif
0084
0085 /* This is weak to keep gas from relaxing the jumps */
0086 SYM_INNER_LABEL_ALIGN(ftrace_stub, SYM_L_WEAK)
0087     RET
0088 SYM_CODE_END(ftrace_caller)
0089
0090 SYM_CODE_START(ftrace_regs_caller)
0091     /*
0092      * We're here from an mcount/fentry CALL, and the stack frame looks like:
0093      *
0094      *  <previous context>
0095      *  RET-IP
0096      *
0097      * The purpose of this function is to call out in an emulated INT3
0098      * environment with a stack frame like:
0099      *
0100      *  <previous context>
0101      *  gap / RET-IP
0102      *  gap
0103      *  gap
0104      *  gap
0105      *  pt_regs
0106      *
0107      * We do _NOT_ restore: ss, flags, cs, gs, fs, es, ds
0108      */
0109     subl    $3*4, %esp  # RET-IP + 3 gaps
0110     pushl   %ss     # ss
0111     pushl   %esp        # points at ss
0112     addl    $5*4, (%esp)    #   make it point at <previous context>
0113     pushfl          # flags
0114     pushl   $__KERNEL_CS    # cs
0115     pushl   7*4(%esp)   # ip <- RET-IP
0116     pushl   $0      # orig_eax
0117
0118     pushl   %gs
0119     pushl   %fs
0120     pushl   %es
0121     pushl   %ds
0122
0123     pushl   %eax
0124     pushl   %ebp
0125     pushl   %edi
0126     pushl   %esi
0127     pushl   %edx
0128     pushl   %ecx
0129     pushl   %ebx
0130
0131     ENCODE_FRAME_POINTER
0132
0133     movl    PT_EIP(%esp), %eax  # 1st argument: IP
0134     subl    $MCOUNT_INSN_SIZE, %eax
0135     movl    21*4(%esp), %edx    # 2nd argument: parent ip
0136     movl    function_trace_op, %ecx # 3rd argument: ftrace_pos
0137     pushl   %esp            # 4th argument: pt_regs
0138
0139 SYM_INNER_LABEL(ftrace_regs_call, SYM_L_GLOBAL)
0140     call    ftrace_stub
0141
0142     addl    $4, %esp        # skip 4th argument
0143
0144     /* place IP below the new SP */
0145     movl    PT_OLDESP(%esp), %eax
0146     movl    PT_EIP(%esp), %ecx
0147     movl    %ecx, -4(%eax)
0148
0149     /* place EAX below that */
0150     movl    PT_EAX(%esp), %ecx
0151     movl    %ecx, -8(%eax)
0152
0153     popl    %ebx
0154     popl    %ecx
0155     popl    %edx
0156     popl    %esi
0157     popl    %edi
0158     popl    %ebp
0159
0160     lea -8(%eax), %esp
0161     popl    %eax
0162
0163     jmp .Lftrace_ret
0164 SYM_CODE_END(ftrace_regs_caller)
0165
0166 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
0167 SYM_CODE_START(ftrace_graph_caller)
0168     pushl   %eax
0169     pushl   %ecx
0170     pushl   %edx
0171     movl    3*4(%esp), %eax
0172     /* Even with frame pointers, fentry doesn't have one here */
0173     lea 4*4(%esp), %edx
0174     movl    $0, %ecx
0175     subl    $MCOUNT_INSN_SIZE, %eax
0176     call    prepare_ftrace_return
0177     popl    %edx
0178     popl    %ecx
0179     popl    %eax
0180     RET
0181 SYM_CODE_END(ftrace_graph_caller)
0182
0183 .globl return_to_handler
0184 return_to_handler:
0185     pushl   %eax
0186     pushl   %edx
0187     movl    $0, %eax
0188     call    ftrace_return_to_handler
0189     movl    %eax, %ecx
0190     popl    %edx
0191     popl    %eax
0192     JMP_NOSPEC ecx
0193 #endif