Back to home page

OSCL-LXR
 
 

     

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 /*
0003  *    S390 low-level entry points.
0004  *
0005  *    Copyright IBM Corp. 1999, 2012
0006  *    Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com),
0007  *       Hartmut Penner (hp@de.ibm.com),
0008  *       Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com),
0009  */
0010 
0011 #include <linux/init.h>
0012 #include <linux/linkage.h>
0013 #include <asm/asm-extable.h>
0014 #include <asm/alternative-asm.h>
0015 #include <asm/processor.h>
0016 #include <asm/cache.h>
0017 #include <asm/dwarf.h>
0018 #include <asm/errno.h>
0019 #include <asm/ptrace.h>
0020 #include <asm/thread_info.h>
0021 #include <asm/asm-offsets.h>
0022 #include <asm/unistd.h>
0023 #include <asm/page.h>
0024 #include <asm/sigp.h>
0025 #include <asm/irq.h>
0026 #include <asm/vx-insn.h>
0027 #include <asm/setup.h>
0028 #include <asm/nmi.h>
0029 #include <asm/export.h>
0030 #include <asm/nospec-insn.h>
0031 
0032 STACK_SHIFT = PAGE_SHIFT + THREAD_SIZE_ORDER
0033 STACK_SIZE  = 1 << STACK_SHIFT
0034 STACK_INIT = STACK_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE
0035 
0036 _LPP_OFFSET = __LC_LPP
0037 
0038     .macro STBEAR address
0039     ALTERNATIVE "nop", ".insn s,0xb2010000,\address", 193
0040     .endm
0041 
0042     .macro LBEAR address
0043     ALTERNATIVE "nop", ".insn s,0xb2000000,\address", 193
0044     .endm
0045 
0046     .macro LPSWEY address,lpswe
0047     ALTERNATIVE "b \lpswe; nopr", ".insn siy,0xeb0000000071,\address,0", 193
0048     .endm
0049 
0050     .macro MBEAR reg
0051     ALTERNATIVE "brcl 0,0", __stringify(mvc __PT_LAST_BREAK(8,\reg),__LC_LAST_BREAK), 193
0052     .endm
0053 
0054     .macro  CHECK_STACK savearea
0055 #ifdef CONFIG_CHECK_STACK
0056     tml %r15,STACK_SIZE - CONFIG_STACK_GUARD
0057     lghi    %r14,\savearea
0058     jz  stack_overflow
0059 #endif
0060     .endm
0061 
0062     .macro  CHECK_VMAP_STACK savearea,oklabel
0063 #ifdef CONFIG_VMAP_STACK
0064     lgr %r14,%r15
0065     nill    %r14,0x10000 - STACK_SIZE
0066     oill    %r14,STACK_INIT
0067     clg %r14,__LC_KERNEL_STACK
0068     je  \oklabel
0069     clg %r14,__LC_ASYNC_STACK
0070     je  \oklabel
0071     clg %r14,__LC_MCCK_STACK
0072     je  \oklabel
0073     clg %r14,__LC_NODAT_STACK
0074     je  \oklabel
0075     clg %r14,__LC_RESTART_STACK
0076     je  \oklabel
0077     lghi    %r14,\savearea
0078     j   stack_overflow
0079 #else
0080     j   \oklabel
0081 #endif
0082     .endm
0083 
0084     /*
0085      * The TSTMSK macro generates a test-under-mask instruction by
0086      * calculating the memory offset for the specified mask value.
0087      * Mask value can be any constant.  The macro shifts the mask
0088      * value to calculate the memory offset for the test-under-mask
0089      * instruction.
0090      */
0091     .macro TSTMSK addr, mask, size=8, bytepos=0
0092         .if (\bytepos < \size) && (\mask >> 8)
0093             .if (\mask & 0xff)
0094                 .error "Mask exceeds byte boundary"
0095             .endif
0096             TSTMSK \addr, "(\mask >> 8)", \size, "(\bytepos + 1)"
0097             .exitm
0098         .endif
0099         .ifeq \mask
0100             .error "Mask must not be zero"
0101         .endif
0102         off = \size - \bytepos - 1
0103         tm  off+\addr, \mask
0104     .endm
0105 
0106     .macro BPOFF
0107     ALTERNATIVE "nop", ".insn rrf,0xb2e80000,0,0,12,0", 82
0108     .endm
0109 
0110     .macro BPON
0111     ALTERNATIVE "nop", ".insn rrf,0xb2e80000,0,0,13,0", 82
0112     .endm
0113 
0114     .macro BPENTER tif_ptr,tif_mask
0115     ALTERNATIVE "TSTMSK \tif_ptr,\tif_mask; jz .+8; .insn rrf,0xb2e80000,0,0,13,0", \
0116             "j .+12; nop; nop", 82
0117     .endm
0118 
0119     .macro BPEXIT tif_ptr,tif_mask
0120     TSTMSK  \tif_ptr,\tif_mask
0121     ALTERNATIVE "jz .+8;  .insn rrf,0xb2e80000,0,0,12,0", \
0122             "jnz .+8; .insn rrf,0xb2e80000,0,0,13,0", 82
0123     .endm
0124 
0125     /*
0126      * The CHKSTG macro jumps to the provided label in case the
0127      * machine check interruption code reports one of unrecoverable
0128      * storage errors:
0129      * - Storage error uncorrected
0130      * - Storage key error uncorrected
0131      * - Storage degradation with Failing-storage-address validity
0132      */
0133     .macro CHKSTG errlabel
0134     TSTMSK  __LC_MCCK_CODE,(MCCK_CODE_STG_ERROR|MCCK_CODE_STG_KEY_ERROR)
0135     jnz \errlabel
0136     TSTMSK  __LC_MCCK_CODE,MCCK_CODE_STG_DEGRAD
0137     jz  .Loklabel\@
0138     TSTMSK  __LC_MCCK_CODE,MCCK_CODE_STG_FAIL_ADDR
0139     jnz \errlabel
0140 .Loklabel\@:
0141     .endm
0142 
0143 #if IS_ENABLED(CONFIG_KVM)
0144     /*
0145      * The OUTSIDE macro jumps to the provided label in case the value
0146      * in the provided register is outside of the provided range. The
0147      * macro is useful for checking whether a PSW stored in a register
0148      * pair points inside or outside of a block of instructions.
0149      * @reg: register to check
0150      * @start: start of the range
0151      * @end: end of the range
0152      * @outside_label: jump here if @reg is outside of [@start..@end)
0153      */
0154     .macro OUTSIDE reg,start,end,outside_label
0155     lgr %r14,\reg
0156     larl    %r13,\start
0157     slgr    %r14,%r13
0158 #ifdef CONFIG_AS_IS_LLVM
0159     clgfrl  %r14,.Lrange_size\@
0160 #else
0161     clgfi   %r14,\end - \start
0162 #endif
0163     jhe \outside_label
0164 #ifdef CONFIG_AS_IS_LLVM
0165     .section .rodata, "a"
0166     .align 4
0167 .Lrange_size\@:
0168     .long   \end - \start
0169     .previous
0170 #endif
0171     .endm
0172 
0173     .macro SIEEXIT
0174     lg  %r9,__SF_SIE_CONTROL(%r15)  # get control block pointer
0175     ni  __SIE_PROG0C+3(%r9),0xfe    # no longer in SIE
0176     lctlg   %c1,%c1,__LC_KERNEL_ASCE    # load primary asce
0177     larl    %r9,sie_exit            # skip forward to sie_exit
0178     .endm
0179 #endif
0180 
0181     GEN_BR_THUNK %r14
0182 
0183     .section .kprobes.text, "ax"
0184 .Ldummy:
0185     /*
0186      * This nop exists only in order to avoid that __bpon starts at
0187      * the beginning of the kprobes text section. In that case we would
0188      * have several symbols at the same address. E.g. objdump would take
0189      * an arbitrary symbol name when disassembling this code.
0190      * With the added nop in between the __bpon symbol is unique
0191      * again.
0192      */
0193     nop 0
0194 
0195 ENTRY(__bpon)
0196     .globl __bpon
0197     BPON
0198     BR_EX   %r14
0199 ENDPROC(__bpon)
0200 
0201 /*
0202  * Scheduler resume function, called by switch_to
0203  *  gpr2 = (task_struct *) prev
0204  *  gpr3 = (task_struct *) next
0205  * Returns:
0206  *  gpr2 = prev
0207  */
0208 ENTRY(__switch_to)
0209     stmg    %r6,%r15,__SF_GPRS(%r15)    # store gprs of prev task
0210     lghi    %r4,__TASK_stack
0211     lghi    %r1,__TASK_thread
0212     llill   %r5,STACK_INIT
0213     stg %r15,__THREAD_ksp(%r1,%r2)  # store kernel stack of prev
0214     lg  %r15,0(%r4,%r3)         # start of kernel stack of next
0215     agr %r15,%r5            # end of kernel stack of next
0216     stg %r3,__LC_CURRENT        # store task struct of next
0217     stg %r15,__LC_KERNEL_STACK      # store end of kernel stack
0218     lg  %r15,__THREAD_ksp(%r1,%r3)  # load kernel stack of next
0219     aghi    %r3,__TASK_pid
0220     mvc __LC_CURRENT_PID(4,%r0),0(%r3)  # store pid of next
0221     lmg %r6,%r15,__SF_GPRS(%r15)    # load gprs of next task
0222     ALTERNATIVE "nop", "lpp _LPP_OFFSET", 40
0223     BR_EX   %r14
0224 ENDPROC(__switch_to)
0225 
0226 #if IS_ENABLED(CONFIG_KVM)
0227 /*
0228  * sie64a calling convention:
0229  * %r2 pointer to sie control block
0230  * %r3 guest register save area
0231  */
0232 ENTRY(sie64a)
0233     stmg    %r6,%r14,__SF_GPRS(%r15)    # save kernel registers
0234     lg  %r12,__LC_CURRENT
0235     stg %r2,__SF_SIE_CONTROL(%r15)  # save control block pointer
0236     stg %r3,__SF_SIE_SAVEAREA(%r15) # save guest register save area
0237     xc  __SF_SIE_REASON(8,%r15),__SF_SIE_REASON(%r15) # reason code = 0
0238     mvc __SF_SIE_FLAGS(8,%r15),__TI_flags(%r12) # copy thread flags
0239     lmg %r0,%r13,0(%r3)         # load guest gprs 0-13
0240     lg  %r14,__LC_GMAP          # get gmap pointer
0241     ltgr    %r14,%r14
0242     jz  .Lsie_gmap
0243     lctlg   %c1,%c1,__GMAP_ASCE(%r14)   # load primary asce
0244 .Lsie_gmap:
0245     lg  %r14,__SF_SIE_CONTROL(%r15) # get control block pointer
0246     oi  __SIE_PROG0C+3(%r14),1      # we are going into SIE now
0247     tm  __SIE_PROG20+3(%r14),3      # last exit...
0248     jnz .Lsie_skip
0249     TSTMSK  __LC_CPU_FLAGS,_CIF_FPU
0250     jo  .Lsie_skip          # exit if fp/vx regs changed
0251     BPEXIT  __SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST)
0252 .Lsie_entry:
0253     sie 0(%r14)
0254 # Let the next instruction be NOP to avoid triggering a machine check
0255 # and handling it in a guest as result of the instruction execution.
0256     nopr    7
0257 .Lsie_leave:
0258     BPOFF
0259     BPENTER __SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST)
0260 .Lsie_skip:
0261     ni  __SIE_PROG0C+3(%r14),0xfe   # no longer in SIE
0262     lctlg   %c1,%c1,__LC_KERNEL_ASCE    # load primary asce
0263 .Lsie_done:
0264 # some program checks are suppressing. C code (e.g. do_protection_exception)
0265 # will rewind the PSW by the ILC, which is often 4 bytes in case of SIE. There
0266 # are some corner cases (e.g. runtime instrumentation) where ILC is unpredictable.
0267 # Other instructions between sie64a and .Lsie_done should not cause program
0268 # interrupts. So lets use 3 nops as a landing pad for all possible rewinds.
0269 .Lrewind_pad6:
0270     nopr    7
0271 .Lrewind_pad4:
0272     nopr    7
0273 .Lrewind_pad2:
0274     nopr    7
0275     .globl sie_exit
0276 sie_exit:
0277     lg  %r14,__SF_SIE_SAVEAREA(%r15)    # load guest register save area
0278     stmg    %r0,%r13,0(%r14)        # save guest gprs 0-13
0279     xgr %r0,%r0             # clear guest registers to
0280     xgr %r1,%r1             # prevent speculative use
0281     xgr %r3,%r3
0282     xgr %r4,%r4
0283     xgr %r5,%r5
0284     lmg %r6,%r14,__SF_GPRS(%r15)    # restore kernel registers
0285     lg  %r2,__SF_SIE_REASON(%r15)   # return exit reason code
0286     BR_EX   %r14
0287 .Lsie_fault:
0288     lghi    %r14,-EFAULT
0289     stg %r14,__SF_SIE_REASON(%r15)  # set exit reason code
0290     j   sie_exit
0291 
0292     EX_TABLE(.Lrewind_pad6,.Lsie_fault)
0293     EX_TABLE(.Lrewind_pad4,.Lsie_fault)
0294     EX_TABLE(.Lrewind_pad2,.Lsie_fault)
0295     EX_TABLE(sie_exit,.Lsie_fault)
0296 ENDPROC(sie64a)
0297 EXPORT_SYMBOL(sie64a)
0298 EXPORT_SYMBOL(sie_exit)
0299 #endif
0300 
0301 /*
0302  * SVC interrupt handler routine. System calls are synchronous events and
0303  * are entered with interrupts disabled.
0304  */
0305 
0306 ENTRY(system_call)
0307     stpt    __LC_SYS_ENTER_TIMER
0308     stmg    %r8,%r15,__LC_SAVE_AREA_SYNC
0309     BPOFF
0310     lghi    %r14,0
0311 .Lsysc_per:
0312     STBEAR  __LC_LAST_BREAK
0313     lctlg   %c1,%c1,__LC_KERNEL_ASCE
0314     lg  %r12,__LC_CURRENT
0315     lg  %r15,__LC_KERNEL_STACK
0316     xc  __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
0317     stmg    %r0,%r7,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
0318     BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
0319     # clear user controlled register to prevent speculative use
0320     xgr %r0,%r0
0321     xgr %r1,%r1
0322     xgr %r4,%r4
0323     xgr %r5,%r5
0324     xgr %r6,%r6
0325     xgr %r7,%r7
0326     xgr %r8,%r8
0327     xgr %r9,%r9
0328     xgr %r10,%r10
0329     xgr %r11,%r11
0330     la  %r2,STACK_FRAME_OVERHEAD(%r15)  # pointer to pt_regs
0331     mvc __PT_R8(64,%r2),__LC_SAVE_AREA_SYNC
0332     MBEAR   %r2
0333     lgr %r3,%r14
0334     brasl   %r14,__do_syscall
0335     lctlg   %c1,%c1,__LC_USER_ASCE
0336     mvc __LC_RETURN_PSW(16),STACK_FRAME_OVERHEAD+__PT_PSW(%r15)
0337     BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP
0338     LBEAR   STACK_FRAME_OVERHEAD+__PT_LAST_BREAK(%r15)
0339     lmg %r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
0340     stpt    __LC_EXIT_TIMER
0341     LPSWEY  __LC_RETURN_PSW,__LC_RETURN_LPSWE
0342 ENDPROC(system_call)
0343 
0344 #
0345 # a new process exits the kernel with ret_from_fork
0346 #
0347 ENTRY(ret_from_fork)
0348     lgr %r3,%r11
0349     brasl   %r14,__ret_from_fork
0350     lctlg   %c1,%c1,__LC_USER_ASCE
0351     mvc __LC_RETURN_PSW(16),STACK_FRAME_OVERHEAD+__PT_PSW(%r15)
0352     BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP
0353     LBEAR   STACK_FRAME_OVERHEAD+__PT_LAST_BREAK(%r15)
0354     lmg %r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
0355     stpt    __LC_EXIT_TIMER
0356     LPSWEY  __LC_RETURN_PSW,__LC_RETURN_LPSWE
0357 ENDPROC(ret_from_fork)
0358 
0359 /*
0360  * Program check handler routine
0361  */
0362 
0363 ENTRY(pgm_check_handler)
0364     stpt    __LC_SYS_ENTER_TIMER
0365     BPOFF
0366     stmg    %r8,%r15,__LC_SAVE_AREA_SYNC
0367     lg  %r12,__LC_CURRENT
0368     lghi    %r10,0
0369     lmg %r8,%r9,__LC_PGM_OLD_PSW
0370     tmhh    %r8,0x0001      # coming from user space?
0371     jno .Lpgm_skip_asce
0372     lctlg   %c1,%c1,__LC_KERNEL_ASCE
0373     j   3f          # -> fault in user space
0374 .Lpgm_skip_asce:
0375 #if IS_ENABLED(CONFIG_KVM)
0376     # cleanup critical section for program checks in sie64a
0377     OUTSIDE %r9,.Lsie_gmap,.Lsie_done,1f
0378     SIEEXIT
0379     lghi    %r10,_PIF_GUEST_FAULT
0380 #endif
0381 1:  tmhh    %r8,0x4000      # PER bit set in old PSW ?
0382     jnz 2f          # -> enabled, can't be a double fault
0383     tm  __LC_PGM_ILC+3,0x80 # check for per exception
0384     jnz .Lpgm_svcper        # -> single stepped svc
0385 2:  CHECK_STACK __LC_SAVE_AREA_SYNC
0386     aghi    %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
0387     # CHECK_VMAP_STACK branches to stack_overflow or 4f
0388     CHECK_VMAP_STACK __LC_SAVE_AREA_SYNC,4f
0389 3:  BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
0390     lg  %r15,__LC_KERNEL_STACK
0391 4:  la  %r11,STACK_FRAME_OVERHEAD(%r15)
0392     stg %r10,__PT_FLAGS(%r11)
0393     xc  __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
0394     stmg    %r0,%r7,__PT_R0(%r11)
0395     mvc __PT_R8(64,%r11),__LC_SAVE_AREA_SYNC
0396     mvc __PT_LAST_BREAK(8,%r11),__LC_PGM_LAST_BREAK
0397     stmg    %r8,%r9,__PT_PSW(%r11)
0398 
0399     # clear user controlled registers to prevent speculative use
0400     xgr %r0,%r0
0401     xgr %r1,%r1
0402     xgr %r3,%r3
0403     xgr %r4,%r4
0404     xgr %r5,%r5
0405     xgr %r6,%r6
0406     xgr %r7,%r7
0407     lgr %r2,%r11
0408     brasl   %r14,__do_pgm_check
0409     tmhh    %r8,0x0001      # returning to user space?
0410     jno .Lpgm_exit_kernel
0411     lctlg   %c1,%c1,__LC_USER_ASCE
0412     BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP
0413     stpt    __LC_EXIT_TIMER
0414 .Lpgm_exit_kernel:
0415     mvc __LC_RETURN_PSW(16),STACK_FRAME_OVERHEAD+__PT_PSW(%r15)
0416     LBEAR   STACK_FRAME_OVERHEAD+__PT_LAST_BREAK(%r15)
0417     lmg %r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
0418     LPSWEY  __LC_RETURN_PSW,__LC_RETURN_LPSWE
0419 
0420 #
0421 # single stepped system call
0422 #
0423 .Lpgm_svcper:
0424     mvc __LC_RETURN_PSW(8),__LC_SVC_NEW_PSW
0425     larl    %r14,.Lsysc_per
0426     stg %r14,__LC_RETURN_PSW+8
0427     lghi    %r14,1
0428     LBEAR   __LC_PGM_LAST_BREAK
0429     LPSWEY  __LC_RETURN_PSW,__LC_RETURN_LPSWE # branch to .Lsysc_per
0430 ENDPROC(pgm_check_handler)
0431 
0432 /*
0433  * Interrupt handler macro used for external and IO interrupts.
0434  */
0435 .macro INT_HANDLER name,lc_old_psw,handler
0436 ENTRY(\name)
0437     stckf   __LC_INT_CLOCK
0438     stpt    __LC_SYS_ENTER_TIMER
0439     STBEAR  __LC_LAST_BREAK
0440     BPOFF
0441     stmg    %r8,%r15,__LC_SAVE_AREA_ASYNC
0442     lg  %r12,__LC_CURRENT
0443     lmg %r8,%r9,\lc_old_psw
0444     tmhh    %r8,0x0001          # interrupting from user ?
0445     jnz 1f
0446 #if IS_ENABLED(CONFIG_KVM)
0447     OUTSIDE %r9,.Lsie_gmap,.Lsie_done,0f
0448     BPENTER __SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST)
0449     SIEEXIT
0450 #endif
0451 0:  CHECK_STACK __LC_SAVE_AREA_ASYNC
0452     aghi    %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
0453     j   2f
0454 1:  BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
0455     lctlg   %c1,%c1,__LC_KERNEL_ASCE
0456     lg  %r15,__LC_KERNEL_STACK
0457 2:  xc  __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
0458     la  %r11,STACK_FRAME_OVERHEAD(%r15)
0459     stmg    %r0,%r7,__PT_R0(%r11)
0460     # clear user controlled registers to prevent speculative use
0461     xgr %r0,%r0
0462     xgr %r1,%r1
0463     xgr %r3,%r3
0464     xgr %r4,%r4
0465     xgr %r5,%r5
0466     xgr %r6,%r6
0467     xgr %r7,%r7
0468     xgr %r10,%r10
0469     xc  __PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
0470     mvc __PT_R8(64,%r11),__LC_SAVE_AREA_ASYNC
0471     MBEAR   %r11
0472     stmg    %r8,%r9,__PT_PSW(%r11)
0473     lgr %r2,%r11        # pass pointer to pt_regs
0474     brasl   %r14,\handler
0475     mvc __LC_RETURN_PSW(16),__PT_PSW(%r11)
0476     tmhh    %r8,0x0001      # returning to user ?
0477     jno 2f
0478     lctlg   %c1,%c1,__LC_USER_ASCE
0479     BPEXIT  __TI_flags(%r12),_TIF_ISOLATE_BP
0480     stpt    __LC_EXIT_TIMER
0481 2:  LBEAR   __PT_LAST_BREAK(%r11)
0482     lmg %r0,%r15,__PT_R0(%r11)
0483     LPSWEY  __LC_RETURN_PSW,__LC_RETURN_LPSWE
0484 ENDPROC(\name)
0485 .endm
0486 
0487 INT_HANDLER ext_int_handler,__LC_EXT_OLD_PSW,do_ext_irq
0488 INT_HANDLER io_int_handler,__LC_IO_OLD_PSW,do_io_irq
0489 
0490 /*
0491  * Load idle PSW.
0492  */
0493 ENTRY(psw_idle)
0494     stg %r14,(__SF_GPRS+8*8)(%r15)
0495     stg %r3,__SF_EMPTY(%r15)
0496     larl    %r1,psw_idle_exit
0497     stg %r1,__SF_EMPTY+8(%r15)
0498     larl    %r1,smp_cpu_mtid
0499     llgf    %r1,0(%r1)
0500     ltgr    %r1,%r1
0501     jz  .Lpsw_idle_stcctm
0502     .insn   rsy,0xeb0000000017,%r1,5,__MT_CYCLES_ENTER(%r2)
0503 .Lpsw_idle_stcctm:
0504     oi  __LC_CPU_FLAGS+7,_CIF_ENABLED_WAIT
0505     BPON
0506     stckf   __CLOCK_IDLE_ENTER(%r2)
0507     stpt    __TIMER_IDLE_ENTER(%r2)
0508     lpswe   __SF_EMPTY(%r15)
0509 .globl psw_idle_exit
0510 psw_idle_exit:
0511     BR_EX   %r14
0512 ENDPROC(psw_idle)
0513 
0514 /*
0515  * Machine check handler routines
0516  */
0517 ENTRY(mcck_int_handler)
0518     stckf   __LC_MCCK_CLOCK
0519     BPOFF
0520     la  %r1,4095        # validate r1
0521     spt __LC_CPU_TIMER_SAVE_AREA-4095(%r1)  # validate cpu timer
0522     LBEAR   __LC_LAST_BREAK_SAVE_AREA-4095(%r1)     # validate bear
0523     lmg %r0,%r15,__LC_GPREGS_SAVE_AREA-4095(%r1)# validate gprs
0524     lg  %r12,__LC_CURRENT
0525     lmg %r8,%r9,__LC_MCK_OLD_PSW
0526     TSTMSK  __LC_MCCK_CODE,MCCK_CODE_SYSTEM_DAMAGE
0527     jo  .Lmcck_panic        # yes -> rest of mcck code invalid
0528     TSTMSK  __LC_MCCK_CODE,MCCK_CODE_CR_VALID
0529     jno .Lmcck_panic        # control registers invalid -> panic
0530     la  %r14,4095
0531     lctlg   %c0,%c15,__LC_CREGS_SAVE_AREA-4095(%r14) # validate ctl regs
0532     ptlb
0533     lghi    %r14,__LC_CPU_TIMER_SAVE_AREA
0534     mvc __LC_MCCK_ENTER_TIMER(8),0(%r14)
0535     TSTMSK  __LC_MCCK_CODE,MCCK_CODE_CPU_TIMER_VALID
0536     jo  3f
0537     la  %r14,__LC_SYS_ENTER_TIMER
0538     clc 0(8,%r14),__LC_EXIT_TIMER
0539     jl  1f
0540     la  %r14,__LC_EXIT_TIMER
0541 1:  clc 0(8,%r14),__LC_LAST_UPDATE_TIMER
0542     jl  2f
0543     la  %r14,__LC_LAST_UPDATE_TIMER
0544 2:  spt 0(%r14)
0545     mvc __LC_MCCK_ENTER_TIMER(8),0(%r14)
0546 3:  TSTMSK  __LC_MCCK_CODE,MCCK_CODE_PSW_MWP_VALID
0547     jno .Lmcck_panic
0548     tmhh    %r8,0x0001      # interrupting from user ?
0549     jnz 6f
0550     TSTMSK  __LC_MCCK_CODE,MCCK_CODE_PSW_IA_VALID
0551     jno .Lmcck_panic
0552 #if IS_ENABLED(CONFIG_KVM)
0553     OUTSIDE %r9,.Lsie_gmap,.Lsie_done,6f
0554     OUTSIDE %r9,.Lsie_entry,.Lsie_leave,4f
0555     oi  __LC_CPU_FLAGS+7, _CIF_MCCK_GUEST
0556     j   5f
0557 4:  CHKSTG  .Lmcck_panic
0558 5:  larl    %r14,.Lstosm_tmp
0559     stosm   0(%r14),0x04        # turn dat on, keep irqs off
0560     BPENTER __SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST)
0561     SIEEXIT
0562     j   .Lmcck_stack
0563 #endif
0564 6:  CHKSTG  .Lmcck_panic
0565     larl    %r14,.Lstosm_tmp
0566     stosm   0(%r14),0x04        # turn dat on, keep irqs off
0567     tmhh    %r8,0x0001      # interrupting from user ?
0568     jz  .Lmcck_stack
0569     BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
0570 .Lmcck_stack:
0571     lg  %r15,__LC_MCCK_STACK
0572     la  %r11,STACK_FRAME_OVERHEAD(%r15)
0573     stctg   %c1,%c1,__PT_CR1(%r11)
0574     lctlg   %c1,%c1,__LC_KERNEL_ASCE
0575     xc  __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
0576     lghi    %r14,__LC_GPREGS_SAVE_AREA+64
0577     stmg    %r0,%r7,__PT_R0(%r11)
0578     # clear user controlled registers to prevent speculative use
0579     xgr %r0,%r0
0580     xgr %r1,%r1
0581     xgr %r3,%r3
0582     xgr %r4,%r4
0583     xgr %r5,%r5
0584     xgr %r6,%r6
0585     xgr %r7,%r7
0586     xgr %r10,%r10
0587     mvc __PT_R8(64,%r11),0(%r14)
0588     stmg    %r8,%r9,__PT_PSW(%r11)
0589     xc  __PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
0590     xc  __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
0591     lgr %r2,%r11        # pass pointer to pt_regs
0592     brasl   %r14,s390_do_machine_check
0593     cghi    %r2,0
0594     je  .Lmcck_return
0595     lg  %r1,__LC_KERNEL_STACK   # switch to kernel stack
0596     mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
0597     xc  __SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1)
0598     la  %r11,STACK_FRAME_OVERHEAD(%r1)
0599     lgr %r2,%r11
0600     lgr %r15,%r1
0601     brasl   %r14,s390_handle_mcck
0602 .Lmcck_return:
0603     lctlg   %c1,%c1,__PT_CR1(%r11)
0604     lmg %r0,%r10,__PT_R0(%r11)
0605     mvc __LC_RETURN_MCCK_PSW(16),__PT_PSW(%r11) # move return PSW
0606     tm  __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ?
0607     jno 0f
0608     BPEXIT  __TI_flags(%r12),_TIF_ISOLATE_BP
0609     stpt    __LC_EXIT_TIMER
0610 0:  ALTERNATIVE "nop", __stringify(lghi %r12,__LC_LAST_BREAK_SAVE_AREA),193
0611     LBEAR   0(%r12)
0612     lmg %r11,%r15,__PT_R11(%r11)
0613     LPSWEY  __LC_RETURN_MCCK_PSW,__LC_RETURN_MCCK_LPSWE
0614 
0615 .Lmcck_panic:
0616     /*
0617      * Iterate over all possible CPU addresses in the range 0..0xffff
0618      * and stop each CPU using signal processor. Use compare and swap
0619      * to allow just one CPU-stopper and prevent concurrent CPUs from
0620      * stopping each other while leaving the others running.
0621      */
0622     lhi %r5,0
0623     lhi %r6,1
0624     larl    %r7,.Lstop_lock
0625     cs  %r5,%r6,0(%r7)      # single CPU-stopper only
0626     jnz 4f
0627     larl    %r7,.Lthis_cpu
0628     stap    0(%r7)          # this CPU address
0629     lh  %r4,0(%r7)
0630     nilh    %r4,0
0631     lhi %r0,1
0632     sll %r0,16          # CPU counter
0633     lhi %r3,0           # next CPU address
0634 0:  cr  %r3,%r4
0635     je  2f
0636 1:  sigp    %r1,%r3,SIGP_STOP   # stop next CPU
0637     brc SIGP_CC_BUSY,1b
0638 2:  ahi %r3,1
0639     brct    %r0,0b
0640 3:  sigp    %r1,%r4,SIGP_STOP   # stop this CPU
0641     brc SIGP_CC_BUSY,3b
0642 4:  j   4b
0643 ENDPROC(mcck_int_handler)
0644 
0645 ENTRY(restart_int_handler)
0646     ALTERNATIVE "nop", "lpp _LPP_OFFSET", 40
0647     stg %r15,__LC_SAVE_AREA_RESTART
0648     TSTMSK  __LC_RESTART_FLAGS,RESTART_FLAG_CTLREGS,4
0649     jz  0f
0650     la  %r15,4095
0651     lctlg   %c0,%c15,__LC_CREGS_SAVE_AREA-4095(%r15)
0652 0:  larl    %r15,.Lstosm_tmp
0653     stosm   0(%r15),0x04            # turn dat on, keep irqs off
0654     lg  %r15,__LC_RESTART_STACK
0655     xc  STACK_FRAME_OVERHEAD(__PT_SIZE,%r15),STACK_FRAME_OVERHEAD(%r15)
0656     stmg    %r0,%r14,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
0657     mvc STACK_FRAME_OVERHEAD+__PT_R15(8,%r15),__LC_SAVE_AREA_RESTART
0658     mvc STACK_FRAME_OVERHEAD+__PT_PSW(16,%r15),__LC_RST_OLD_PSW
0659     xc  0(STACK_FRAME_OVERHEAD,%r15),0(%r15)
0660     lg  %r1,__LC_RESTART_FN     # load fn, parm & source cpu
0661     lg  %r2,__LC_RESTART_DATA
0662     lgf %r3,__LC_RESTART_SOURCE
0663     ltgr    %r3,%r3             # test source cpu address
0664     jm  1f              # negative -> skip source stop
0665 0:  sigp    %r4,%r3,SIGP_SENSE      # sigp sense to source cpu
0666     brc 10,0b               # wait for status stored
0667 1:  basr    %r14,%r1            # call function
0668     stap    __SF_EMPTY(%r15)        # store cpu address
0669     llgh    %r3,__SF_EMPTY(%r15)
0670 2:  sigp    %r4,%r3,SIGP_STOP       # sigp stop to current cpu
0671     brc 2,2b
0672 3:  j   3b
0673 ENDPROC(restart_int_handler)
0674 
0675     .section .kprobes.text, "ax"
0676 
0677 #if defined(CONFIG_CHECK_STACK) || defined(CONFIG_VMAP_STACK)
0678 /*
0679  * The synchronous or the asynchronous stack overflowed. We are dead.
0680  * No need to properly save the registers, we are going to panic anyway.
0681  * Setup a pt_regs so that show_trace can provide a good call trace.
0682  */
0683 ENTRY(stack_overflow)
0684     lg  %r15,__LC_NODAT_STACK   # change to panic stack
0685     la  %r11,STACK_FRAME_OVERHEAD(%r15)
0686     stmg    %r0,%r7,__PT_R0(%r11)
0687     stmg    %r8,%r9,__PT_PSW(%r11)
0688     mvc __PT_R8(64,%r11),0(%r14)
0689     stg %r10,__PT_ORIG_GPR2(%r11) # store last break to orig_gpr2
0690     xc  __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
0691     lgr %r2,%r11        # pass pointer to pt_regs
0692     jg  kernel_stack_overflow
0693 ENDPROC(stack_overflow)
0694 #endif
0695 
0696     .section .data, "aw"
0697         .align  4
0698 .Lstop_lock:    .long   0
0699 .Lthis_cpu: .short  0
0700 .Lstosm_tmp:    .byte   0
0701     .section .rodata, "a"
0702 #define SYSCALL(esame,emu)  .quad __s390x_ ## esame
0703     .globl  sys_call_table
0704 sys_call_table:
0705 #include "asm/syscall_table.h"
0706 #undef SYSCALL
0707 
0708 #ifdef CONFIG_COMPAT
0709 
0710 #define SYSCALL(esame,emu)  .quad __s390_ ## emu
0711     .globl  sys_call_table_emu
0712 sys_call_table_emu:
0713 #include "asm/syscall_table.h"
0714 #undef SYSCALL
0715 #endif
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!