0001
0002
0003
0004
0005
0006
0007
0008
0009
0010
0011
0012
0013
0014
0015 #include <linux/init.h>
0016
0017 #include <asm/assembler.h>
0018 #include <asm/memory.h>
0019 #include <asm/glue-df.h>
0020 #include <asm/glue-pf.h>
0021 #include <asm/vfpmacros.h>
0022 #include <asm/thread_notify.h>
0023 #include <asm/unwind.h>
0024 #include <asm/unistd.h>
0025 #include <asm/tls.h>
0026 #include <asm/system_info.h>
0027 #include <asm/uaccess-asm.h>
0028
0029 #include "entry-header.S"
0030 #include <asm/probes.h>
0031
0032
0033
0034
0035 .macro irq_handler, from_user:req
0036 mov r1, sp
0037 ldr_this_cpu r2, irq_stack_ptr, r2, r3
0038 .if \from_user == 0
0039 @
0040 @ If we took the interrupt while running in the kernel, we may already
0041 @ be using the IRQ stack, so revert to the original value in that case.
0042 @
0043 subs r3, r2, r1 @ SP above bottom of IRQ stack?
0044 rsbscs r3, r3, #THREAD_SIZE @ ... and below the top?
0045 #ifdef CONFIG_VMAP_STACK
0046 ldr_va r3, high_memory, cc @ End of the linear region
0047 cmpcc r3, r1 @ Stack pointer was below it?
0048 #endif
0049 bcc 0f @ If not, switch to the IRQ stack
0050 mov r0, r1
0051 bl generic_handle_arch_irq
0052 b 1f
0053 0:
0054 .endif
0055
0056 mov_l r0, generic_handle_arch_irq
0057 bl call_with_stack
0058 1:
0059 .endm
0060
0061 .macro pabt_helper
0062 @ PABORT handler takes pt_regs in r2, fault address in r4 and psr in r5
0063 #ifdef MULTI_PABORT
0064 ldr_va ip, processor, offset=PROCESSOR_PABT_FUNC
0065 bl_r ip
0066 #else
0067 bl CPU_PABORT_HANDLER
0068 #endif
0069 .endm
0070
0071 .macro dabt_helper
0072
0073 @
0074 @ Call the processor-specific abort handler:
0075 @
0076 @ r2 - pt_regs
0077 @ r4 - aborted context pc
0078 @ r5 - aborted context psr
0079 @
0080 @ The abort handler must return the aborted address in r0, and
0081 @ the fault status register in r1. r9 must be preserved.
0082 @
0083 #ifdef MULTI_DABORT
0084 ldr_va ip, processor, offset=PROCESSOR_DABT_FUNC
0085 bl_r ip
0086 #else
0087 bl CPU_DABORT_HANDLER
0088 #endif
0089 .endm
0090
0091 .section .entry.text,"ax",%progbits
0092
0093
0094
0095
0096 .macro inv_entry, reason
0097 sub sp, sp, #PT_REGS_SIZE
0098 ARM( stmib sp, {r1 - lr} )
0099 THUMB( stmia sp, {r0 - r12} )
0100 THUMB( str sp, [sp, #S_SP] )
0101 THUMB( str lr, [sp, #S_LR] )
0102 mov r1, #\reason
0103 .endm
0104
0105 __pabt_invalid:
0106 inv_entry BAD_PREFETCH
0107 b common_invalid
0108 ENDPROC(__pabt_invalid)
0109
0110 __dabt_invalid:
0111 inv_entry BAD_DATA
0112 b common_invalid
0113 ENDPROC(__dabt_invalid)
0114
0115 __irq_invalid:
0116 inv_entry BAD_IRQ
0117 b common_invalid
0118 ENDPROC(__irq_invalid)
0119
0120 __und_invalid:
0121 inv_entry BAD_UNDEFINSTR
0122
0123 @
0124 @ XXX fall through to common_invalid
0125 @
0126
0127 @
0128 @ common_invalid - generic code for failed exception (re-entrant version of handlers)
0129 @
0130 common_invalid:
0131 zero_fp
0132
0133 ldmia r0, {r4 - r6}
0134 add r0, sp, #S_PC @ here for interlock avoidance
0135 mov r7, #-1 @ "" "" "" ""
0136 str r4, [sp] @ save preserved r0
0137 stmia r0, {r5 - r7} @ lr_<exception>,
0138 @ cpsr_<exception>, "old_r0"
0139
0140 mov r0, sp
0141 b bad_mode
0142 ENDPROC(__und_invalid)
0143
0144
0145
0146
0147
0148 #if defined(CONFIG_AEABI) && (__LINUX_ARM_ARCH__ >= 5)
0149 #define SPFIX(code...) code
0150 #else
0151 #define SPFIX(code...)
0152 #endif
0153
0154 .macro svc_entry, stack_hole=0, trace=1, uaccess=1, overflow_check=1
0155 UNWIND(.fnstart )
0156 sub sp, sp, #(SVC_REGS_SIZE + \stack_hole)
0157 THUMB( add sp, r1 ) @ get SP in a GPR without
0158 THUMB( sub r1, sp, r1 ) @ using a temp register
0159
0160 .if \overflow_check
0161 UNWIND(.save {r0 - pc} )
0162 do_overflow_check (SVC_REGS_SIZE + \stack_hole)
0163 .endif
0164
0165 #ifdef CONFIG_THUMB2_KERNEL
0166 tst r1, #4 @ test stack pointer alignment
0167 sub r1, sp, r1 @ restore original R1
0168 sub sp, r1 @ restore original SP
0169 #else
0170 SPFIX( tst sp, #4 )
0171 #endif
0172 SPFIX( subne sp, sp, #4 )
0173
0174 ARM( stmib sp, {r1 - r12} )
0175 THUMB( stmia sp, {r0 - r12} ) @ No STMIB in Thumb-2
0176
0177 ldmia r0, {r3 - r5}
0178 add r7, sp, #S_SP @ here for interlock avoidance
0179 mov r6, #-1 @ "" "" "" ""
0180 add r2, sp, #(SVC_REGS_SIZE + \stack_hole)
0181 SPFIX( addne r2, r2, #4 )
0182 str r3, [sp] @ save the "real" r0 copied
0183 @ from the exception stack
0184
0185 mov r3, lr
0186
0187 @
0188 @ We are now ready to fill in the remaining blanks on the stack:
0189 @
0190 @ r2 - sp_svc
0191 @ r3 - lr_svc
0192 @ r4 - lr_<exception>, already fixed up for correct return/restart
0193 @ r5 - spsr_<exception>
0194 @ r6 - orig_r0 (see pt_regs definition in ptrace.h)
0195 @
0196 stmia r7, {r2 - r6}
0197
0198 get_thread_info tsk
0199 uaccess_entry tsk, r0, r1, r2, \uaccess
0200
0201 .if \trace
0202 #ifdef CONFIG_TRACE_IRQFLAGS
0203 bl trace_hardirqs_off
0204 #endif
0205 .endif
0206 .endm
0207
0208 .align 5
0209 __dabt_svc:
0210 svc_entry uaccess=0
0211 mov r2, sp
0212 dabt_helper
0213 THUMB( ldr r5, [sp, #S_PSR] ) @ potentially updated CPSR
0214 svc_exit r5 @ return from exception
0215 UNWIND(.fnend )
0216 ENDPROC(__dabt_svc)
0217
0218 .align 5
0219 __irq_svc:
0220 svc_entry
0221 irq_handler from_user=0
0222
0223 #ifdef CONFIG_PREEMPTION
0224 ldr r8, [tsk, #TI_PREEMPT] @ get preempt count
0225 ldr r0, [tsk, #TI_FLAGS] @ get flags
0226 teq r8, #0 @ if preempt count != 0
0227 movne r0, #0 @ force flags to 0
0228 tst r0, #_TIF_NEED_RESCHED
0229 blne svc_preempt
0230 #endif
0231
0232 svc_exit r5, irq = 1 @ return from exception
0233 UNWIND(.fnend )
0234 ENDPROC(__irq_svc)
0235
0236 .ltorg
0237
0238 #ifdef CONFIG_PREEMPTION
0239 svc_preempt:
0240 mov r8, lr
0241 1: bl preempt_schedule_irq @ irq en/disable is done inside
0242 ldr r0, [tsk, #TI_FLAGS] @ get new tasks TI_FLAGS
0243 tst r0, #_TIF_NEED_RESCHED
0244 reteq r8 @ go again
0245 b 1b
0246 #endif
0247
0248 __und_fault:
0249 @ Correct the PC such that it is pointing at the instruction
0250 @ which caused the fault. If the faulting instruction was ARM
0251 @ the PC will be pointing at the next instruction, and have to
0252 @ subtract 4. Otherwise, it is Thumb, and the PC will be
0253 @ pointing at the second half of the Thumb instruction. We
0254 @ have to subtract 2.
0255 ldr r2, [r0, #S_PC]
0256 sub r2, r2, r1
0257 str r2, [r0, #S_PC]
0258 b do_undefinstr
0259 ENDPROC(__und_fault)
0260
0261 .align 5
0262 __und_svc:
0263 #ifdef CONFIG_KPROBES
0264 @ If a kprobe is about to simulate a "stmdb sp..." instruction,
0265 @ it obviously needs free stack space which then will belong to
0266 @ the saved context.
0267 svc_entry MAX_STACK_SIZE
0268 #else
0269 svc_entry
0270 #endif
0271
0272 mov r1, #4 @ PC correction to apply
0273 THUMB( tst r5, #PSR_T_BIT ) @ exception taken in Thumb mode?
0274 THUMB( movne r1, #2 ) @ if so, fix up PC correction
0275 mov r0, sp @ struct pt_regs *regs
0276 bl __und_fault
0277
0278 __und_svc_finish:
0279 get_thread_info tsk
0280 ldr r5, [sp, #S_PSR] @ Get SVC cpsr
0281 svc_exit r5 @ return from exception
0282 UNWIND(.fnend )
0283 ENDPROC(__und_svc)
0284
0285 .align 5
0286 __pabt_svc:
0287 svc_entry
0288 mov r2, sp @ regs
0289 pabt_helper
0290 svc_exit r5 @ return from exception
0291 UNWIND(.fnend )
0292 ENDPROC(__pabt_svc)
0293
0294 .align 5
0295 __fiq_svc:
0296 svc_entry trace=0
0297 mov r0, sp @ struct pt_regs *regs
0298 bl handle_fiq_as_nmi
0299 svc_exit_via_fiq
0300 UNWIND(.fnend )
0301 ENDPROC(__fiq_svc)
0302
0303
0304
0305
0306
0307 @
0308 @ Taking a FIQ in abort mode is similar to taking a FIQ in SVC mode
0309 @ and reuses the same macros. However in abort mode we must also
0310 @ save/restore lr_abt and spsr_abt to make nested aborts safe.
0311 @
0312 .align 5
0313 __fiq_abt:
0314 svc_entry trace=0
0315
0316 ARM( msr cpsr_c, #ABT_MODE | PSR_I_BIT | PSR_F_BIT )
0317 THUMB( mov r0, #ABT_MODE | PSR_I_BIT | PSR_F_BIT )
0318 THUMB( msr cpsr_c, r0 )
0319 mov r1, lr @ Save lr_abt
0320 mrs r2, spsr @ Save spsr_abt, abort is now safe
0321 ARM( msr cpsr_c, #SVC_MODE | PSR_I_BIT | PSR_F_BIT )
0322 THUMB( mov r0, #SVC_MODE | PSR_I_BIT | PSR_F_BIT )
0323 THUMB( msr cpsr_c, r0 )
0324 stmfd sp!, {r1 - r2}
0325
0326 add r0, sp, #8 @ struct pt_regs *regs
0327 bl handle_fiq_as_nmi
0328
0329 ldmfd sp!, {r1 - r2}
0330 ARM( msr cpsr_c, #ABT_MODE | PSR_I_BIT | PSR_F_BIT )
0331 THUMB( mov r0, #ABT_MODE | PSR_I_BIT | PSR_F_BIT )
0332 THUMB( msr cpsr_c, r0 )
0333 mov lr, r1 @ Restore lr_abt, abort is unsafe
0334 msr spsr_cxsf, r2 @ Restore spsr_abt
0335 ARM( msr cpsr_c, #SVC_MODE | PSR_I_BIT | PSR_F_BIT )
0336 THUMB( mov r0, #SVC_MODE | PSR_I_BIT | PSR_F_BIT )
0337 THUMB( msr cpsr_c, r0 )
0338
0339 svc_exit_via_fiq
0340 UNWIND(.fnend )
0341 ENDPROC(__fiq_abt)
0342
0343
0344
0345
0346
0347
0348
0349 #if defined(CONFIG_AEABI) && (__LINUX_ARM_ARCH__ >= 5) && (PT_REGS_SIZE & 7)
0350 #error "sizeof(struct pt_regs) must be a multiple of 8"
0351 #endif
0352
0353 .macro usr_entry, trace=1, uaccess=1
0354 UNWIND(.fnstart )
0355 UNWIND(.cantunwind ) @ don't unwind the user space
0356 sub sp, sp, #PT_REGS_SIZE
0357 ARM( stmib sp, {r1 - r12} )
0358 THUMB( stmia sp, {r0 - r12} )
0359
0360 ATRAP( mrc p15, 0, r7, c1, c0, 0)
0361 ATRAP( ldr_va r8, cr_alignment)
0362
0363 ldmia r0, {r3 - r5}
0364 add r0, sp, #S_PC @ here for interlock avoidance
0365 mov r6, #-1 @ "" "" "" ""
0366
0367 str r3, [sp] @ save the "real" r0 copied
0368 @ from the exception stack
0369
0370 @
0371 @ We are now ready to fill in the remaining blanks on the stack:
0372 @
0373 @ r4 - lr_<exception>, already fixed up for correct return/restart
0374 @ r5 - spsr_<exception>
0375 @ r6 - orig_r0 (see pt_regs definition in ptrace.h)
0376 @
0377 @ Also, separately save sp_usr and lr_usr
0378 @
0379 stmia r0, {r4 - r6}
0380 ARM( stmdb r0, {sp, lr}^ )
0381 THUMB( store_user_sp_lr r0, r1, S_SP - S_PC )
0382
0383 .if \uaccess
0384 uaccess_disable ip
0385 .endif
0386
0387 @ Enable the alignment trap while in kernel mode
0388 ATRAP( teq r8, r7)
0389 ATRAP( mcrne p15, 0, r8, c1, c0, 0)
0390
0391 reload_current r7, r8
0392
0393 @
0394 @ Clear FP to mark the first stack frame
0395 @
0396 zero_fp
0397
0398 .if \trace
0399 #ifdef CONFIG_TRACE_IRQFLAGS
0400 bl trace_hardirqs_off
0401 #endif
0402 ct_user_exit save = 0
0403 .endif
0404 .endm
0405
0406 .macro kuser_cmpxchg_check
0407 #if !defined(CONFIG_CPU_32v6K) && defined(CONFIG_KUSER_HELPERS)
0408 #ifndef CONFIG_MMU
0409 #warning "NPTL on non MMU needs fixing"
0410 #else
0411 @ Make sure our user space atomic helper is restarted
0412 @ if it was interrupted in a critical region. Here we
0413 @ perform a quick test inline since it should be false
0414 @ 99.9999% of the time. The rest is done out of line.
0415 ldr r0, =TASK_SIZE
0416 cmp r4, r0
0417 blhs kuser_cmpxchg64_fixup
0418 #endif
0419 #endif
0420 .endm
0421
0422 .align 5
0423 __dabt_usr:
0424 usr_entry uaccess=0
0425 kuser_cmpxchg_check
0426 mov r2, sp
0427 dabt_helper
0428 b ret_from_exception
0429 UNWIND(.fnend )
0430 ENDPROC(__dabt_usr)
0431
0432 .align 5
0433 __irq_usr:
0434 usr_entry
0435 kuser_cmpxchg_check
0436 irq_handler from_user=1
0437 get_thread_info tsk
0438 mov why, #0
0439 b ret_to_user_from_irq
0440 UNWIND(.fnend )
0441 ENDPROC(__irq_usr)
0442
0443 .ltorg
0444
0445 .align 5
0446 __und_usr:
0447 usr_entry uaccess=0
0448
0449 mov r2, r4
0450 mov r3, r5
0451
0452 @ r2 = regs->ARM_pc, which is either 2 or 4 bytes ahead of the
0453 @ faulting instruction depending on Thumb mode.
0454 @ r3 = regs->ARM_cpsr
0455 @
0456 @ The emulation code returns using r9 if it has emulated the
0457 @ instruction, or the more conventional lr if we are to treat
0458 @ this as a real undefined instruction
0459 @
0460 badr r9, ret_from_exception
0461
0462 @ IRQs must be enabled before attempting to read the instruction from
0463 @ user space since that could cause a page/translation fault if the
0464 @ page table was modified by another CPU.
0465 enable_irq
0466
0467 tst r3, #PSR_T_BIT @ Thumb mode?
0468 bne __und_usr_thumb
0469 sub r4, r2, #4 @ ARM instr at LR - 4
0470 1: ldrt r0, [r4]
0471 ARM_BE8(rev r0, r0) @ little endian instruction
0472
0473 uaccess_disable ip
0474
0475 @ r0 = 32-bit ARM instruction which caused the exception
0476 @ r2 = PC value for the following instruction (:= regs->ARM_pc)
0477 @ r4 = PC value for the faulting instruction
0478 @ lr = 32-bit undefined instruction function
0479 badr lr, __und_usr_fault_32
0480 b call_fpe
0481
0482 __und_usr_thumb:
0483 @ Thumb instruction
0484 sub r4, r2, #2 @ First half of thumb instr at LR - 2
0485 #if CONFIG_ARM_THUMB && __LINUX_ARM_ARCH__ >= 6 && CONFIG_CPU_V7
0486
0487
0488
0489
0490
0491
0492 #if __LINUX_ARM_ARCH__ < 7
0493
0494 ldr_va r5, cpu_architecture
0495 cmp r5, #CPU_ARCH_ARMv7
0496 blo __und_usr_fault_16 @ 16bit undefined instruction
0497
0498
0499
0500
0501
0502 .arch armv6t2
0503 #endif
0504 2: ldrht r5, [r4]
0505 ARM_BE8(rev16 r5, r5) @ little endian instruction
0506 cmp r5, #0xe800 @ 32bit instruction if xx != 0
0507 blo __und_usr_fault_16_pan @ 16bit undefined instruction
0508 3: ldrht r0, [r2]
0509 ARM_BE8(rev16 r0, r0) @ little endian instruction
0510 uaccess_disable ip
0511 add r2, r2, #2 @ r2 is PC + 2, make it PC + 4
0512 str r2, [sp, #S_PC] @ it's a 2x16bit instr, update
0513 orr r0, r0, r5, lsl #16
0514 badr lr, __und_usr_fault_32
0515 @ r0 = the two 16-bit Thumb instructions which caused the exception
0516 @ r2 = PC value for the following Thumb instruction (:= regs->ARM_pc)
0517 @ r4 = PC value for the first 16-bit Thumb instruction
0518 @ lr = 32bit undefined instruction function
0519
0520 #if __LINUX_ARM_ARCH__ < 7
0521
0522 #ifdef CONFIG_CPU_32v6K
0523 .arch armv6k
0524 #else
0525 .arch armv6
0526 #endif
0527 #endif
0528 #else
0529 b __und_usr_fault_16
0530 #endif
0531 UNWIND(.fnend)
0532 ENDPROC(__und_usr)
0533
0534
0535
0536
0537 .pushsection .text.fixup, "ax"
0538 .align 2
0539 4: str r4, [sp, #S_PC] @ retry current instruction
0540 ret r9
0541 .popsection
0542 .pushsection __ex_table,"a"
0543 .long 1b, 4b
0544 #if CONFIG_ARM_THUMB && __LINUX_ARM_ARCH__ >= 6 && CONFIG_CPU_V7
0545 .long 2b, 4b
0546 .long 3b, 4b
0547 #endif
0548 .popsection
0549
0550
0551
0552
0553
0554
0555
0556
0557
0558
0559
0560
0561
0562
0563
0564
0565
0566
0567
0568
0569
0570
0571
0572
0573
0574
0575 @
0576 @ Fall-through from Thumb-2 __und_usr
0577 @
0578 #ifdef CONFIG_NEON
0579 get_thread_info r10 @ get current thread
0580 adr r6, .LCneon_thumb_opcodes
0581 b 2f
0582 #endif
0583 call_fpe:
0584 get_thread_info r10 @ get current thread
0585 #ifdef CONFIG_NEON
0586 adr r6, .LCneon_arm_opcodes
0587 2: ldr r5, [r6], #4 @ mask value
0588 ldr r7, [r6], #4 @ opcode bits matching in mask
0589 cmp r5, #0 @ end mask?
0590 beq 1f
0591 and r8, r0, r5
0592 cmp r8, r7 @ NEON instruction?
0593 bne 2b
0594 mov r7, #1
0595 strb r7, [r10, #TI_USED_CP + 10] @ mark CP#10 as used
0596 strb r7, [r10, #TI_USED_CP + 11] @ mark CP#11 as used
0597 b do_vfp @ let VFP handler handle this
0598 1:
0599 #endif
0600 tst r0, #0x08000000 @ only CDP/CPRT/LDC/STC have bit 27
0601 tstne r0, #0x04000000 @ bit 26 set on both ARM and Thumb-2
0602 reteq lr
0603 and r8, r0, #0x00000f00 @ mask out CP number
0604 mov r7, #1
0605 add r6, r10, r8, lsr #8 @ add used_cp[] array offset first
0606 strb r7, [r6, #TI_USED_CP] @ set appropriate used_cp[]
0607 #ifdef CONFIG_IWMMXT
0608 @ Test if we need to give access to iWMMXt coprocessors
0609 ldr r5, [r10, #TI_FLAGS]
0610 rsbs r7, r8, #(1 << 8) @ CP 0 or 1 only
0611 movscs r7, r5, lsr #(TIF_USING_IWMMXT + 1)
0612 bcs iwmmxt_task_enable
0613 #endif
0614 ARM( add pc, pc, r8, lsr #6 )
0615 THUMB( lsr r8, r8, #6 )
0616 THUMB( add pc, r8 )
0617 nop
0618
0619 ret.w lr @ CP#0
0620 W(b) do_fpe @ CP#1 (FPE)
0621 W(b) do_fpe @ CP#2 (FPE)
0622 ret.w lr @ CP#3
0623 ret.w lr @ CP#4
0624 ret.w lr @ CP#5
0625 ret.w lr @ CP#6
0626 ret.w lr @ CP#7
0627 ret.w lr @ CP#8
0628 ret.w lr @ CP#9
0629 #ifdef CONFIG_VFP
0630 W(b) do_vfp @ CP#10 (VFP)
0631 W(b) do_vfp @ CP#11 (VFP)
0632 #else
0633 ret.w lr @ CP#10 (VFP)
0634 ret.w lr @ CP#11 (VFP)
0635 #endif
0636 ret.w lr @ CP#12
0637 ret.w lr @ CP#13
0638 ret.w lr @ CP#14 (Debug)
0639 ret.w lr @ CP#15 (Control)
0640
0641 #ifdef CONFIG_NEON
0642 .align 6
0643
0644 .LCneon_arm_opcodes:
0645 .word 0xfe000000 @ mask
0646 .word 0xf2000000 @ opcode
0647
0648 .word 0xff100000 @ mask
0649 .word 0xf4000000 @ opcode
0650
0651 .word 0x00000000 @ mask
0652 .word 0x00000000 @ opcode
0653
0654 .LCneon_thumb_opcodes:
0655 .word 0xef000000 @ mask
0656 .word 0xef000000 @ opcode
0657
0658 .word 0xff100000 @ mask
0659 .word 0xf9000000 @ opcode
0660
0661 .word 0x00000000 @ mask
0662 .word 0x00000000 @ opcode
0663 #endif
0664
0665 do_fpe:
0666 add r10, r10, #TI_FPSTATE @ r10 = workspace
0667 ldr_va pc, fp_enter, tmp=r4 @ Call FP module USR entry point
0668
0669
0670
0671
0672
0673
0674
0675
0676
0677
0678 .pushsection .data
0679 .align 2
0680 ENTRY(fp_enter)
0681 .word no_fp
0682 .popsection
0683
0684 ENTRY(no_fp)
0685 ret lr
0686 ENDPROC(no_fp)
0687
0688 __und_usr_fault_32:
0689 mov r1, #4
0690 b 1f
0691 __und_usr_fault_16_pan:
0692 uaccess_disable ip
0693 __und_usr_fault_16:
0694 mov r1, #2
0695 1: mov r0, sp
0696 badr lr, ret_from_exception
0697 b __und_fault
0698 ENDPROC(__und_usr_fault_32)
0699 ENDPROC(__und_usr_fault_16)
0700
0701 .align 5
0702 __pabt_usr:
0703 usr_entry
0704 mov r2, sp @ regs
0705 pabt_helper
0706 UNWIND(.fnend )
0707
0708
0709
0710
0711 ENTRY(ret_from_exception)
0712 UNWIND(.fnstart )
0713 UNWIND(.cantunwind )
0714 get_thread_info tsk
0715 mov why, #0
0716 b ret_to_user
0717 UNWIND(.fnend )
0718 ENDPROC(__pabt_usr)
0719 ENDPROC(ret_from_exception)
0720
0721 .align 5
0722 __fiq_usr:
0723 usr_entry trace=0
0724 kuser_cmpxchg_check
0725 mov r0, sp @ struct pt_regs *regs
0726 bl handle_fiq_as_nmi
0727 get_thread_info tsk
0728 restore_user_regs fast = 0, offset = 0
0729 UNWIND(.fnend )
0730 ENDPROC(__fiq_usr)
0731
0732
0733
0734
0735
0736
0737 ENTRY(__switch_to)
0738 UNWIND(.fnstart )
0739 UNWIND(.cantunwind )
0740 add ip, r1, #TI_CPU_SAVE
0741 ARM( stmia ip!, {r4 - sl, fp, sp, lr} ) @ Store most regs on stack
0742 THUMB( stmia ip!, {r4 - sl, fp} ) @ Store most regs on stack
0743 THUMB( str sp, [ip], #4 )
0744 THUMB( str lr, [ip], #4 )
0745 ldr r4, [r2, #TI_TP_VALUE]
0746 ldr r5, [r2, #TI_TP_VALUE + 4]
0747 #ifdef CONFIG_CPU_USE_DOMAINS
0748 mrc p15, 0, r6, c3, c0, 0 @ Get domain register
0749 str r6, [r1, #TI_CPU_DOMAIN] @ Save old domain register
0750 ldr r6, [r2, #TI_CPU_DOMAIN]
0751 #endif
0752 switch_tls r1, r4, r5, r3, r7
0753 #if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_SMP) && \
0754 !defined(CONFIG_STACKPROTECTOR_PER_TASK)
0755 ldr r8, =__stack_chk_guard
0756 .if (TSK_STACK_CANARY > IMM12_MASK)
0757 add r9, r2, #TSK_STACK_CANARY & ~IMM12_MASK
0758 ldr r9, [r9, #TSK_STACK_CANARY & IMM12_MASK]
0759 .else
0760 ldr r9, [r2, #TSK_STACK_CANARY & IMM12_MASK]
0761 .endif
0762 #endif
0763 mov r7, r2 @ Preserve 'next'
0764 #ifdef CONFIG_CPU_USE_DOMAINS
0765 mcr p15, 0, r6, c3, c0, 0 @ Set domain register
0766 #endif
0767 mov r5, r0
0768 add r4, r2, #TI_CPU_SAVE
0769 ldr r0, =thread_notify_head
0770 mov r1, #THREAD_NOTIFY_SWITCH
0771 bl atomic_notifier_call_chain
0772 #if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_SMP) && \
0773 !defined(CONFIG_STACKPROTECTOR_PER_TASK)
0774 str r9, [r8]
0775 #endif
0776 mov r0, r5
0777 #if !defined(CONFIG_THUMB2_KERNEL) && !defined(CONFIG_VMAP_STACK)
0778 set_current r7, r8
0779 ldmia r4, {r4 - sl, fp, sp, pc} @ Load all regs saved previously
0780 #else
0781 mov r1, r7
0782 ldmia r4, {r4 - sl, fp, ip, lr} @ Load all regs saved previously
0783 #ifdef CONFIG_VMAP_STACK
0784 @
0785 @ Do a dummy read from the new stack while running from the old one so
0786 @ that we can rely on do_translation_fault() to fix up any stale PMD
0787 @ entries covering the vmalloc region.
0788 @
0789 ldr r2, [ip]
0790 #endif
0791
0792 @ When CONFIG_THREAD_INFO_IN_TASK=n, the update of SP itself is what
0793 @ effectuates the task switch, as that is what causes the observable
0794 @ values of current and current_thread_info to change. When
0795 @ CONFIG_THREAD_INFO_IN_TASK=y, setting current (and therefore
0796 @ current_thread_info) is done explicitly, and the update of SP just
0797 @ switches us to another stack, with few other side effects. In order
0798 @ to prevent this distinction from causing any inconsistencies, let's
0799 @ keep the 'set_current' call as close as we can to the update of SP.
0800 set_current r1, r2
0801 mov sp, ip
0802 ret lr
0803 #endif
0804 UNWIND(.fnend )
0805 ENDPROC(__switch_to)
0806
0807 #ifdef CONFIG_VMAP_STACK
0808 .text
0809 .align 2
0810 __bad_stack:
0811 @
0812 @ We've just detected an overflow. We need to load the address of this
0813 @ CPU's overflow stack into the stack pointer register. We have only one
0814 @ scratch register so let's use a sequence of ADDs including one
0815 @ involving the PC, and decorate them with PC-relative group
0816 @ relocations. As these are ARM only, switch to ARM mode first.
0817 @
0818 @ We enter here with IP clobbered and its value stashed on the mode
0819 @ stack.
0820 @
0821 THUMB( bx pc )
0822 THUMB( nop )
0823 THUMB( .arm )
0824 ldr_this_cpu_armv6 ip, overflow_stack_ptr
0825
0826 str sp, [ip, #-4]! @ Preserve original SP value
0827 mov sp, ip @ Switch to overflow stack
0828 pop {ip} @ Original SP in IP
0829
0830 #if defined(CONFIG_UNWINDER_FRAME_POINTER) && defined(CONFIG_CC_IS_GCC)
0831 mov ip, ip @ mov expected by unwinder
0832 push {fp, ip, lr, pc} @ GCC flavor frame record
0833 #else
0834 str ip, [sp, #-8]! @ store original SP
0835 push {fpreg, lr} @ Clang flavor frame record
0836 #endif
0837 UNWIND( ldr ip, [r0, #4] ) @ load exception LR
0838 UNWIND( str ip, [sp, #12] ) @ store in the frame record
0839 ldr ip, [r0, #12] @ reload IP
0840
0841 @ Store the original GPRs to the new stack.
0842 svc_entry uaccess=0, overflow_check=0
0843
0844 UNWIND( .save {sp, pc} )
0845 UNWIND( .save {fpreg, lr} )
0846 UNWIND( .setfp fpreg, sp )
0847
0848 ldr fpreg, [sp, #S_SP] @ Add our frame record
0849 @ to the linked list
0850 #if defined(CONFIG_UNWINDER_FRAME_POINTER) && defined(CONFIG_CC_IS_GCC)
0851 ldr r1, [fp, #4] @ reload SP at entry
0852 add fp, fp, #12
0853 #else
0854 ldr r1, [fpreg, #8]
0855 #endif
0856 str r1, [sp, #S_SP] @ store in pt_regs
0857
0858 @ Stash the regs for handle_bad_stack
0859 mov r0, sp
0860
0861 @ Time to die
0862 bl handle_bad_stack
0863 nop
0864 UNWIND( .fnend )
0865 ENDPROC(__bad_stack)
0866 #endif
0867
0868 __INIT
0869
0870
0871
0872
0873
0874
0875
0876
0877
0878
0879
0880 THUMB( .arm )
0881
0882 .macro usr_ret, reg
0883 #ifdef CONFIG_ARM_THUMB
0884 bx \reg
0885 #else
0886 ret \reg
0887 #endif
0888 .endm
0889
0890 .macro kuser_pad, sym, size
0891 .if (. - \sym) & 3
0892 .rept 4 - (. - \sym) & 3
0893 .byte 0
0894 .endr
0895 .endif
0896 .rept (\size - (. - \sym)) / 4
0897 .word 0xe7fddef1
0898 .endr
0899 .endm
0900
0901 #ifdef CONFIG_KUSER_HELPERS
0902 .align 5
0903 .globl __kuser_helper_start
0904 __kuser_helper_start:
0905
0906
0907
0908
0909
0910
0911 __kuser_cmpxchg64: @ 0xffff0f60
0912
0913 #if defined(CONFIG_CPU_32v6K)
0914
0915 stmfd sp!, {r4, r5, r6, r7}
0916 ldrd r4, r5, [r0] @ load old val
0917 ldrd r6, r7, [r1] @ load new val
0918 smp_dmb arm
0919 1: ldrexd r0, r1, [r2] @ load current val
0920 eors r3, r0, r4 @ compare with oldval (1)
0921 eorseq r3, r1, r5 @ compare with oldval (2)
0922 strexdeq r3, r6, r7, [r2] @ store newval if eq
0923 teqeq r3, #1 @ success?
0924 beq 1b @ if no then retry
0925 smp_dmb arm
0926 rsbs r0, r3, #0 @ set returned val and C flag
0927 ldmfd sp!, {r4, r5, r6, r7}
0928 usr_ret lr
0929
0930 #elif !defined(CONFIG_SMP)
0931
0932 #ifdef CONFIG_MMU
0933
0934
0935
0936
0937
0938
0939
0940 stmfd sp!, {r4, r5, r6, lr}
0941 ldmia r0, {r4, r5} @ load old val
0942 ldmia r1, {r6, lr} @ load new val
0943 1: ldmia r2, {r0, r1} @ load current val
0944 eors r3, r0, r4 @ compare with oldval (1)
0945 eorseq r3, r1, r5 @ compare with oldval (2)
0946 2: stmiaeq r2, {r6, lr} @ store newval if eq
0947 rsbs r0, r3, #0 @ set return val and C flag
0948 ldmfd sp!, {r4, r5, r6, pc}
0949
0950 .text
0951 kuser_cmpxchg64_fixup:
0952 @ Called from kuser_cmpxchg_fixup.
0953 @ r4 = address of interrupted insn (must be preserved).
0954 @ sp = saved regs. r7 and r8 are clobbered.
0955 @ 1b = first critical insn, 2b = last critical insn.
0956 @ If r4 >= 1b and r4 <= 2b then saved pc_usr is set to 1b.
0957 mov r7, #0xffff0fff
0958 sub r7, r7, #(0xffff0fff - (0xffff0f60 + (1b - __kuser_cmpxchg64)))
0959 subs r8, r4, r7
0960 rsbscs r8, r8, #(2b - 1b)
0961 strcs r7, [sp, #S_PC]
0962 #if __LINUX_ARM_ARCH__ < 6
0963 bcc kuser_cmpxchg32_fixup
0964 #endif
0965 ret lr
0966 .previous
0967
0968 #else
0969 #warning "NPTL on non MMU needs fixing"
0970 mov r0, #-1
0971 adds r0, r0, #0
0972 usr_ret lr
0973 #endif
0974
0975 #else
0976 #error "incoherent kernel configuration"
0977 #endif
0978
0979 kuser_pad __kuser_cmpxchg64, 64
0980
0981 __kuser_memory_barrier: @ 0xffff0fa0
0982 smp_dmb arm
0983 usr_ret lr
0984
0985 kuser_pad __kuser_memory_barrier, 32
0986
0987 __kuser_cmpxchg: @ 0xffff0fc0
0988
0989 #if __LINUX_ARM_ARCH__ < 6
0990
0991 #ifdef CONFIG_MMU
0992
0993
0994
0995
0996
0997
0998
0999
1000
1001
1002 1: ldr r3, [r2] @ load current val
1003 subs r3, r3, r0 @ compare with oldval
1004 2: streq r1, [r2] @ store newval if eq
1005 rsbs r0, r3, #0 @ set return val and C flag
1006 usr_ret lr
1007
1008 .text
1009 kuser_cmpxchg32_fixup:
1010 @ Called from kuser_cmpxchg_check macro.
1011 @ r4 = address of interrupted insn (must be preserved).
1012 @ sp = saved regs. r7 and r8 are clobbered.
1013 @ 1b = first critical insn, 2b = last critical insn.
1014 @ If r4 >= 1b and r4 <= 2b then saved pc_usr is set to 1b.
1015 mov r7, #0xffff0fff
1016 sub r7, r7, #(0xffff0fff - (0xffff0fc0 + (1b - __kuser_cmpxchg)))
1017 subs r8, r4, r7
1018 rsbscs r8, r8, #(2b - 1b)
1019 strcs r7, [sp, #S_PC]
1020 ret lr
1021 .previous
1022
1023 #else
1024 #warning "NPTL on non MMU needs fixing"
1025 mov r0, #-1
1026 adds r0, r0, #0
1027 usr_ret lr
1028 #endif
1029
1030 #else
1031
1032 smp_dmb arm
1033 1: ldrex r3, [r2]
1034 subs r3, r3, r0
1035 strexeq r3, r1, [r2]
1036 teqeq r3, #1
1037 beq 1b
1038 rsbs r0, r3, #0
1039
1040 ALT_SMP(b __kuser_memory_barrier)
1041 ALT_UP(usr_ret lr)
1042
1043 #endif
1044
1045 kuser_pad __kuser_cmpxchg, 32
1046
1047 __kuser_get_tls: @ 0xffff0fe0
1048 ldr r0, [pc, #(16 - 8)] @ read TLS, set in kuser_get_tls_init
1049 usr_ret lr
1050 mrc p15, 0, r0, c13, c0, 3 @ 0xffff0fe8 hardware TLS code
1051 kuser_pad __kuser_get_tls, 16
1052 .rep 3
1053 .word 0 @ 0xffff0ff0 software TLS value, then
1054 .endr @ pad up to __kuser_helper_version
1055
1056 __kuser_helper_version: @ 0xffff0ffc
1057 .word ((__kuser_helper_end - __kuser_helper_start) >> 5)
1058
1059 .globl __kuser_helper_end
1060 __kuser_helper_end:
1061
1062 #endif
1063
1064 THUMB( .thumb )
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079 .macro vector_stub, name, mode, correction=0
1080 .align 5
1081 #ifdef CONFIG_HARDEN_BRANCH_HISTORY
1082 vector_bhb_bpiall_\name:
1083 mcr p15, 0, r0, c7, c5, 6 @ BPIALL
1084 @ isb not needed due to "movs pc, lr" in the vector stub
1085 @ which gives a "context synchronisation".
1086 #endif
1087
1088 vector_\name:
1089 .if \correction
1090 sub lr, lr, #\correction
1091 .endif
1092
1093 @ Save r0, lr_<exception> (parent PC)
1094 stmia sp, {r0, lr} @ save r0, lr
1095
1096 @ Save spsr_<exception> (parent CPSR)
1097 .Lvec_\name:
1098 mrs lr, spsr
1099 str lr, [sp, #8] @ save spsr
1100
1101 @
1102 @ Prepare for SVC32 mode. IRQs remain disabled.
1103 @
1104 mrs r0, cpsr
1105 eor r0, r0, #(\mode ^ SVC_MODE | PSR_ISETSTATE)
1106 msr spsr_cxsf, r0
1107
1108 @
1109 @ the branch table must immediately follow this code
1110 @
1111 and lr, lr, #0x0f
1112 THUMB( adr r0, 1f )
1113 THUMB( ldr lr, [r0, lr, lsl #2] )
1114 mov r0, sp
1115 ARM( ldr lr, [pc, lr, lsl #2] )
1116 movs pc, lr @ branch to handler in SVC mode
1117 ENDPROC(vector_\name)
1118
1119 #ifdef CONFIG_HARDEN_BRANCH_HISTORY
1120 .subsection 1
1121 .align 5
1122 vector_bhb_loop8_\name:
1123 .if \correction
1124 sub lr, lr, #\correction
1125 .endif
1126
1127 @ Save r0, lr_<exception> (parent PC)
1128 stmia sp, {r0, lr}
1129
1130 @ bhb workaround
1131 mov r0, #8
1132 3: W(b) . + 4
1133 subs r0, r0, #1
1134 bne 3b
1135 dsb nsh
1136 @ isb not needed due to "movs pc, lr" in the vector stub
1137 @ which gives a "context synchronisation".
1138 b .Lvec_\name
1139 ENDPROC(vector_bhb_loop8_\name)
1140 .previous
1141 #endif
1142
1143 .align 2
1144 @ handler addresses follow this label
1145 1:
1146 .endm
1147
1148 .section .stubs, "ax", %progbits
1149 @ These need to remain at the start of the section so that
1150 @ they are in range of the 'SWI' entries in the vector tables
1151 @ located 4k down.
1152 .L__vector_swi:
1153 .word vector_swi
1154 #ifdef CONFIG_HARDEN_BRANCH_HISTORY
1155 .L__vector_bhb_loop8_swi:
1156 .word vector_bhb_loop8_swi
1157 .L__vector_bhb_bpiall_swi:
1158 .word vector_bhb_bpiall_swi
1159 #endif
1160
1161 vector_rst:
1162 ARM( swi SYS_ERROR0 )
1163 THUMB( svc #0 )
1164 THUMB( nop )
1165 b vector_und
1166
1167
1168
1169
1170 vector_stub irq, IRQ_MODE, 4
1171
1172 .long __irq_usr @ 0 (USR_26 / USR_32)
1173 .long __irq_invalid @ 1 (FIQ_26 / FIQ_32)
1174 .long __irq_invalid @ 2 (IRQ_26 / IRQ_32)
1175 .long __irq_svc @ 3 (SVC_26 / SVC_32)
1176 .long __irq_invalid @ 4
1177 .long __irq_invalid @ 5
1178 .long __irq_invalid @ 6
1179 .long __irq_invalid @ 7
1180 .long __irq_invalid @ 8
1181 .long __irq_invalid @ 9
1182 .long __irq_invalid @ a
1183 .long __irq_invalid @ b
1184 .long __irq_invalid @ c
1185 .long __irq_invalid @ d
1186 .long __irq_invalid @ e
1187 .long __irq_invalid @ f
1188
1189
1190
1191
1192
1193 vector_stub dabt, ABT_MODE, 8
1194
1195 .long __dabt_usr @ 0 (USR_26 / USR_32)
1196 .long __dabt_invalid @ 1 (FIQ_26 / FIQ_32)
1197 .long __dabt_invalid @ 2 (IRQ_26 / IRQ_32)
1198 .long __dabt_svc @ 3 (SVC_26 / SVC_32)
1199 .long __dabt_invalid @ 4
1200 .long __dabt_invalid @ 5
1201 .long __dabt_invalid @ 6
1202 .long __dabt_invalid @ 7
1203 .long __dabt_invalid @ 8
1204 .long __dabt_invalid @ 9
1205 .long __dabt_invalid @ a
1206 .long __dabt_invalid @ b
1207 .long __dabt_invalid @ c
1208 .long __dabt_invalid @ d
1209 .long __dabt_invalid @ e
1210 .long __dabt_invalid @ f
1211
1212
1213
1214
1215
1216 vector_stub pabt, ABT_MODE, 4
1217
1218 .long __pabt_usr @ 0 (USR_26 / USR_32)
1219 .long __pabt_invalid @ 1 (FIQ_26 / FIQ_32)
1220 .long __pabt_invalid @ 2 (IRQ_26 / IRQ_32)
1221 .long __pabt_svc @ 3 (SVC_26 / SVC_32)
1222 .long __pabt_invalid @ 4
1223 .long __pabt_invalid @ 5
1224 .long __pabt_invalid @ 6
1225 .long __pabt_invalid @ 7
1226 .long __pabt_invalid @ 8
1227 .long __pabt_invalid @ 9
1228 .long __pabt_invalid @ a
1229 .long __pabt_invalid @ b
1230 .long __pabt_invalid @ c
1231 .long __pabt_invalid @ d
1232 .long __pabt_invalid @ e
1233 .long __pabt_invalid @ f
1234
1235
1236
1237
1238
1239 vector_stub und, UND_MODE
1240
1241 .long __und_usr @ 0 (USR_26 / USR_32)
1242 .long __und_invalid @ 1 (FIQ_26 / FIQ_32)
1243 .long __und_invalid @ 2 (IRQ_26 / IRQ_32)
1244 .long __und_svc @ 3 (SVC_26 / SVC_32)
1245 .long __und_invalid @ 4
1246 .long __und_invalid @ 5
1247 .long __und_invalid @ 6
1248 .long __und_invalid @ 7
1249 .long __und_invalid @ 8
1250 .long __und_invalid @ 9
1251 .long __und_invalid @ a
1252 .long __und_invalid @ b
1253 .long __und_invalid @ c
1254 .long __und_invalid @ d
1255 .long __und_invalid @ e
1256 .long __und_invalid @ f
1257
1258 .align 5
1259
1260
1261
1262
1263
1264
1265
1266
1267 vector_addrexcptn:
1268 b vector_addrexcptn
1269
1270
1271
1272
1273
1274
1275
1276
1277 .subsection 2
1278 vector_stub fiq, FIQ_MODE, 4
1279
1280 .long __fiq_usr @ 0 (USR_26 / USR_32)
1281 .long __fiq_svc @ 1 (FIQ_26 / FIQ_32)
1282 .long __fiq_svc @ 2 (IRQ_26 / IRQ_32)
1283 .long __fiq_svc @ 3 (SVC_26 / SVC_32)
1284 .long __fiq_svc @ 4
1285 .long __fiq_svc @ 5
1286 .long __fiq_svc @ 6
1287 .long __fiq_abt @ 7
1288 .long __fiq_svc @ 8
1289 .long __fiq_svc @ 9
1290 .long __fiq_svc @ a
1291 .long __fiq_svc @ b
1292 .long __fiq_svc @ c
1293 .long __fiq_svc @ d
1294 .long __fiq_svc @ e
1295 .long __fiq_svc @ f
1296
1297 .globl vector_fiq
1298
1299 .section .vectors, "ax", %progbits
1300 W(b) vector_rst
1301 W(b) vector_und
1302 ARM( .reloc ., R_ARM_LDR_PC_G0, .L__vector_swi )
1303 THUMB( .reloc ., R_ARM_THM_PC12, .L__vector_swi )
1304 W(ldr) pc, .
1305 W(b) vector_pabt
1306 W(b) vector_dabt
1307 W(b) vector_addrexcptn
1308 W(b) vector_irq
1309 W(b) vector_fiq
1310
1311 #ifdef CONFIG_HARDEN_BRANCH_HISTORY
1312 .section .vectors.bhb.loop8, "ax", %progbits
1313 W(b) vector_rst
1314 W(b) vector_bhb_loop8_und
1315 ARM( .reloc ., R_ARM_LDR_PC_G0, .L__vector_bhb_loop8_swi )
1316 THUMB( .reloc ., R_ARM_THM_PC12, .L__vector_bhb_loop8_swi )
1317 W(ldr) pc, .
1318 W(b) vector_bhb_loop8_pabt
1319 W(b) vector_bhb_loop8_dabt
1320 W(b) vector_addrexcptn
1321 W(b) vector_bhb_loop8_irq
1322 W(b) vector_bhb_loop8_fiq
1323
1324 .section .vectors.bhb.bpiall, "ax", %progbits
1325 W(b) vector_rst
1326 W(b) vector_bhb_bpiall_und
1327 ARM( .reloc ., R_ARM_LDR_PC_G0, .L__vector_bhb_bpiall_swi )
1328 THUMB( .reloc ., R_ARM_THM_PC12, .L__vector_bhb_bpiall_swi )
1329 W(ldr) pc, .
1330 W(b) vector_bhb_bpiall_pabt
1331 W(b) vector_bhb_bpiall_dabt
1332 W(b) vector_addrexcptn
1333 W(b) vector_bhb_bpiall_irq
1334 W(b) vector_bhb_bpiall_fiq
1335 #endif
1336
1337 .data
1338 .align 2
1339
1340 .globl cr_alignment
1341 cr_alignment:
1342 .space 4
