arc/mm/tlbex.S

0001 /* SPDX-License-Identifier: GPL-2.0-only */
0002 /*
0003  * TLB Exception Handling for ARC
0004  *
0005  * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
0006  *
0007  * Vineetg: April 2011 :
0008  *  -MMU v1: moved out legacy code into a seperate file
0009  *  -MMU v3: PD{0,1} bits layout changed: They don't overlap anymore,
0010  *      helps avoid a shift when preparing PD0 from PTE
0011  *
0012  * Vineetg: July 2009
0013  *  -For MMU V2, we need not do heuristics at the time of commiting a D-TLB
0014  *   entry, so that it doesn't knock out it's I-TLB entry
0015  *  -Some more fine tuning:
0016  *   bmsk instead of add, asl.cc instead of branch, delay slot utilise etc
0017  *
0018  * Vineetg: July 2009
0019  *  -Practically rewrote the I/D TLB Miss handlers
0020  *   Now 40 and 135 instructions a peice as compared to 131 and 449 resp.
0021  *   Hence Leaner by 1.5 K
0022  *   Used Conditional arithmetic to replace excessive branching
0023  *   Also used short instructions wherever possible
0024  *
0025  * Vineetg: Aug 13th 2008
0026  *  -Passing ECR (Exception Cause REG) to do_page_fault( ) for printing
0027  *   more information in case of a Fatality
0028  *
0029  * Vineetg: March 25th Bug #92690
0030  *  -Added Debug Code to check if sw-ASID == hw-ASID
0031
0032  * Rahul Trivedi, Amit Bhor: Codito Technologies 2004
0033  */
0034
0035 #include <linux/linkage.h>
0036 #include <linux/pgtable.h>
0037 #include <asm/entry.h>
0038 #include <asm/mmu.h>
0039 #include <asm/arcregs.h>
0040 #include <asm/cache.h>
0041 #include <asm/processor.h>
0042
0043 #ifdef CONFIG_ISA_ARCOMPACT
0044 ;-----------------------------------------------------------------
0045 ; ARC700 Exception Handling doesn't auto-switch stack and it only provides
0046 ; ONE scratch AUX reg "ARC_REG_SCRATCH_DATA0"
0047 ;
0048 ; For Non-SMP, the scratch AUX reg is repurposed to cache task PGD, so a
0049 ; "global" is used to free-up FIRST core reg to be able to code the rest of
0050 ; exception prologue (IRQ auto-disabled on Exceptions, so it's IRQ-safe).
0051 ; Since the Fast Path TLB Miss handler is coded with 4 regs, the remaining 3
0052 ; need to be saved as well by extending the "global" to be 4 words. Hence
0053 ;   ".size   ex_saved_reg1, 16"
0054 ; [All of this dance is to avoid stack switching for each TLB Miss, since we
0055 ; only need to save only a handful of regs, as opposed to complete reg file]
0056 ;
0057 ; For ARC700 SMP, the "global" obviously can't be used for free up the FIRST
0058 ; core reg as it will not be SMP safe.
0059 ; Thus scratch AUX reg is used (and no longer used to cache task PGD).
0060 ; To save the rest of 3 regs - per cpu, the global is made "per-cpu".
0061 ; Epilogue thus has to locate the "per-cpu" storage for regs.
0062 ; To avoid cache line bouncing the per-cpu global is aligned/sized per
0063 ; L1_CACHE_SHIFT, despite fundamentally needing to be 12 bytes only. Hence
0064 ;   ".size   ex_saved_reg1, (CONFIG_NR_CPUS << L1_CACHE_SHIFT)"
0065
0066 ; As simple as that....
0067 ;--------------------------------------------------------------------------
0068
0069 ; scratch memory to save [r0-r3] used to code TLB refill Handler
0070 ARCFP_DATA ex_saved_reg1
0071     .align 1 << L1_CACHE_SHIFT
0072     .type   ex_saved_reg1, @object
0073 #ifdef CONFIG_SMP
0074     .size   ex_saved_reg1, (CONFIG_NR_CPUS << L1_CACHE_SHIFT)
0075 ex_saved_reg1:
0076     .zero (CONFIG_NR_CPUS << L1_CACHE_SHIFT)
0077 #else
0078     .size   ex_saved_reg1, 16
0079 ex_saved_reg1:
0080     .zero 16
0081 #endif
0082
0083 .macro TLBMISS_FREEUP_REGS
0084 #ifdef CONFIG_SMP
0085     sr  r0, [ARC_REG_SCRATCH_DATA0] ; freeup r0 to code with
0086     GET_CPU_ID  r0          ; get to per cpu scratch mem,
0087     asl r0, r0, L1_CACHE_SHIFT  ; cache line wide per cpu
0088     add r0, @ex_saved_reg1, r0
0089 #else
0090     st    r0, [@ex_saved_reg1]
0091     mov_s r0, @ex_saved_reg1
0092 #endif
0093     st_s  r1, [r0, 4]
0094     st_s  r2, [r0, 8]
0095     st_s  r3, [r0, 12]
0096 .endm
0097
0098 .macro TLBMISS_RESTORE_REGS
0099 #ifdef CONFIG_SMP
0100     GET_CPU_ID  r0          ; get to per cpu scratch mem
0101     asl r0, r0, L1_CACHE_SHIFT  ; each is cache line wide
0102     add r0, @ex_saved_reg1, r0
0103     ld_s  r3, [r0,12]
0104     ld_s  r2, [r0, 8]
0105     ld_s  r1, [r0, 4]
0106     lr    r0, [ARC_REG_SCRATCH_DATA0]
0107 #else
0108     mov_s r0, @ex_saved_reg1
0109     ld_s  r3, [r0,12]
0110     ld_s  r2, [r0, 8]
0111     ld_s  r1, [r0, 4]
0112     ld_s  r0, [r0]
0113 #endif
0114 .endm
0115
0116 #else   /* ARCv2 */
0117
0118 .macro TLBMISS_FREEUP_REGS
0119 #ifdef CONFIG_ARC_HAS_LL64
0120     std   r0, [sp, -16]
0121     std   r2, [sp, -8]
0122 #else
0123     PUSH  r0
0124     PUSH  r1
0125     PUSH  r2
0126     PUSH  r3
0127 #endif
0128 .endm
0129
0130 .macro TLBMISS_RESTORE_REGS
0131 #ifdef CONFIG_ARC_HAS_LL64
0132     ldd   r0, [sp, -16]
0133     ldd   r2, [sp, -8]
0134 #else
0135     POP   r3
0136     POP   r2
0137     POP   r1
0138     POP   r0
0139 #endif
0140 .endm
0141
0142 #endif
0143
0144 ;============================================================================
0145 ;TLB Miss handling Code
0146 ;============================================================================
0147
0148 #ifndef PMD_SHIFT
0149 #define PMD_SHIFT PUD_SHIFT
0150 #endif
0151
0152 #ifndef PUD_SHIFT
0153 #define PUD_SHIFT PGDIR_SHIFT
0154 #endif
0155
0156 ;-----------------------------------------------------------------------------
0157 ; This macro does the page-table lookup for the faulting address.
0158 ; OUT: r0 = PTE faulted on, r1 = ptr to PTE, r2 = Faulting V-address
0159 .macro LOAD_FAULT_PTE
0160
0161     lr  r2, [efa]
0162
0163 #ifdef CONFIG_ISA_ARCV2
0164     lr  r1, [ARC_REG_SCRATCH_DATA0] ; current pgd
0165 #else
0166     GET_CURR_TASK_ON_CPU  r1
0167     ld  r1, [r1, TASK_ACT_MM]
0168     ld  r1, [r1, MM_PGD]
0169 #endif
0170
0171     lsr     r0, r2, PGDIR_SHIFT     ; Bits for indexing into PGD
0172     ld.as   r3, [r1, r0]            ; PGD entry corresp to faulting addr
0173     tst r3, r3
0174     bz  do_slow_path_pf         ; if no Page Table, do page fault
0175
0176 #if CONFIG_PGTABLE_LEVELS > 3
0177     lsr     r0, r2, PUD_SHIFT   ; Bits for indexing into PUD
0178     and r0, r0, (PTRS_PER_PUD - 1)
0179     ld.as   r1, [r3, r0]        ; PMD entry
0180     tst r1, r1
0181     bz  do_slow_path_pf
0182     mov r3, r1
0183 #endif
0184
0185 #if CONFIG_PGTABLE_LEVELS > 2
0186     lsr     r0, r2, PMD_SHIFT   ; Bits for indexing into PMD
0187     and r0, r0, (PTRS_PER_PMD - 1)
0188     ld.as   r1, [r3, r0]        ; PMD entry
0189     tst r1, r1
0190     bz  do_slow_path_pf
0191     mov r3, r1
0192 #endif
0193
0194 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
0195     and.f   0, r3, _PAGE_HW_SZ  ; Is this Huge PMD (thp)
0196     add2.nz r1, r1, r0
0197     bnz.d   2f      ; YES: PGD == PMD has THP PTE: stop pgd walk
0198     mov.nz  r0, r3
0199
0200 #endif
0201     and r1, r3, PAGE_MASK
0202
0203     ; Get the PTE entry: The idea is
0204     ; (1) x = addr >> PAGE_SHIFT    -> masks page-off bits from @fault-addr
0205     ; (2) y = x & (PTRS_PER_PTE - 1) -> to get index
0206     ; (3) z = (pgtbl + y * 4)
0207
0208 #ifdef CONFIG_ARC_HAS_PAE40
0209 #define PTE_SIZE_LOG    3   /* 8 == 2 ^ 3 */
0210 #else
0211 #define PTE_SIZE_LOG    2   /* 4 == 2 ^ 2 */
0212 #endif
0213
0214     ; multiply in step (3) above avoided by shifting lesser in step (1)
0215     lsr     r0, r2, ( PAGE_SHIFT - PTE_SIZE_LOG )
0216     and     r0, r0, ( (PTRS_PER_PTE - 1) << PTE_SIZE_LOG )
0217     ld.aw   r0, [r1, r0]            ; r0: PTE (lower word only for PAE40)
0218                     ; r1: PTE ptr
0219
0220 2:
0221
0222 .endm
0223
0224 ;-----------------------------------------------------------------
0225 ; Convert Linux PTE entry into TLB entry
0226 ; A one-word PTE entry is programmed as two-word TLB Entry [PD0:PD1] in mmu
0227 ;    (for PAE40, two-words PTE, while three-word TLB Entry [PD0:PD1:PD1HI])
0228 ; IN: r0 = PTE, r1 = ptr to PTE
0229
0230 .macro CONV_PTE_TO_TLB
0231     and    r3, r0, PTE_BITS_RWX ;          r  w  x
0232     asl    r2, r3, 3        ; Kr Kw Kx 0  0  0 (GLOBAL, kernel only)
0233     and.f  0,  r0, _PAGE_GLOBAL
0234     or.z   r2, r2, r3       ; Kr Kw Kx Ur Uw Ux (!GLOBAL, user page)
0235
0236     and r3, r0, PTE_BITS_NON_RWX_IN_PD1 ; Extract PFN+cache bits from PTE
0237     or  r3, r3, r2
0238
0239     sr  r3, [ARC_REG_TLBPD1]        ; paddr[31..13] | Kr Kw Kx Ur Uw Ux | C
0240 #ifdef  CONFIG_ARC_HAS_PAE40
0241     ld  r3, [r1, 4]     ; paddr[39..32]
0242     sr  r3, [ARC_REG_TLBPD1HI]
0243 #endif
0244
0245     and r2, r0, PTE_BITS_IN_PD0 ; Extract other PTE flags: (V)alid, (G)lb
0246
0247     lr  r3,[ARC_REG_TLBPD0]     ; MMU prepares PD0 with vaddr and asid
0248
0249     or  r3, r3, r2              ; S | vaddr | {sasid|asid}
0250     sr  r3,[ARC_REG_TLBPD0]     ; rewrite PD0
0251 .endm
0252
0253 ;-----------------------------------------------------------------
0254 ; Commit the TLB entry into MMU
0255
0256 .macro COMMIT_ENTRY_TO_MMU
0257 #ifdef CONFIG_ARC_MMU_V3
0258
0259     /* Get free TLB slot: Set = computed from vaddr, way = random */
0260     sr  TLBGetIndex, [ARC_REG_TLBCOMMAND]
0261
0262     /* Commit the Write */
0263     sr TLBWriteNI, [ARC_REG_TLBCOMMAND]
0264
0265 #else
0266     sr TLBInsertEntry, [ARC_REG_TLBCOMMAND]
0267 #endif
0268
0269 88:
0270 .endm
0271
0272
0273 ARCFP_CODE  ;Fast Path Code, candidate for ICCM
0274
0275 ;-----------------------------------------------------------------------------
0276 ; I-TLB Miss Exception Handler
0277 ;-----------------------------------------------------------------------------
0278
0279 ENTRY(EV_TLBMissI)
0280
0281     TLBMISS_FREEUP_REGS
0282
0283     ;----------------------------------------------------------------
0284     ; Get the PTE corresponding to V-addr accessed, r2 is setup with EFA
0285     LOAD_FAULT_PTE
0286
0287     ;----------------------------------------------------------------
0288     ; VERIFY_PTE: Check if PTE permissions approp for executing code
0289     cmp_s   r2, VMALLOC_START
0290     mov_s   r2, (_PAGE_PRESENT | _PAGE_EXECUTE)
0291     or.hs   r2, r2, _PAGE_GLOBAL
0292
0293     and     r3, r0, r2  ; Mask out NON Flag bits from PTE
0294     xor.f   r3, r3, r2  ; check ( ( pte & flags_test ) == flags_test )
0295     bnz     do_slow_path_pf
0296
0297     ; Let Linux VM know that the page was accessed
0298     or      r0, r0, _PAGE_ACCESSED  ; set Accessed Bit
0299     st_s    r0, [r1]                ; Write back PTE
0300
0301     CONV_PTE_TO_TLB
0302     COMMIT_ENTRY_TO_MMU
0303     TLBMISS_RESTORE_REGS
0304 EV_TLBMissI_fast_ret:   ; additional label for VDK OS-kit instrumentation
0305     rtie
0306
0307 END(EV_TLBMissI)
0308
0309 ;-----------------------------------------------------------------------------
0310 ; D-TLB Miss Exception Handler
0311 ;-----------------------------------------------------------------------------
0312
0313 ENTRY(EV_TLBMissD)
0314
0315     TLBMISS_FREEUP_REGS
0316
0317     ;----------------------------------------------------------------
0318     ; Get the PTE corresponding to V-addr accessed
0319     ; If PTE exists, it will setup, r0 = PTE, r1 = Ptr to PTE, r2 = EFA
0320     LOAD_FAULT_PTE
0321
0322     ;----------------------------------------------------------------
0323     ; VERIFY_PTE: Chk if PTE permissions approp for data access (R/W/R+W)
0324
0325     cmp_s   r2, VMALLOC_START
0326     mov_s   r2, _PAGE_PRESENT   ; common bit for K/U PTE
0327     or.hs   r2, r2, _PAGE_GLOBAL    ; kernel PTE only
0328
0329     ; Linux PTE [RWX] bits are semantically overloaded:
0330     ; -If PAGE_GLOBAL set, they refer to kernel-only flags (vmalloc)
0331     ; -Otherwise they are user-mode permissions, and those are exactly
0332     ;  same for kernel mode as well (e.g. copy_(to|from)_user)
0333
0334     lr      r3, [ecr]
0335     btst_s  r3, ECR_C_BIT_DTLB_LD_MISS  ; Read Access
0336     or.nz   r2, r2, _PAGE_READ          ; chk for Read flag in PTE
0337     btst_s  r3, ECR_C_BIT_DTLB_ST_MISS  ; Write Access
0338     or.nz   r2, r2, _PAGE_WRITE         ; chk for Write flag in PTE
0339     ; Above laddering takes care of XCHG access (both R and W)
0340
0341     ; By now, r2 setup with all the Flags we need to check in PTE
0342     and     r3, r0, r2              ; Mask out NON Flag bits from PTE
0343     brne.d  r3, r2, do_slow_path_pf ; is ((pte & flags_test) == flags_test)
0344
0345     ;----------------------------------------------------------------
0346     ; UPDATE_PTE: Let Linux VM know that page was accessed/dirty
0347     or      r0, r0, _PAGE_ACCESSED        ; Accessed bit always
0348     or.nz   r0, r0, _PAGE_DIRTY           ; if Write, set Dirty bit as well
0349     st_s    r0, [r1]                      ; Write back PTE
0350
0351     CONV_PTE_TO_TLB
0352
0353     COMMIT_ENTRY_TO_MMU
0354     TLBMISS_RESTORE_REGS
0355 EV_TLBMissD_fast_ret:   ; additional label for VDK OS-kit instrumentation
0356     rtie
0357
0358 ;-------- Common routine to call Linux Page Fault Handler -----------
0359 do_slow_path_pf:
0360
0361 #ifdef CONFIG_ISA_ARCV2
0362     ; Set Z flag if exception in U mode. Hardware micro-ops do this on any
0363     ; taken interrupt/exception, and thus is already the case at the entry
0364     ; above, but ensuing code would have already clobbered.
0365     ; EXCEPTION_PROLOGUE called in slow path, relies on correct Z flag set
0366
0367     lr  r2, [erstatus]
0368     and r2, r2, STATUS_U_MASK
0369     bxor.f  0, r2, STATUS_U_BIT
0370 #endif
0371
0372     ; Restore the 4-scratch regs saved by fast path miss handler
0373     TLBMISS_RESTORE_REGS
0374
0375     ; Slow path TLB Miss handled as a regular ARC Exception
0376     ; (stack switching / save the complete reg-file).
0377     b  call_do_page_fault
0378 END(EV_TLBMissD)