Back to home page

OSCL-LXR
 
 

     

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 /* sun4v_tlb_miss.S: Sun4v TLB miss handlers.
0003  *
0004  * Copyright (C) 2006 <davem@davemloft.net>
0005  */
0006 
0007     .text
0008     .align  32
0009 
0010     /* Load ITLB fault information into VADDR and CTX, using BASE.  */
0011 #define LOAD_ITLB_INFO(BASE, VADDR, CTX) \
0012     ldx [BASE + HV_FAULT_I_ADDR_OFFSET], VADDR; \
0013     ldx [BASE + HV_FAULT_I_CTX_OFFSET], CTX;
0014 
0015     /* Load DTLB fault information into VADDR and CTX, using BASE.  */
0016 #define LOAD_DTLB_INFO(BASE, VADDR, CTX) \
0017     ldx [BASE + HV_FAULT_D_ADDR_OFFSET], VADDR; \
0018     ldx [BASE + HV_FAULT_D_CTX_OFFSET], CTX;
0019 
0020     /* DEST = (VADDR >> 22)
0021      *
0022      * Branch to ZERO_CTX_LABEL if context is zero.
0023      */
0024 #define COMPUTE_TAG_TARGET(DEST, VADDR, CTX, ZERO_CTX_LABEL) \
0025     srlx    VADDR, 22, DEST; \
0026     brz,pn  CTX, ZERO_CTX_LABEL; \
0027      nop;
0028 
0029     /* Create TSB pointer.  This is something like:
0030      *
0031      * index_mask = (512 << (tsb_reg & 0x7UL)) - 1UL;
0032      * tsb_base = tsb_reg & ~0x7UL;
0033      * tsb_index = ((vaddr >> HASH_SHIFT) & tsb_mask);
0034      * tsb_ptr = tsb_base + (tsb_index * 16);
0035      */
0036 #define COMPUTE_TSB_PTR(TSB_PTR, VADDR, HASH_SHIFT, TMP1, TMP2) \
0037     and TSB_PTR, 0x7, TMP1;         \
0038     mov 512, TMP2;              \
0039     andn    TSB_PTR, 0x7, TSB_PTR;          \
0040     sllx    TMP2, TMP1, TMP2;           \
0041     srlx    VADDR, HASH_SHIFT, TMP1;        \
0042     sub TMP2, 1, TMP2;              \
0043     and TMP1, TMP2, TMP1;           \
0044     sllx    TMP1, 4, TMP1;              \
0045     add TSB_PTR, TMP1, TSB_PTR;
0046 
0047 sun4v_itlb_miss:
0048     /* Load MMU Miss base into %g2.  */
0049     ldxa    [%g0] ASI_SCRATCHPAD, %g2
0050     
0051     /* Load UTSB reg into %g1.  */
0052     mov SCRATCHPAD_UTSBREG1, %g1
0053     ldxa    [%g1] ASI_SCRATCHPAD, %g1
0054 
0055     LOAD_ITLB_INFO(%g2, %g4, %g5)
0056     COMPUTE_TAG_TARGET(%g6, %g4, %g5, kvmap_itlb_4v)
0057     COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g3, %g7)
0058 
0059     /* Load TSB tag/pte into %g2/%g3 and compare the tag.  */
0060     ldda    [%g1] ASI_QUAD_LDD_PHYS_4V, %g2
0061     cmp %g2, %g6
0062     bne,a,pn %xcc, tsb_miss_page_table_walk
0063      mov    FAULT_CODE_ITLB, %g3
0064     andcc   %g3, _PAGE_EXEC_4V, %g0
0065     be,a,pn %xcc, tsb_do_fault
0066      mov    FAULT_CODE_ITLB, %g3
0067 
0068     /* We have a valid entry, make hypervisor call to load
0069      * I-TLB and return from trap.
0070      *
0071      * %g3: PTE
0072      * %g4: vaddr
0073      */
0074 sun4v_itlb_load:
0075     ldxa    [%g0] ASI_SCRATCHPAD, %g6
0076     mov %o0, %g1        ! save %o0
0077     mov %o1, %g2        ! save %o1
0078     mov %o2, %g5        ! save %o2
0079     mov %o3, %g7        ! save %o3
0080     mov %g4, %o0        ! vaddr
0081     ldx [%g6 + HV_FAULT_I_CTX_OFFSET], %o1  ! ctx
0082     mov %g3, %o2        ! PTE
0083     mov HV_MMU_IMMU, %o3    ! flags
0084     ta  HV_MMU_MAP_ADDR_TRAP
0085     brnz,pn %o0, sun4v_itlb_error
0086      mov    %g2, %o1        ! restore %o1
0087     mov %g1, %o0        ! restore %o0
0088     mov %g5, %o2        ! restore %o2
0089     mov %g7, %o3        ! restore %o3
0090 
0091     retry
0092 
0093 sun4v_dtlb_miss:
0094     /* Load MMU Miss base into %g2.  */
0095     ldxa    [%g0] ASI_SCRATCHPAD, %g2
0096     
0097     /* Load UTSB reg into %g1.  */
0098     mov SCRATCHPAD_UTSBREG1, %g1
0099     ldxa    [%g1] ASI_SCRATCHPAD, %g1
0100 
0101     LOAD_DTLB_INFO(%g2, %g4, %g5)
0102     COMPUTE_TAG_TARGET(%g6, %g4, %g5, kvmap_dtlb_4v)
0103     COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g3, %g7)
0104 
0105     /* Load TSB tag/pte into %g2/%g3 and compare the tag.  */
0106     ldda    [%g1] ASI_QUAD_LDD_PHYS_4V, %g2
0107     cmp %g2, %g6
0108     bne,a,pn %xcc, tsb_miss_page_table_walk
0109      mov    FAULT_CODE_DTLB, %g3
0110 
0111     /* We have a valid entry, make hypervisor call to load
0112      * D-TLB and return from trap.
0113      *
0114      * %g3: PTE
0115      * %g4: vaddr
0116      */
0117 sun4v_dtlb_load:
0118     ldxa    [%g0] ASI_SCRATCHPAD, %g6
0119     mov %o0, %g1        ! save %o0
0120     mov %o1, %g2        ! save %o1
0121     mov %o2, %g5        ! save %o2
0122     mov %o3, %g7        ! save %o3
0123     mov %g4, %o0        ! vaddr
0124     ldx [%g6 + HV_FAULT_D_CTX_OFFSET], %o1  ! ctx
0125     mov %g3, %o2        ! PTE
0126     mov HV_MMU_DMMU, %o3    ! flags
0127     ta  HV_MMU_MAP_ADDR_TRAP
0128     brnz,pn %o0, sun4v_dtlb_error
0129      mov    %g2, %o1        ! restore %o1
0130     mov %g1, %o0        ! restore %o0
0131     mov %g5, %o2        ! restore %o2
0132     mov %g7, %o3        ! restore %o3
0133 
0134     retry
0135 
0136 sun4v_dtlb_prot:
0137     SET_GL(1)
0138 
0139     /* Load MMU Miss base into %g5.  */
0140     ldxa    [%g0] ASI_SCRATCHPAD, %g5
0141     
0142     ldx [%g5 + HV_FAULT_D_ADDR_OFFSET], %g5
0143     rdpr    %tl, %g1
0144     cmp %g1, 1
0145     bgu,pn  %xcc, winfix_trampoline
0146      mov    FAULT_CODE_DTLB | FAULT_CODE_WRITE, %g4
0147     ba,pt   %xcc, sparc64_realfault_common
0148      nop
0149 
0150     /* Called from trap table:
0151      * %g4: vaddr
0152      * %g5: context
0153      * %g6: TAG TARGET
0154      */
0155 sun4v_itsb_miss:
0156     mov SCRATCHPAD_UTSBREG1, %g1
0157     ldxa    [%g1] ASI_SCRATCHPAD, %g1
0158     brz,pn  %g5, kvmap_itlb_4v
0159      mov    FAULT_CODE_ITLB, %g3
0160     ba,a,pt %xcc, sun4v_tsb_miss_common
0161 
0162     /* Called from trap table:
0163      * %g4: vaddr
0164      * %g5: context
0165      * %g6: TAG TARGET
0166      */
0167 sun4v_dtsb_miss:
0168     mov SCRATCHPAD_UTSBREG1, %g1
0169     ldxa    [%g1] ASI_SCRATCHPAD, %g1
0170     brz,pn  %g5, kvmap_dtlb_4v
0171      mov    FAULT_CODE_DTLB, %g3
0172 
0173     /* fallthrough */
0174 
0175 sun4v_tsb_miss_common:
0176     COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g5, %g7)
0177 
0178     sub %g2, TRAP_PER_CPU_FAULT_INFO, %g2
0179 
0180 #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
0181     mov SCRATCHPAD_UTSBREG2, %g5
0182     ldxa    [%g5] ASI_SCRATCHPAD, %g5
0183     cmp %g5, -1
0184     be,pt   %xcc, 80f
0185      nop
0186     COMPUTE_TSB_PTR(%g5, %g4, REAL_HPAGE_SHIFT, %g2, %g7)
0187 
0188     /* That clobbered %g2, reload it.  */
0189     ldxa    [%g0] ASI_SCRATCHPAD, %g2
0190     sub %g2, TRAP_PER_CPU_FAULT_INFO, %g2
0191 
0192 80: stx %g5, [%g2 + TRAP_PER_CPU_TSB_HUGE_TEMP]
0193 #endif
0194 
0195     ba,pt   %xcc, tsb_miss_page_table_walk_sun4v_fastpath
0196      ldx    [%g2 + TRAP_PER_CPU_PGD_PADDR], %g7
0197 
0198 sun4v_itlb_error:
0199     rdpr    %tl, %g1
0200     cmp %g1, 1
0201     ble,pt  %icc, sun4v_bad_ra
0202      or %g0, FAULT_CODE_BAD_RA | FAULT_CODE_ITLB, %g1
0203 
0204     sethi   %hi(sun4v_err_itlb_vaddr), %g1
0205     stx %g4, [%g1 + %lo(sun4v_err_itlb_vaddr)]
0206     sethi   %hi(sun4v_err_itlb_ctx), %g1
0207     ldxa    [%g0] ASI_SCRATCHPAD, %g6
0208     ldx [%g6 + HV_FAULT_I_CTX_OFFSET], %o1
0209     stx %o1, [%g1 + %lo(sun4v_err_itlb_ctx)]
0210     sethi   %hi(sun4v_err_itlb_pte), %g1
0211     stx %g3, [%g1 + %lo(sun4v_err_itlb_pte)]
0212     sethi   %hi(sun4v_err_itlb_error), %g1
0213     stx %o0, [%g1 + %lo(sun4v_err_itlb_error)]
0214 
0215     sethi   %hi(1f), %g7
0216     rdpr    %tl, %g4
0217     ba,pt   %xcc, etraptl1
0218 1:   or %g7, %lo(1f), %g7
0219     mov %l4, %o1
0220     call    sun4v_itlb_error_report
0221      add    %sp, PTREGS_OFF, %o0
0222 
0223     /* NOTREACHED */
0224 
0225 sun4v_dtlb_error:
0226     rdpr    %tl, %g1
0227     cmp %g1, 1
0228     ble,pt  %icc, sun4v_bad_ra
0229      or %g0, FAULT_CODE_BAD_RA | FAULT_CODE_DTLB, %g1
0230 
0231     sethi   %hi(sun4v_err_dtlb_vaddr), %g1
0232     stx %g4, [%g1 + %lo(sun4v_err_dtlb_vaddr)]
0233     sethi   %hi(sun4v_err_dtlb_ctx), %g1
0234     ldxa    [%g0] ASI_SCRATCHPAD, %g6
0235     ldx [%g6 + HV_FAULT_D_CTX_OFFSET], %o1
0236     stx %o1, [%g1 + %lo(sun4v_err_dtlb_ctx)]
0237     sethi   %hi(sun4v_err_dtlb_pte), %g1
0238     stx %g3, [%g1 + %lo(sun4v_err_dtlb_pte)]
0239     sethi   %hi(sun4v_err_dtlb_error), %g1
0240     stx %o0, [%g1 + %lo(sun4v_err_dtlb_error)]
0241 
0242     sethi   %hi(1f), %g7
0243     rdpr    %tl, %g4
0244     ba,pt   %xcc, etraptl1
0245 1:   or %g7, %lo(1f), %g7
0246     mov %l4, %o1
0247     call    sun4v_dtlb_error_report
0248      add    %sp, PTREGS_OFF, %o0
0249 
0250     /* NOTREACHED */
0251 
0252 sun4v_bad_ra:
0253     or  %g0, %g4, %g5
0254     ba,pt   %xcc, sparc64_realfault_common
0255      or %g1, %g0, %g4
0256 
0257     /* NOTREACHED */
0258 
0259     /* Instruction Access Exception, tl0. */
0260 sun4v_iacc:
0261     ldxa    [%g0] ASI_SCRATCHPAD, %g2
0262     ldx [%g2 + HV_FAULT_I_TYPE_OFFSET], %g3
0263     ldx [%g2 + HV_FAULT_I_ADDR_OFFSET], %g4
0264     ldx [%g2 + HV_FAULT_I_CTX_OFFSET], %g5
0265     sllx    %g3, 16, %g3
0266     or  %g5, %g3, %g5
0267     ba,pt   %xcc, etrap
0268      rd %pc, %g7
0269     mov %l4, %o1
0270     mov %l5, %o2
0271     call    sun4v_insn_access_exception
0272      add    %sp, PTREGS_OFF, %o0
0273     ba,a,pt %xcc, rtrap
0274 
0275     /* Instruction Access Exception, tl1. */
0276 sun4v_iacc_tl1:
0277     ldxa    [%g0] ASI_SCRATCHPAD, %g2
0278     ldx [%g2 + HV_FAULT_I_TYPE_OFFSET], %g3
0279     ldx [%g2 + HV_FAULT_I_ADDR_OFFSET], %g4
0280     ldx [%g2 + HV_FAULT_I_CTX_OFFSET], %g5
0281     sllx    %g3, 16, %g3
0282     or  %g5, %g3, %g5
0283     ba,pt   %xcc, etraptl1
0284      rd %pc, %g7
0285     mov %l4, %o1
0286     mov %l5, %o2
0287     call    sun4v_insn_access_exception_tl1
0288      add    %sp, PTREGS_OFF, %o0
0289     ba,a,pt %xcc, rtrap
0290 
0291     /* Data Access Exception, tl0. */
0292 sun4v_dacc:
0293     ldxa    [%g0] ASI_SCRATCHPAD, %g2
0294     ldx [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3
0295     ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
0296     ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5
0297     sllx    %g3, 16, %g3
0298     or  %g5, %g3, %g5
0299     ba,pt   %xcc, etrap
0300      rd %pc, %g7
0301     mov %l4, %o1
0302     mov %l5, %o2
0303     call    sun4v_data_access_exception
0304      add    %sp, PTREGS_OFF, %o0
0305     ba,a,pt %xcc, rtrap
0306 
0307     /* Data Access Exception, tl1. */
0308 sun4v_dacc_tl1:
0309     ldxa    [%g0] ASI_SCRATCHPAD, %g2
0310     ldx [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3
0311     ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
0312     ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5
0313     sllx    %g3, 16, %g3
0314     or  %g5, %g3, %g5
0315     ba,pt   %xcc, etraptl1
0316      rd %pc, %g7
0317     mov %l4, %o1
0318     mov %l5, %o2
0319     call    sun4v_data_access_exception_tl1
0320      add    %sp, PTREGS_OFF, %o0
0321     ba,a,pt %xcc, rtrap
0322 
0323     /* Memory Address Unaligned.  */
0324 sun4v_mna:
0325     /* Window fixup? */
0326     rdpr    %tl, %g2
0327     cmp %g2, 1
0328     ble,pt  %icc, 1f
0329      nop
0330 
0331     SET_GL(1)
0332     ldxa    [%g0] ASI_SCRATCHPAD, %g2
0333     ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g5
0334     mov HV_FAULT_TYPE_UNALIGNED, %g3
0335     ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g4
0336     sllx    %g3, 16, %g3
0337     or  %g4, %g3, %g4
0338     ba,pt   %xcc, winfix_mna
0339      rdpr   %tpc, %g3
0340     /* not reached */
0341 
0342 1:  ldxa    [%g0] ASI_SCRATCHPAD, %g2
0343     mov HV_FAULT_TYPE_UNALIGNED, %g3
0344     ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
0345     ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5
0346     sllx    %g3, 16, %g3
0347     or  %g5, %g3, %g5
0348 
0349     ba,pt   %xcc, etrap
0350      rd %pc, %g7
0351     mov %l4, %o1
0352     mov %l5, %o2
0353     call    sun4v_do_mna
0354      add    %sp, PTREGS_OFF, %o0
0355     ba,a,pt %xcc, rtrap
0356      nop
0357 
0358     /* Privileged Action.  */
0359 sun4v_privact:
0360     ba,pt   %xcc, etrap
0361      rd %pc, %g7
0362     call    do_privact
0363      add    %sp, PTREGS_OFF, %o0
0364     ba,a,pt %xcc, rtrap
0365 
0366     /* Unaligned ldd float, tl0. */
0367 sun4v_lddfmna:
0368     ldxa    [%g0] ASI_SCRATCHPAD, %g2
0369     ldx [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3
0370     ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
0371     ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5
0372     sllx    %g3, 16, %g3
0373     or  %g5, %g3, %g5
0374     ba,pt   %xcc, etrap
0375      rd %pc, %g7
0376     mov %l4, %o1
0377     mov %l5, %o2
0378     call    handle_lddfmna
0379      add    %sp, PTREGS_OFF, %o0
0380     ba,a,pt %xcc, rtrap
0381 
0382     /* Unaligned std float, tl0. */
0383 sun4v_stdfmna:
0384     ldxa    [%g0] ASI_SCRATCHPAD, %g2
0385     ldx [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3
0386     ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
0387     ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5
0388     sllx    %g3, 16, %g3
0389     or  %g5, %g3, %g5
0390     ba,pt   %xcc, etrap
0391      rd %pc, %g7
0392     mov %l4, %o1
0393     mov %l5, %o2
0394     call    handle_stdfmna
0395      add    %sp, PTREGS_OFF, %o0
0396     ba,a,pt %xcc, rtrap
0397 
0398 #define BRANCH_ALWAYS   0x10680000
0399 #define NOP     0x01000000
0400 #define SUN4V_DO_PATCH(OLD, NEW)    \
0401     sethi   %hi(NEW), %g1; \
0402     or  %g1, %lo(NEW), %g1; \
0403     sethi   %hi(OLD), %g2; \
0404     or  %g2, %lo(OLD), %g2; \
0405     sub %g1, %g2, %g1; \
0406     sethi   %hi(BRANCH_ALWAYS), %g3; \
0407     sll %g1, 11, %g1; \
0408     srl %g1, 11 + 2, %g1; \
0409     or  %g3, %lo(BRANCH_ALWAYS), %g3; \
0410     or  %g3, %g1, %g3; \
0411     stw %g3, [%g2]; \
0412     sethi   %hi(NOP), %g3; \
0413     or  %g3, %lo(NOP), %g3; \
0414     stw %g3, [%g2 + 0x4]; \
0415     flush   %g2;
0416 
0417     .globl  sun4v_patch_tlb_handlers
0418     .type   sun4v_patch_tlb_handlers,#function
0419 sun4v_patch_tlb_handlers:
0420     SUN4V_DO_PATCH(tl0_iamiss, sun4v_itlb_miss)
0421     SUN4V_DO_PATCH(tl1_iamiss, sun4v_itlb_miss)
0422     SUN4V_DO_PATCH(tl0_damiss, sun4v_dtlb_miss)
0423     SUN4V_DO_PATCH(tl1_damiss, sun4v_dtlb_miss)
0424     SUN4V_DO_PATCH(tl0_daprot, sun4v_dtlb_prot)
0425     SUN4V_DO_PATCH(tl1_daprot, sun4v_dtlb_prot)
0426     SUN4V_DO_PATCH(tl0_iax, sun4v_iacc)
0427     SUN4V_DO_PATCH(tl1_iax, sun4v_iacc_tl1)
0428     SUN4V_DO_PATCH(tl0_dax, sun4v_dacc)
0429     SUN4V_DO_PATCH(tl1_dax, sun4v_dacc_tl1)
0430     SUN4V_DO_PATCH(tl0_mna, sun4v_mna)
0431     SUN4V_DO_PATCH(tl1_mna, sun4v_mna)
0432     SUN4V_DO_PATCH(tl0_lddfmna, sun4v_lddfmna)
0433     SUN4V_DO_PATCH(tl0_stdfmna, sun4v_stdfmna)
0434     SUN4V_DO_PATCH(tl0_privact, sun4v_privact)
0435     retl
0436      nop
0437     .size   sun4v_patch_tlb_handlers,.-sun4v_patch_tlb_handlers
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!