sparc/mm/hypersparc.S

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 /*
0003  * hypersparc.S: High speed Hypersparc mmu/cache operations.
0004  *
0005  * Copyright (C) 1997 David S. Miller (davem@caip.rutgers.edu)
0006  */
0007
0008 #include <asm/ptrace.h>
0009 #include <asm/psr.h>
0010 #include <asm/asm-offsets.h>
0011 #include <asm/asi.h>
0012 #include <asm/page.h>
0013 #include <asm/pgtable.h>
0014 #include <asm/pgtsrmmu.h>
0015 #include <linux/init.h>
0016
0017     .text
0018     .align  4
0019
0020     .globl  hypersparc_flush_cache_all, hypersparc_flush_cache_mm
0021     .globl  hypersparc_flush_cache_range, hypersparc_flush_cache_page
0022     .globl  hypersparc_flush_page_to_ram
0023     .globl  hypersparc_flush_page_for_dma, hypersparc_flush_sig_insns
0024     .globl  hypersparc_flush_tlb_all, hypersparc_flush_tlb_mm
0025     .globl  hypersparc_flush_tlb_range, hypersparc_flush_tlb_page
0026
0027 hypersparc_flush_cache_all:
0028     WINDOW_FLUSH(%g4, %g5)
0029     sethi   %hi(vac_cache_size), %g4
0030     ld  [%g4 + %lo(vac_cache_size)], %g5
0031     sethi   %hi(vac_line_size), %g1
0032     ld  [%g1 + %lo(vac_line_size)], %g2
0033 1:
0034     subcc   %g5, %g2, %g5           ! hyper_flush_unconditional_combined
0035     bne 1b
0036      sta    %g0, [%g5] ASI_M_FLUSH_CTX
0037     retl
0038      sta    %g0, [%g0] ASI_M_FLUSH_IWHOLE   ! hyper_flush_whole_icache
0039
0040     /* We expand the window flush to get maximum performance. */
0041 hypersparc_flush_cache_mm:
0042 #ifndef CONFIG_SMP
0043     ld  [%o0 + AOFF_mm_context], %g1
0044     cmp %g1, -1
0045     be  hypersparc_flush_cache_mm_out
0046 #endif
0047     WINDOW_FLUSH(%g4, %g5)
0048
0049     sethi   %hi(vac_line_size), %g1
0050     ld  [%g1 + %lo(vac_line_size)], %o1
0051     sethi   %hi(vac_cache_size), %g2
0052     ld  [%g2 + %lo(vac_cache_size)], %o0
0053     add %o1, %o1, %g1
0054     add %o1, %g1, %g2
0055     add %o1, %g2, %g3
0056     add %o1, %g3, %g4
0057     add %o1, %g4, %g5
0058     add %o1, %g5, %o4
0059     add %o1, %o4, %o5
0060
0061     /* BLAMMO! */
0062 1:
0063     subcc   %o0, %o5, %o0               ! hyper_flush_cache_user
0064     sta %g0, [%o0 + %g0] ASI_M_FLUSH_USER
0065     sta %g0, [%o0 + %o1] ASI_M_FLUSH_USER
0066     sta %g0, [%o0 + %g1] ASI_M_FLUSH_USER
0067     sta %g0, [%o0 + %g2] ASI_M_FLUSH_USER
0068     sta %g0, [%o0 + %g3] ASI_M_FLUSH_USER
0069     sta %g0, [%o0 + %g4] ASI_M_FLUSH_USER
0070     sta %g0, [%o0 + %g5] ASI_M_FLUSH_USER
0071     bne 1b
0072      sta    %g0, [%o0 + %o4] ASI_M_FLUSH_USER
0073 hypersparc_flush_cache_mm_out:
0074     retl
0075      nop
0076
0077     /* The things we do for performance... */
0078 hypersparc_flush_cache_range:
0079     ld  [%o0 + VMA_VM_MM], %o0
0080 #ifndef CONFIG_SMP
0081     ld  [%o0 + AOFF_mm_context], %g1
0082     cmp %g1, -1
0083     be  hypersparc_flush_cache_range_out
0084 #endif
0085     WINDOW_FLUSH(%g4, %g5)
0086
0087     sethi   %hi(vac_line_size), %g1
0088     ld  [%g1 + %lo(vac_line_size)], %o4
0089     sethi   %hi(vac_cache_size), %g2
0090     ld  [%g2 + %lo(vac_cache_size)], %o3
0091
0092     /* Here comes the fun part... */
0093     add %o2, (PAGE_SIZE - 1), %o2
0094     andn    %o1, (PAGE_SIZE - 1), %o1
0095     add %o4, %o4, %o5
0096     andn    %o2, (PAGE_SIZE - 1), %o2
0097     add %o4, %o5, %g1
0098     sub %o2, %o1, %g4
0099     add %o4, %g1, %g2
0100     sll %o3, 2, %g5
0101     add %o4, %g2, %g3
0102     cmp %g4, %g5
0103     add %o4, %g3, %g4
0104     blu 0f
0105      add    %o4, %g4, %g5
0106     add %o4, %g5, %g7
0107
0108     /* Flush entire user space, believe it or not this is quicker
0109      * than page at a time flushings for range > (cache_size<<2).
0110      */
0111 1:
0112     subcc   %o3, %g7, %o3
0113     sta %g0, [%o3 + %g0] ASI_M_FLUSH_USER
0114     sta %g0, [%o3 + %o4] ASI_M_FLUSH_USER
0115     sta %g0, [%o3 + %o5] ASI_M_FLUSH_USER
0116     sta %g0, [%o3 + %g1] ASI_M_FLUSH_USER
0117     sta %g0, [%o3 + %g2] ASI_M_FLUSH_USER
0118     sta %g0, [%o3 + %g3] ASI_M_FLUSH_USER
0119     sta %g0, [%o3 + %g4] ASI_M_FLUSH_USER
0120     bne 1b
0121      sta    %g0, [%o3 + %g5] ASI_M_FLUSH_USER
0122     retl
0123      nop
0124
0125     /* Below our threshold, flush one page at a time. */
0126 0:
0127     ld  [%o0 + AOFF_mm_context], %o0
0128     mov SRMMU_CTX_REG, %g7
0129     lda [%g7] ASI_M_MMUREGS, %o3
0130     sta %o0, [%g7] ASI_M_MMUREGS
0131     add %o2, -PAGE_SIZE, %o0
0132 1:
0133     or  %o0, 0x400, %g7
0134     lda [%g7] ASI_M_FLUSH_PROBE, %g7
0135     orcc    %g7, 0, %g0
0136     be,a    3f
0137      mov    %o0, %o2
0138     add %o4, %g5, %g7
0139 2:
0140     sub %o2, %g7, %o2
0141     sta %g0, [%o2 + %g0] ASI_M_FLUSH_PAGE
0142     sta %g0, [%o2 + %o4] ASI_M_FLUSH_PAGE
0143     sta %g0, [%o2 + %o5] ASI_M_FLUSH_PAGE
0144     sta %g0, [%o2 + %g1] ASI_M_FLUSH_PAGE
0145     sta %g0, [%o2 + %g2] ASI_M_FLUSH_PAGE
0146     sta %g0, [%o2 + %g3] ASI_M_FLUSH_PAGE
0147     andcc   %o2, 0xffc, %g0
0148     sta %g0, [%o2 + %g4] ASI_M_FLUSH_PAGE
0149     bne 2b
0150      sta    %g0, [%o2 + %g5] ASI_M_FLUSH_PAGE
0151 3:
0152     cmp %o2, %o1
0153     bne 1b
0154      add    %o2, -PAGE_SIZE, %o0
0155     mov SRMMU_FAULT_STATUS, %g5
0156     lda [%g5] ASI_M_MMUREGS, %g0
0157     mov SRMMU_CTX_REG, %g7
0158     sta %o3, [%g7] ASI_M_MMUREGS
0159 hypersparc_flush_cache_range_out:
0160     retl
0161      nop
0162
0163     /* HyperSparc requires a valid mapping where we are about to flush
0164      * in order to check for a physical tag match during the flush.
0165      */
0166     /* Verified, my ass... */
0167 hypersparc_flush_cache_page:
0168     ld  [%o0 + VMA_VM_MM], %o0
0169     ld  [%o0 + AOFF_mm_context], %g2
0170 #ifndef CONFIG_SMP
0171     cmp %g2, -1
0172     be  hypersparc_flush_cache_page_out
0173 #endif
0174     WINDOW_FLUSH(%g4, %g5)
0175
0176     sethi   %hi(vac_line_size), %g1
0177     ld  [%g1 + %lo(vac_line_size)], %o4
0178     mov SRMMU_CTX_REG, %o3
0179     andn    %o1, (PAGE_SIZE - 1), %o1
0180     lda [%o3] ASI_M_MMUREGS, %o2
0181     sta %g2, [%o3] ASI_M_MMUREGS
0182     or  %o1, 0x400, %o5
0183     lda [%o5] ASI_M_FLUSH_PROBE, %g1
0184     orcc    %g0, %g1, %g0
0185     be  2f
0186      add    %o4, %o4, %o5
0187     sub %o1, -PAGE_SIZE, %o1
0188     add %o4, %o5, %g1
0189     add %o4, %g1, %g2
0190     add %o4, %g2, %g3
0191     add %o4, %g3, %g4
0192     add %o4, %g4, %g5
0193     add %o4, %g5, %g7
0194
0195     /* BLAMMO! */
0196 1:
0197     sub %o1, %g7, %o1
0198     sta %g0, [%o1 + %g0] ASI_M_FLUSH_PAGE
0199     sta %g0, [%o1 + %o4] ASI_M_FLUSH_PAGE
0200     sta %g0, [%o1 + %o5] ASI_M_FLUSH_PAGE
0201     sta %g0, [%o1 + %g1] ASI_M_FLUSH_PAGE
0202     sta %g0, [%o1 + %g2] ASI_M_FLUSH_PAGE
0203     sta %g0, [%o1 + %g3] ASI_M_FLUSH_PAGE
0204     andcc   %o1, 0xffc, %g0
0205     sta %g0, [%o1 + %g4] ASI_M_FLUSH_PAGE
0206     bne 1b
0207      sta    %g0, [%o1 + %g5] ASI_M_FLUSH_PAGE
0208 2:
0209     mov SRMMU_FAULT_STATUS, %g7
0210     mov SRMMU_CTX_REG, %g4
0211     lda [%g7] ASI_M_MMUREGS, %g0
0212     sta %o2, [%g4] ASI_M_MMUREGS
0213 hypersparc_flush_cache_page_out:
0214     retl
0215      nop
0216
0217 hypersparc_flush_sig_insns:
0218     flush   %o1
0219     retl
0220      flush  %o1 + 4
0221
0222     /* HyperSparc is copy-back. */
0223 hypersparc_flush_page_to_ram:
0224     sethi   %hi(vac_line_size), %g1
0225     ld  [%g1 + %lo(vac_line_size)], %o4
0226     andn    %o0, (PAGE_SIZE - 1), %o0
0227     add %o4, %o4, %o5
0228     or  %o0, 0x400, %g7
0229     lda [%g7] ASI_M_FLUSH_PROBE, %g5
0230     add %o4, %o5, %g1
0231     orcc    %g5, 0, %g0
0232     be  2f
0233      add    %o4, %g1, %g2
0234     add %o4, %g2, %g3
0235     sub %o0, -PAGE_SIZE, %o0
0236     add %o4, %g3, %g4
0237     add %o4, %g4, %g5
0238     add %o4, %g5, %g7
0239
0240     /* BLAMMO! */
0241 1:
0242     sub %o0, %g7, %o0
0243     sta %g0, [%o0 + %g0] ASI_M_FLUSH_PAGE
0244     sta %g0, [%o0 + %o4] ASI_M_FLUSH_PAGE
0245     sta %g0, [%o0 + %o5] ASI_M_FLUSH_PAGE
0246     sta %g0, [%o0 + %g1] ASI_M_FLUSH_PAGE
0247     sta %g0, [%o0 + %g2] ASI_M_FLUSH_PAGE
0248     sta %g0, [%o0 + %g3] ASI_M_FLUSH_PAGE
0249     andcc   %o0, 0xffc, %g0
0250     sta %g0, [%o0 + %g4] ASI_M_FLUSH_PAGE
0251     bne 1b
0252      sta    %g0, [%o0 + %g5] ASI_M_FLUSH_PAGE
0253 2:
0254     mov SRMMU_FAULT_STATUS, %g1
0255     retl
0256      lda    [%g1] ASI_M_MMUREGS, %g0
0257
0258     /* HyperSparc is IO cache coherent. */
0259 hypersparc_flush_page_for_dma:
0260     retl
0261      nop
0262
0263     /* It was noted that at boot time a TLB flush all in a delay slot
0264      * can deliver an illegal instruction to the processor if the timing
0265      * is just right...
0266      */
0267 hypersparc_flush_tlb_all:
0268     mov 0x400, %g1
0269     sta %g0, [%g1] ASI_M_FLUSH_PROBE
0270     retl
0271      nop
0272
0273 hypersparc_flush_tlb_mm:
0274     mov SRMMU_CTX_REG, %g1
0275     ld  [%o0 + AOFF_mm_context], %o1
0276     lda [%g1] ASI_M_MMUREGS, %g5
0277 #ifndef CONFIG_SMP
0278     cmp %o1, -1
0279     be  hypersparc_flush_tlb_mm_out
0280 #endif
0281      mov    0x300, %g2
0282     sta %o1, [%g1] ASI_M_MMUREGS
0283     sta %g0, [%g2] ASI_M_FLUSH_PROBE
0284 hypersparc_flush_tlb_mm_out:
0285     retl
0286      sta    %g5, [%g1] ASI_M_MMUREGS
0287
0288 hypersparc_flush_tlb_range:
0289     ld  [%o0 + VMA_VM_MM], %o0
0290     mov SRMMU_CTX_REG, %g1
0291     ld  [%o0 + AOFF_mm_context], %o3
0292     lda [%g1] ASI_M_MMUREGS, %g5
0293 #ifndef CONFIG_SMP
0294     cmp %o3, -1
0295     be  hypersparc_flush_tlb_range_out
0296 #endif
0297      sethi  %hi(~((1 << PGDIR_SHIFT) - 1)), %o4
0298     sta %o3, [%g1] ASI_M_MMUREGS
0299     and %o1, %o4, %o1
0300     add %o1, 0x200, %o1
0301     sta %g0, [%o1] ASI_M_FLUSH_PROBE
0302 1:
0303     sub %o1, %o4, %o1
0304     cmp %o1, %o2
0305     blu,a   1b
0306      sta    %g0, [%o1] ASI_M_FLUSH_PROBE
0307 hypersparc_flush_tlb_range_out:
0308     retl
0309      sta    %g5, [%g1] ASI_M_MMUREGS
0310
0311 hypersparc_flush_tlb_page:
0312     ld  [%o0 + VMA_VM_MM], %o0
0313     mov SRMMU_CTX_REG, %g1
0314     ld  [%o0 + AOFF_mm_context], %o3
0315     andn    %o1, (PAGE_SIZE - 1), %o1
0316 #ifndef CONFIG_SMP
0317     cmp %o3, -1
0318     be  hypersparc_flush_tlb_page_out
0319 #endif
0320      lda    [%g1] ASI_M_MMUREGS, %g5
0321     sta %o3, [%g1] ASI_M_MMUREGS
0322     sta %g0, [%o1] ASI_M_FLUSH_PROBE
0323 hypersparc_flush_tlb_page_out:
0324     retl
0325      sta    %g5, [%g1] ASI_M_MMUREGS
0326
0327     __INIT
0328
0329     /* High speed page clear/copy. */
0330 hypersparc_bzero_1page:
0331 /* NOTE: This routine has to be shorter than 40insns --jj */
0332     clr %g1
0333     mov 32, %g2
0334     mov 64, %g3
0335     mov 96, %g4
0336     mov 128, %g5
0337     mov 160, %g7
0338     mov 192, %o2
0339     mov 224, %o3
0340     mov 16, %o1
0341 1:
0342     stda    %g0, [%o0 + %g0] ASI_M_BFILL
0343     stda    %g0, [%o0 + %g2] ASI_M_BFILL
0344     stda    %g0, [%o0 + %g3] ASI_M_BFILL
0345     stda    %g0, [%o0 + %g4] ASI_M_BFILL
0346     stda    %g0, [%o0 + %g5] ASI_M_BFILL
0347     stda    %g0, [%o0 + %g7] ASI_M_BFILL
0348     stda    %g0, [%o0 + %o2] ASI_M_BFILL
0349     stda    %g0, [%o0 + %o3] ASI_M_BFILL
0350     subcc   %o1, 1, %o1
0351     bne 1b
0352      add    %o0, 256, %o0
0353
0354     retl
0355      nop
0356
0357 hypersparc_copy_1page:
0358 /* NOTE: This routine has to be shorter than 70insns --jj */
0359     sub %o1, %o0, %o2       ! difference
0360     mov 16, %g1
0361 1:
0362     sta %o0, [%o0 + %o2] ASI_M_BCOPY
0363     add %o0, 32, %o0
0364     sta %o0, [%o0 + %o2] ASI_M_BCOPY
0365     add %o0, 32, %o0
0366     sta %o0, [%o0 + %o2] ASI_M_BCOPY
0367     add %o0, 32, %o0
0368     sta %o0, [%o0 + %o2] ASI_M_BCOPY
0369     add %o0, 32, %o0
0370     sta %o0, [%o0 + %o2] ASI_M_BCOPY
0371     add %o0, 32, %o0
0372     sta %o0, [%o0 + %o2] ASI_M_BCOPY
0373     add %o0, 32, %o0
0374     sta %o0, [%o0 + %o2] ASI_M_BCOPY
0375     add %o0, 32, %o0
0376     sta %o0, [%o0 + %o2] ASI_M_BCOPY
0377     subcc   %g1, 1, %g1
0378     bne 1b
0379      add    %o0, 32, %o0
0380
0381     retl
0382      nop
0383
0384     .globl  hypersparc_setup_blockops
0385 hypersparc_setup_blockops:
0386     sethi   %hi(bzero_1page), %o0
0387     or  %o0, %lo(bzero_1page), %o0
0388     sethi   %hi(hypersparc_bzero_1page), %o1
0389     or  %o1, %lo(hypersparc_bzero_1page), %o1
0390     sethi   %hi(hypersparc_copy_1page), %o2
0391     or  %o2, %lo(hypersparc_copy_1page), %o2
0392     ld  [%o1], %o4
0393 1:
0394     add %o1, 4, %o1
0395     st  %o4, [%o0]
0396     add %o0, 4, %o0
0397     cmp %o1, %o2
0398     bne 1b
0399      ld [%o1], %o4
0400     sethi   %hi(__copy_1page), %o0
0401     or  %o0, %lo(__copy_1page), %o0
0402     sethi   %hi(hypersparc_setup_blockops), %o2
0403     or  %o2, %lo(hypersparc_setup_blockops), %o2
0404     ld  [%o1], %o4
0405 1:
0406     add %o1, 4, %o1
0407     st  %o4, [%o0]
0408     add %o0, 4, %o0
0409     cmp %o1, %o2
0410     bne 1b
0411      ld [%o1], %o4
0412     sta %g0, [%g0] ASI_M_FLUSH_IWHOLE
0413     retl
0414      nop