arm/mm/cache-v6.S

0001 /* SPDX-License-Identifier: GPL-2.0-only */
0002 /*
0003  *  linux/arch/arm/mm/cache-v6.S
0004  *
0005  *  Copyright (C) 2001 Deep Blue Solutions Ltd.
0006  *
0007  *  This is the "shell" of the ARMv6 processor support.
0008  */
0009 #include <linux/linkage.h>
0010 #include <linux/init.h>
0011 #include <asm/assembler.h>
0012 #include <asm/errno.h>
0013 #include <asm/unwind.h>
0014
0015 #include "proc-macros.S"
0016
0017 #define HARVARD_CACHE
0018 #define CACHE_LINE_SIZE     32
0019 #define D_CACHE_LINE_SIZE   32
0020 #define BTB_FLUSH_SIZE      8
0021
0022 /*
0023  *  v6_flush_icache_all()
0024  *
0025  *  Flush the whole I-cache.
0026  *
0027  *  ARM1136 erratum 411920 - Invalidate Instruction Cache operation can fail.
0028  *  This erratum is present in 1136, 1156 and 1176. It does not affect the
0029  *  MPCore.
0030  *
0031  *  Registers:
0032  *  r0 - set to 0
0033  *  r1 - corrupted
0034  */
0035 ENTRY(v6_flush_icache_all)
0036     mov r0, #0
0037 #ifdef CONFIG_ARM_ERRATA_411920
0038     mrs r1, cpsr
0039     cpsid   ifa             @ disable interrupts
0040     mcr p15, 0, r0, c7, c5, 0       @ invalidate entire I-cache
0041     mcr p15, 0, r0, c7, c5, 0       @ invalidate entire I-cache
0042     mcr p15, 0, r0, c7, c5, 0       @ invalidate entire I-cache
0043     mcr p15, 0, r0, c7, c5, 0       @ invalidate entire I-cache
0044     msr cpsr_cx, r1         @ restore interrupts
0045     .rept   11              @ ARM Ltd recommends at least
0046     nop                 @ 11 NOPs
0047     .endr
0048 #else
0049     mcr p15, 0, r0, c7, c5, 0       @ invalidate I-cache
0050 #endif
0051     ret lr
0052 ENDPROC(v6_flush_icache_all)
0053
0054 /*
0055  *  v6_flush_cache_all()
0056  *
0057  *  Flush the entire cache.
0058  *
0059  *  It is assumed that:
0060  */
0061 ENTRY(v6_flush_kern_cache_all)
0062     mov r0, #0
0063 #ifdef HARVARD_CACHE
0064     mcr p15, 0, r0, c7, c14, 0      @ D cache clean+invalidate
0065 #ifndef CONFIG_ARM_ERRATA_411920
0066     mcr p15, 0, r0, c7, c5, 0       @ I+BTB cache invalidate
0067 #else
0068     b   v6_flush_icache_all
0069 #endif
0070 #else
0071     mcr p15, 0, r0, c7, c15, 0      @ Cache clean+invalidate
0072 #endif
0073     ret lr
0074
0075 /*
0076  *  v6_flush_cache_all()
0077  *
0078  *  Flush all TLB entries in a particular address space
0079  *
0080  *  - mm    - mm_struct describing address space
0081  */
0082 ENTRY(v6_flush_user_cache_all)
0083     /*FALLTHROUGH*/
0084
0085 /*
0086  *  v6_flush_cache_range(start, end, flags)
0087  *
0088  *  Flush a range of TLB entries in the specified address space.
0089  *
0090  *  - start - start address (may not be aligned)
0091  *  - end   - end address (exclusive, may not be aligned)
0092  *  - flags - vm_area_struct flags describing address space
0093  *
0094  *  It is assumed that:
0095  *  - we have a VIPT cache.
0096  */
0097 ENTRY(v6_flush_user_cache_range)
0098     ret lr
0099
0100 /*
0101  *  v6_coherent_kern_range(start,end)
0102  *
0103  *  Ensure that the I and D caches are coherent within specified
0104  *  region.  This is typically used when code has been written to
0105  *  a memory region, and will be executed.
0106  *
0107  *  - start   - virtual start address of region
0108  *  - end     - virtual end address of region
0109  *
0110  *  It is assumed that:
0111  *  - the Icache does not read data from the write buffer
0112  */
0113 ENTRY(v6_coherent_kern_range)
0114     /* FALLTHROUGH */
0115
0116 /*
0117  *  v6_coherent_user_range(start,end)
0118  *
0119  *  Ensure that the I and D caches are coherent within specified
0120  *  region.  This is typically used when code has been written to
0121  *  a memory region, and will be executed.
0122  *
0123  *  - start   - virtual start address of region
0124  *  - end     - virtual end address of region
0125  *
0126  *  It is assumed that:
0127  *  - the Icache does not read data from the write buffer
0128  */
0129 ENTRY(v6_coherent_user_range)
0130  UNWIND(.fnstart        )
0131 #ifdef HARVARD_CACHE
0132     bic r0, r0, #CACHE_LINE_SIZE - 1
0133 1:
0134  USER(  mcr p15, 0, r0, c7, c10, 1  )   @ clean D line
0135     add r0, r0, #CACHE_LINE_SIZE
0136     cmp r0, r1
0137     blo 1b
0138 #endif
0139     mov r0, #0
0140 #ifdef HARVARD_CACHE
0141     mcr p15, 0, r0, c7, c10, 4      @ drain write buffer
0142 #ifndef CONFIG_ARM_ERRATA_411920
0143     mcr p15, 0, r0, c7, c5, 0       @ I+BTB cache invalidate
0144 #else
0145     b   v6_flush_icache_all
0146 #endif
0147 #else
0148     mcr p15, 0, r0, c7, c5, 6       @ invalidate BTB
0149 #endif
0150     ret lr
0151
0152 /*
0153  * Fault handling for the cache operation above. If the virtual address in r0
0154  * isn't mapped, fail with -EFAULT.
0155  */
0156 9001:
0157     mov r0, #-EFAULT
0158     ret lr
0159  UNWIND(.fnend      )
0160 ENDPROC(v6_coherent_user_range)
0161 ENDPROC(v6_coherent_kern_range)
0162
0163 /*
0164  *  v6_flush_kern_dcache_area(void *addr, size_t size)
0165  *
0166  *  Ensure that the data held in the page kaddr is written back
0167  *  to the page in question.
0168  *
0169  *  - addr  - kernel address
0170  *  - size  - region size
0171  */
0172 ENTRY(v6_flush_kern_dcache_area)
0173     add r1, r0, r1
0174     bic r0, r0, #D_CACHE_LINE_SIZE - 1
0175 1:
0176 #ifdef HARVARD_CACHE
0177     mcr p15, 0, r0, c7, c14, 1      @ clean & invalidate D line
0178 #else
0179     mcr p15, 0, r0, c7, c15, 1      @ clean & invalidate unified line
0180 #endif
0181     add r0, r0, #D_CACHE_LINE_SIZE
0182     cmp r0, r1
0183     blo 1b
0184 #ifdef HARVARD_CACHE
0185     mov r0, #0
0186     mcr p15, 0, r0, c7, c10, 4
0187 #endif
0188     ret lr
0189
0190
0191 /*
0192  *  v6_dma_inv_range(start,end)
0193  *
0194  *  Invalidate the data cache within the specified region; we will
0195  *  be performing a DMA operation in this region and we want to
0196  *  purge old data in the cache.
0197  *
0198  *  - start   - virtual start address of region
0199  *  - end     - virtual end address of region
0200  */
0201 v6_dma_inv_range:
0202 #ifdef CONFIG_DMA_CACHE_RWFO
0203     ldrb    r2, [r0]            @ read for ownership
0204     strb    r2, [r0]            @ write for ownership
0205 #endif
0206     tst r0, #D_CACHE_LINE_SIZE - 1
0207     bic r0, r0, #D_CACHE_LINE_SIZE - 1
0208 #ifdef HARVARD_CACHE
0209     mcrne   p15, 0, r0, c7, c10, 1      @ clean D line
0210 #else
0211     mcrne   p15, 0, r0, c7, c11, 1      @ clean unified line
0212 #endif
0213     tst r1, #D_CACHE_LINE_SIZE - 1
0214 #ifdef CONFIG_DMA_CACHE_RWFO
0215     ldrbne  r2, [r1, #-1]           @ read for ownership
0216     strbne  r2, [r1, #-1]           @ write for ownership
0217 #endif
0218     bic r1, r1, #D_CACHE_LINE_SIZE - 1
0219 #ifdef HARVARD_CACHE
0220     mcrne   p15, 0, r1, c7, c14, 1      @ clean & invalidate D line
0221 #else
0222     mcrne   p15, 0, r1, c7, c15, 1      @ clean & invalidate unified line
0223 #endif
0224 1:
0225 #ifdef HARVARD_CACHE
0226     mcr p15, 0, r0, c7, c6, 1       @ invalidate D line
0227 #else
0228     mcr p15, 0, r0, c7, c7, 1       @ invalidate unified line
0229 #endif
0230     add r0, r0, #D_CACHE_LINE_SIZE
0231     cmp r0, r1
0232 #ifdef CONFIG_DMA_CACHE_RWFO
0233     ldrlo   r2, [r0]            @ read for ownership
0234     strlo   r2, [r0]            @ write for ownership
0235 #endif
0236     blo 1b
0237     mov r0, #0
0238     mcr p15, 0, r0, c7, c10, 4      @ drain write buffer
0239     ret lr
0240
0241 /*
0242  *  v6_dma_clean_range(start,end)
0243  *  - start   - virtual start address of region
0244  *  - end     - virtual end address of region
0245  */
0246 v6_dma_clean_range:
0247     bic r0, r0, #D_CACHE_LINE_SIZE - 1
0248 1:
0249 #ifdef CONFIG_DMA_CACHE_RWFO
0250     ldr r2, [r0]            @ read for ownership
0251 #endif
0252 #ifdef HARVARD_CACHE
0253     mcr p15, 0, r0, c7, c10, 1      @ clean D line
0254 #else
0255     mcr p15, 0, r0, c7, c11, 1      @ clean unified line
0256 #endif
0257     add r0, r0, #D_CACHE_LINE_SIZE
0258     cmp r0, r1
0259     blo 1b
0260     mov r0, #0
0261     mcr p15, 0, r0, c7, c10, 4      @ drain write buffer
0262     ret lr
0263
0264 /*
0265  *  v6_dma_flush_range(start,end)
0266  *  - start   - virtual start address of region
0267  *  - end     - virtual end address of region
0268  */
0269 ENTRY(v6_dma_flush_range)
0270 #ifdef CONFIG_DMA_CACHE_RWFO
0271     ldrb    r2, [r0]        @ read for ownership
0272     strb    r2, [r0]        @ write for ownership
0273 #endif
0274     bic r0, r0, #D_CACHE_LINE_SIZE - 1
0275 1:
0276 #ifdef HARVARD_CACHE
0277     mcr p15, 0, r0, c7, c14, 1      @ clean & invalidate D line
0278 #else
0279     mcr p15, 0, r0, c7, c15, 1      @ clean & invalidate line
0280 #endif
0281     add r0, r0, #D_CACHE_LINE_SIZE
0282     cmp r0, r1
0283 #ifdef CONFIG_DMA_CACHE_RWFO
0284     ldrblo  r2, [r0]            @ read for ownership
0285     strblo  r2, [r0]            @ write for ownership
0286 #endif
0287     blo 1b
0288     mov r0, #0
0289     mcr p15, 0, r0, c7, c10, 4      @ drain write buffer
0290     ret lr
0291
0292 /*
0293  *  dma_map_area(start, size, dir)
0294  *  - start - kernel virtual start address
0295  *  - size  - size of region
0296  *  - dir   - DMA direction
0297  */
0298 ENTRY(v6_dma_map_area)
0299     add r1, r1, r0
0300     teq r2, #DMA_FROM_DEVICE
0301     beq v6_dma_inv_range
0302 #ifndef CONFIG_DMA_CACHE_RWFO
0303     b   v6_dma_clean_range
0304 #else
0305     teq r2, #DMA_TO_DEVICE
0306     beq v6_dma_clean_range
0307     b   v6_dma_flush_range
0308 #endif
0309 ENDPROC(v6_dma_map_area)
0310
0311 /*
0312  *  dma_unmap_area(start, size, dir)
0313  *  - start - kernel virtual start address
0314  *  - size  - size of region
0315  *  - dir   - DMA direction
0316  */
0317 ENTRY(v6_dma_unmap_area)
0318 #ifndef CONFIG_DMA_CACHE_RWFO
0319     add r1, r1, r0
0320     teq r2, #DMA_TO_DEVICE
0321     bne v6_dma_inv_range
0322 #endif
0323     ret lr
0324 ENDPROC(v6_dma_unmap_area)
0325
0326     .globl  v6_flush_kern_cache_louis
0327     .equ    v6_flush_kern_cache_louis, v6_flush_kern_cache_all
0328
0329     __INITDATA
0330
0331     @ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S)
0332     define_cache_functions v6