powerpc/lib/pmem.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Copyright(c) 2017 IBM Corporation. All rights reserved.
0004  */
0005
0006 #include <linux/string.h>
0007 #include <linux/export.h>
0008 #include <linux/uaccess.h>
0009 #include <linux/libnvdimm.h>
0010
0011 #include <asm/cacheflush.h>
0012
0013 static inline void __clean_pmem_range(unsigned long start, unsigned long stop)
0014 {
0015     unsigned long shift = l1_dcache_shift();
0016     unsigned long bytes = l1_dcache_bytes();
0017     void *addr = (void *)(start & ~(bytes - 1));
0018     unsigned long size = stop - (unsigned long)addr + (bytes - 1);
0019     unsigned long i;
0020
0021     for (i = 0; i < size >> shift; i++, addr += bytes)
0022         asm volatile(PPC_DCBSTPS(%0, %1): :"i"(0), "r"(addr): "memory");
0023 }
0024
0025 static inline void __flush_pmem_range(unsigned long start, unsigned long stop)
0026 {
0027     unsigned long shift = l1_dcache_shift();
0028     unsigned long bytes = l1_dcache_bytes();
0029     void *addr = (void *)(start & ~(bytes - 1));
0030     unsigned long size = stop - (unsigned long)addr + (bytes - 1);
0031     unsigned long i;
0032
0033     for (i = 0; i < size >> shift; i++, addr += bytes)
0034         asm volatile(PPC_DCBFPS(%0, %1): :"i"(0), "r"(addr): "memory");
0035 }
0036
0037 static inline void clean_pmem_range(unsigned long start, unsigned long stop)
0038 {
0039     if (cpu_has_feature(CPU_FTR_ARCH_207S))
0040         return __clean_pmem_range(start, stop);
0041 }
0042
0043 static inline void flush_pmem_range(unsigned long start, unsigned long stop)
0044 {
0045     if (cpu_has_feature(CPU_FTR_ARCH_207S))
0046         return __flush_pmem_range(start, stop);
0047 }
0048
0049 /*
0050  * CONFIG_ARCH_HAS_PMEM_API symbols
0051  */
0052 void arch_wb_cache_pmem(void *addr, size_t size)
0053 {
0054     unsigned long start = (unsigned long) addr;
0055     clean_pmem_range(start, start + size);
0056 }
0057 EXPORT_SYMBOL_GPL(arch_wb_cache_pmem);
0058
0059 void arch_invalidate_pmem(void *addr, size_t size)
0060 {
0061     unsigned long start = (unsigned long) addr;
0062     flush_pmem_range(start, start + size);
0063 }
0064 EXPORT_SYMBOL_GPL(arch_invalidate_pmem);
0065
0066 /*
0067  * CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE symbols
0068  */
0069 long __copy_from_user_flushcache(void *dest, const void __user *src,
0070         unsigned size)
0071 {
0072     unsigned long copied, start = (unsigned long) dest;
0073
0074     copied = __copy_from_user(dest, src, size);
0075     clean_pmem_range(start, start + size);
0076
0077     return copied;
0078 }
0079
0080 void memcpy_flushcache(void *dest, const void *src, size_t size)
0081 {
0082     unsigned long start = (unsigned long) dest;
0083
0084     memcpy(dest, src, size);
0085     clean_pmem_range(start, start + size);
0086 }
0087 EXPORT_SYMBOL(memcpy_flushcache);
0088
0089 void memcpy_page_flushcache(char *to, struct page *page, size_t offset,
0090     size_t len)
0091 {
0092     memcpy_flushcache(to, page_to_virt(page) + offset, len);
0093 }
0094 EXPORT_SYMBOL(memcpy_page_flushcache);