include/asm/mte-kasan.h

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 /*
0003  * Copyright (C) 2020 ARM Ltd.
0004  */
0005 #ifndef __ASM_MTE_KASAN_H
0006 #define __ASM_MTE_KASAN_H
0007
0008 #include <asm/compiler.h>
0009 #include <asm/cputype.h>
0010 #include <asm/mte-def.h>
0011
0012 #ifndef __ASSEMBLY__
0013
0014 #include <linux/types.h>
0015
0016 #ifdef CONFIG_ARM64_MTE
0017
0018 /*
0019  * These functions are meant to be only used from KASAN runtime through
0020  * the arch_*() interface defined in asm/memory.h.
0021  * These functions don't include system_supports_mte() checks,
0022  * as KASAN only calls them when MTE is supported and enabled.
0023  */
0024
0025 static inline u8 mte_get_ptr_tag(void *ptr)
0026 {
0027     /* Note: The format of KASAN tags is 0xF<x> */
0028     u8 tag = 0xF0 | (u8)(((u64)(ptr)) >> MTE_TAG_SHIFT);
0029
0030     return tag;
0031 }
0032
0033 /* Get allocation tag for the address. */
0034 static inline u8 mte_get_mem_tag(void *addr)
0035 {
0036     asm(__MTE_PREAMBLE "ldg %0, [%0]"
0037         : "+r" (addr));
0038
0039     return mte_get_ptr_tag(addr);
0040 }
0041
0042 /* Generate a random tag. */
0043 static inline u8 mte_get_random_tag(void)
0044 {
0045     void *addr;
0046
0047     asm(__MTE_PREAMBLE "irg %0, %0"
0048         : "=r" (addr));
0049
0050     return mte_get_ptr_tag(addr);
0051 }
0052
0053 static inline u64 __stg_post(u64 p)
0054 {
0055     asm volatile(__MTE_PREAMBLE "stg %0, [%0], #16"
0056              : "+r"(p)
0057              :
0058              : "memory");
0059     return p;
0060 }
0061
0062 static inline u64 __stzg_post(u64 p)
0063 {
0064     asm volatile(__MTE_PREAMBLE "stzg %0, [%0], #16"
0065              : "+r"(p)
0066              :
0067              : "memory");
0068     return p;
0069 }
0070
0071 static inline void __dc_gva(u64 p)
0072 {
0073     asm volatile(__MTE_PREAMBLE "dc gva, %0" : : "r"(p) : "memory");
0074 }
0075
0076 static inline void __dc_gzva(u64 p)
0077 {
0078     asm volatile(__MTE_PREAMBLE "dc gzva, %0" : : "r"(p) : "memory");
0079 }
0080
0081 /*
0082  * Assign allocation tags for a region of memory based on the pointer tag.
0083  * Note: The address must be non-NULL and MTE_GRANULE_SIZE aligned and
0084  * size must be MTE_GRANULE_SIZE aligned.
0085  */
0086 static inline void mte_set_mem_tag_range(void *addr, size_t size, u8 tag,
0087                      bool init)
0088 {
0089     u64 curr, mask, dczid, dczid_bs, dczid_dzp, end1, end2, end3;
0090
0091     /* Read DC G(Z)VA block size from the system register. */
0092     dczid = read_cpuid(DCZID_EL0);
0093     dczid_bs = 4ul << (dczid & 0xf);
0094     dczid_dzp = (dczid >> 4) & 1;
0095
0096     curr = (u64)__tag_set(addr, tag);
0097     mask = dczid_bs - 1;
0098     /* STG/STZG up to the end of the first block. */
0099     end1 = curr | mask;
0100     end3 = curr + size;
0101     /* DC GVA / GZVA in [end1, end2) */
0102     end2 = end3 & ~mask;
0103
0104     /*
0105      * The following code uses STG on the first DC GVA block even if the
0106      * start address is aligned - it appears to be faster than an alignment
0107      * check + conditional branch. Also, if the range size is at least 2 DC
0108      * GVA blocks, the first two loops can use post-condition to save one
0109      * branch each.
0110      */
0111 #define SET_MEMTAG_RANGE(stg_post, dc_gva)      \
0112     do {                        \
0113         if (!dczid_dzp && size >= 2 * dczid_bs) {\
0114             do {                \
0115                 curr = stg_post(curr);  \
0116             } while (curr < end1);      \
0117                             \
0118             do {                \
0119                 dc_gva(curr);       \
0120                 curr += dczid_bs;   \
0121             } while (curr < end2);      \
0122         }                   \
0123                             \
0124         while (curr < end3)         \
0125             curr = stg_post(curr);      \
0126     } while (0)
0127
0128     if (init)
0129         SET_MEMTAG_RANGE(__stzg_post, __dc_gzva);
0130     else
0131         SET_MEMTAG_RANGE(__stg_post, __dc_gva);
0132 #undef SET_MEMTAG_RANGE
0133 }
0134
0135 void mte_enable_kernel_sync(void);
0136 void mte_enable_kernel_async(void);
0137 void mte_enable_kernel_asymm(void);
0138
0139 #else /* CONFIG_ARM64_MTE */
0140
0141 static inline u8 mte_get_ptr_tag(void *ptr)
0142 {
0143     return 0xFF;
0144 }
0145
0146 static inline u8 mte_get_mem_tag(void *addr)
0147 {
0148     return 0xFF;
0149 }
0150
0151 static inline u8 mte_get_random_tag(void)
0152 {
0153     return 0xFF;
0154 }
0155
0156 static inline void mte_set_mem_tag_range(void *addr, size_t size,
0157                         u8 tag, bool init)
0158 {
0159 }
0160
0161 static inline void mte_enable_kernel_sync(void)
0162 {
0163 }
0164
0165 static inline void mte_enable_kernel_async(void)
0166 {
0167 }
0168
0169 static inline void mte_enable_kernel_asymm(void)
0170 {
0171 }
0172
0173 #endif /* CONFIG_ARM64_MTE */
0174
0175 #endif /* __ASSEMBLY__ */
0176
0177 #endif /* __ASM_MTE_KASAN_H  */