include/linux/kasan.h

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 #ifndef _LINUX_KASAN_H
0003 #define _LINUX_KASAN_H
0004
0005 #include <linux/bug.h>
0006 #include <linux/kasan-enabled.h>
0007 #include <linux/kernel.h>
0008 #include <linux/static_key.h>
0009 #include <linux/types.h>
0010
0011 struct kmem_cache;
0012 struct page;
0013 struct slab;
0014 struct vm_struct;
0015 struct task_struct;
0016
0017 #ifdef CONFIG_KASAN
0018
0019 #include <linux/linkage.h>
0020 #include <asm/kasan.h>
0021
0022 #endif
0023
0024 typedef unsigned int __bitwise kasan_vmalloc_flags_t;
0025
0026 #define KASAN_VMALLOC_NONE      ((__force kasan_vmalloc_flags_t)0x00u)
0027 #define KASAN_VMALLOC_INIT      ((__force kasan_vmalloc_flags_t)0x01u)
0028 #define KASAN_VMALLOC_VM_ALLOC      ((__force kasan_vmalloc_flags_t)0x02u)
0029 #define KASAN_VMALLOC_PROT_NORMAL   ((__force kasan_vmalloc_flags_t)0x04u)
0030
0031 #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
0032
0033 #include <linux/pgtable.h>
0034
0035 /* Software KASAN implementations use shadow memory. */
0036
0037 #ifdef CONFIG_KASAN_SW_TAGS
0038 /* This matches KASAN_TAG_INVALID. */
0039 #define KASAN_SHADOW_INIT 0xFE
0040 #else
0041 #define KASAN_SHADOW_INIT 0
0042 #endif
0043
0044 #ifndef PTE_HWTABLE_PTRS
0045 #define PTE_HWTABLE_PTRS 0
0046 #endif
0047
0048 extern unsigned char kasan_early_shadow_page[PAGE_SIZE];
0049 extern pte_t kasan_early_shadow_pte[MAX_PTRS_PER_PTE + PTE_HWTABLE_PTRS];
0050 extern pmd_t kasan_early_shadow_pmd[MAX_PTRS_PER_PMD];
0051 extern pud_t kasan_early_shadow_pud[MAX_PTRS_PER_PUD];
0052 extern p4d_t kasan_early_shadow_p4d[MAX_PTRS_PER_P4D];
0053
0054 int kasan_populate_early_shadow(const void *shadow_start,
0055                 const void *shadow_end);
0056
0057 static inline void *kasan_mem_to_shadow(const void *addr)
0058 {
0059     return (void *)((unsigned long)addr >> KASAN_SHADOW_SCALE_SHIFT)
0060         + KASAN_SHADOW_OFFSET;
0061 }
0062
0063 int kasan_add_zero_shadow(void *start, unsigned long size);
0064 void kasan_remove_zero_shadow(void *start, unsigned long size);
0065
0066 /* Enable reporting bugs after kasan_disable_current() */
0067 extern void kasan_enable_current(void);
0068
0069 /* Disable reporting bugs for current task */
0070 extern void kasan_disable_current(void);
0071
0072 #else /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */
0073
0074 static inline int kasan_add_zero_shadow(void *start, unsigned long size)
0075 {
0076     return 0;
0077 }
0078 static inline void kasan_remove_zero_shadow(void *start,
0079                     unsigned long size)
0080 {}
0081
0082 static inline void kasan_enable_current(void) {}
0083 static inline void kasan_disable_current(void) {}
0084
0085 #endif /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */
0086
0087 #ifdef CONFIG_KASAN_HW_TAGS
0088
0089 #else /* CONFIG_KASAN_HW_TAGS */
0090
0091 #endif /* CONFIG_KASAN_HW_TAGS */
0092
0093 static inline bool kasan_has_integrated_init(void)
0094 {
0095     return kasan_hw_tags_enabled();
0096 }
0097
0098 #ifdef CONFIG_KASAN
0099
0100 struct kasan_cache {
0101     int alloc_meta_offset;
0102     int free_meta_offset;
0103     bool is_kmalloc;
0104 };
0105
0106 slab_flags_t __kasan_never_merge(void);
0107 static __always_inline slab_flags_t kasan_never_merge(void)
0108 {
0109     if (kasan_enabled())
0110         return __kasan_never_merge();
0111     return 0;
0112 }
0113
0114 void __kasan_unpoison_range(const void *addr, size_t size);
0115 static __always_inline void kasan_unpoison_range(const void *addr, size_t size)
0116 {
0117     if (kasan_enabled())
0118         __kasan_unpoison_range(addr, size);
0119 }
0120
0121 void __kasan_poison_pages(struct page *page, unsigned int order, bool init);
0122 static __always_inline void kasan_poison_pages(struct page *page,
0123                         unsigned int order, bool init)
0124 {
0125     if (kasan_enabled())
0126         __kasan_poison_pages(page, order, init);
0127 }
0128
0129 void __kasan_unpoison_pages(struct page *page, unsigned int order, bool init);
0130 static __always_inline void kasan_unpoison_pages(struct page *page,
0131                          unsigned int order, bool init)
0132 {
0133     if (kasan_enabled())
0134         __kasan_unpoison_pages(page, order, init);
0135 }
0136
0137 void __kasan_cache_create(struct kmem_cache *cache, unsigned int *size,
0138                 slab_flags_t *flags);
0139 static __always_inline void kasan_cache_create(struct kmem_cache *cache,
0140                 unsigned int *size, slab_flags_t *flags)
0141 {
0142     if (kasan_enabled())
0143         __kasan_cache_create(cache, size, flags);
0144 }
0145
0146 void __kasan_cache_create_kmalloc(struct kmem_cache *cache);
0147 static __always_inline void kasan_cache_create_kmalloc(struct kmem_cache *cache)
0148 {
0149     if (kasan_enabled())
0150         __kasan_cache_create_kmalloc(cache);
0151 }
0152
0153 size_t __kasan_metadata_size(struct kmem_cache *cache);
0154 static __always_inline size_t kasan_metadata_size(struct kmem_cache *cache)
0155 {
0156     if (kasan_enabled())
0157         return __kasan_metadata_size(cache);
0158     return 0;
0159 }
0160
0161 void __kasan_poison_slab(struct slab *slab);
0162 static __always_inline void kasan_poison_slab(struct slab *slab)
0163 {
0164     if (kasan_enabled())
0165         __kasan_poison_slab(slab);
0166 }
0167
0168 void __kasan_unpoison_object_data(struct kmem_cache *cache, void *object);
0169 static __always_inline void kasan_unpoison_object_data(struct kmem_cache *cache,
0170                             void *object)
0171 {
0172     if (kasan_enabled())
0173         __kasan_unpoison_object_data(cache, object);
0174 }
0175
0176 void __kasan_poison_object_data(struct kmem_cache *cache, void *object);
0177 static __always_inline void kasan_poison_object_data(struct kmem_cache *cache,
0178                             void *object)
0179 {
0180     if (kasan_enabled())
0181         __kasan_poison_object_data(cache, object);
0182 }
0183
0184 void * __must_check __kasan_init_slab_obj(struct kmem_cache *cache,
0185                       const void *object);
0186 static __always_inline void * __must_check kasan_init_slab_obj(
0187                 struct kmem_cache *cache, const void *object)
0188 {
0189     if (kasan_enabled())
0190         return __kasan_init_slab_obj(cache, object);
0191     return (void *)object;
0192 }
0193
0194 bool __kasan_slab_free(struct kmem_cache *s, void *object,
0195             unsigned long ip, bool init);
0196 static __always_inline bool kasan_slab_free(struct kmem_cache *s,
0197                         void *object, bool init)
0198 {
0199     if (kasan_enabled())
0200         return __kasan_slab_free(s, object, _RET_IP_, init);
0201     return false;
0202 }
0203
0204 void __kasan_kfree_large(void *ptr, unsigned long ip);
0205 static __always_inline void kasan_kfree_large(void *ptr)
0206 {
0207     if (kasan_enabled())
0208         __kasan_kfree_large(ptr, _RET_IP_);
0209 }
0210
0211 void __kasan_slab_free_mempool(void *ptr, unsigned long ip);
0212 static __always_inline void kasan_slab_free_mempool(void *ptr)
0213 {
0214     if (kasan_enabled())
0215         __kasan_slab_free_mempool(ptr, _RET_IP_);
0216 }
0217
0218 void * __must_check __kasan_slab_alloc(struct kmem_cache *s,
0219                        void *object, gfp_t flags, bool init);
0220 static __always_inline void * __must_check kasan_slab_alloc(
0221         struct kmem_cache *s, void *object, gfp_t flags, bool init)
0222 {
0223     if (kasan_enabled())
0224         return __kasan_slab_alloc(s, object, flags, init);
0225     return object;
0226 }
0227
0228 void * __must_check __kasan_kmalloc(struct kmem_cache *s, const void *object,
0229                     size_t size, gfp_t flags);
0230 static __always_inline void * __must_check kasan_kmalloc(struct kmem_cache *s,
0231                 const void *object, size_t size, gfp_t flags)
0232 {
0233     if (kasan_enabled())
0234         return __kasan_kmalloc(s, object, size, flags);
0235     return (void *)object;
0236 }
0237
0238 void * __must_check __kasan_kmalloc_large(const void *ptr,
0239                       size_t size, gfp_t flags);
0240 static __always_inline void * __must_check kasan_kmalloc_large(const void *ptr,
0241                               size_t size, gfp_t flags)
0242 {
0243     if (kasan_enabled())
0244         return __kasan_kmalloc_large(ptr, size, flags);
0245     return (void *)ptr;
0246 }
0247
0248 void * __must_check __kasan_krealloc(const void *object,
0249                      size_t new_size, gfp_t flags);
0250 static __always_inline void * __must_check kasan_krealloc(const void *object,
0251                          size_t new_size, gfp_t flags)
0252 {
0253     if (kasan_enabled())
0254         return __kasan_krealloc(object, new_size, flags);
0255     return (void *)object;
0256 }
0257
0258 /*
0259  * Unlike kasan_check_read/write(), kasan_check_byte() is performed even for
0260  * the hardware tag-based mode that doesn't rely on compiler instrumentation.
0261  */
0262 bool __kasan_check_byte(const void *addr, unsigned long ip);
0263 static __always_inline bool kasan_check_byte(const void *addr)
0264 {
0265     if (kasan_enabled())
0266         return __kasan_check_byte(addr, _RET_IP_);
0267     return true;
0268 }
0269
0270 #else /* CONFIG_KASAN */
0271
0272 static inline slab_flags_t kasan_never_merge(void)
0273 {
0274     return 0;
0275 }
0276 static inline void kasan_unpoison_range(const void *address, size_t size) {}
0277 static inline void kasan_poison_pages(struct page *page, unsigned int order,
0278                       bool init) {}
0279 static inline void kasan_unpoison_pages(struct page *page, unsigned int order,
0280                     bool init) {}
0281 static inline void kasan_cache_create(struct kmem_cache *cache,
0282                       unsigned int *size,
0283                       slab_flags_t *flags) {}
0284 static inline void kasan_cache_create_kmalloc(struct kmem_cache *cache) {}
0285 static inline size_t kasan_metadata_size(struct kmem_cache *cache) { return 0; }
0286 static inline void kasan_poison_slab(struct slab *slab) {}
0287 static inline void kasan_unpoison_object_data(struct kmem_cache *cache,
0288                     void *object) {}
0289 static inline void kasan_poison_object_data(struct kmem_cache *cache,
0290                     void *object) {}
0291 static inline void *kasan_init_slab_obj(struct kmem_cache *cache,
0292                 const void *object)
0293 {
0294     return (void *)object;
0295 }
0296 static inline bool kasan_slab_free(struct kmem_cache *s, void *object, bool init)
0297 {
0298     return false;
0299 }
0300 static inline void kasan_kfree_large(void *ptr) {}
0301 static inline void kasan_slab_free_mempool(void *ptr) {}
0302 static inline void *kasan_slab_alloc(struct kmem_cache *s, void *object,
0303                    gfp_t flags, bool init)
0304 {
0305     return object;
0306 }
0307 static inline void *kasan_kmalloc(struct kmem_cache *s, const void *object,
0308                 size_t size, gfp_t flags)
0309 {
0310     return (void *)object;
0311 }
0312 static inline void *kasan_kmalloc_large(const void *ptr, size_t size, gfp_t flags)
0313 {
0314     return (void *)ptr;
0315 }
0316 static inline void *kasan_krealloc(const void *object, size_t new_size,
0317                  gfp_t flags)
0318 {
0319     return (void *)object;
0320 }
0321 static inline bool kasan_check_byte(const void *address)
0322 {
0323     return true;
0324 }
0325
0326 #endif /* CONFIG_KASAN */
0327
0328 #if defined(CONFIG_KASAN) && defined(CONFIG_KASAN_STACK)
0329 void kasan_unpoison_task_stack(struct task_struct *task);
0330 #else
0331 static inline void kasan_unpoison_task_stack(struct task_struct *task) {}
0332 #endif
0333
0334 #ifdef CONFIG_KASAN_GENERIC
0335
0336 void kasan_cache_shrink(struct kmem_cache *cache);
0337 void kasan_cache_shutdown(struct kmem_cache *cache);
0338 void kasan_record_aux_stack(void *ptr);
0339 void kasan_record_aux_stack_noalloc(void *ptr);
0340
0341 #else /* CONFIG_KASAN_GENERIC */
0342
0343 static inline void kasan_cache_shrink(struct kmem_cache *cache) {}
0344 static inline void kasan_cache_shutdown(struct kmem_cache *cache) {}
0345 static inline void kasan_record_aux_stack(void *ptr) {}
0346 static inline void kasan_record_aux_stack_noalloc(void *ptr) {}
0347
0348 #endif /* CONFIG_KASAN_GENERIC */
0349
0350 #if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS)
0351
0352 static inline void *kasan_reset_tag(const void *addr)
0353 {
0354     return (void *)arch_kasan_reset_tag(addr);
0355 }
0356
0357 /**
0358  * kasan_report - print a report about a bad memory access detected by KASAN
0359  * @addr: address of the bad access
0360  * @size: size of the bad access
0361  * @is_write: whether the bad access is a write or a read
0362  * @ip: instruction pointer for the accessibility check or the bad access itself
0363  */
0364 bool kasan_report(unsigned long addr, size_t size,
0365         bool is_write, unsigned long ip);
0366
0367 #else /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS */
0368
0369 static inline void *kasan_reset_tag(const void *addr)
0370 {
0371     return (void *)addr;
0372 }
0373
0374 #endif /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS*/
0375
0376 #ifdef CONFIG_KASAN_HW_TAGS
0377
0378 void kasan_report_async(void);
0379
0380 #endif /* CONFIG_KASAN_HW_TAGS */
0381
0382 #ifdef CONFIG_KASAN_SW_TAGS
0383 void __init kasan_init_sw_tags(void);
0384 #else
0385 static inline void kasan_init_sw_tags(void) { }
0386 #endif
0387
0388 #ifdef CONFIG_KASAN_HW_TAGS
0389 void kasan_init_hw_tags_cpu(void);
0390 void __init kasan_init_hw_tags(void);
0391 #else
0392 static inline void kasan_init_hw_tags_cpu(void) { }
0393 static inline void kasan_init_hw_tags(void) { }
0394 #endif
0395
0396 #ifdef CONFIG_KASAN_VMALLOC
0397
0398 #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
0399
0400 void kasan_populate_early_vm_area_shadow(void *start, unsigned long size);
0401 int kasan_populate_vmalloc(unsigned long addr, unsigned long size);
0402 void kasan_release_vmalloc(unsigned long start, unsigned long end,
0403                unsigned long free_region_start,
0404                unsigned long free_region_end);
0405
0406 #else /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */
0407
0408 static inline void kasan_populate_early_vm_area_shadow(void *start,
0409                                unsigned long size)
0410 { }
0411 static inline int kasan_populate_vmalloc(unsigned long start,
0412                     unsigned long size)
0413 {
0414     return 0;
0415 }
0416 static inline void kasan_release_vmalloc(unsigned long start,
0417                      unsigned long end,
0418                      unsigned long free_region_start,
0419                      unsigned long free_region_end) { }
0420
0421 #endif /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */
0422
0423 void *__kasan_unpoison_vmalloc(const void *start, unsigned long size,
0424                    kasan_vmalloc_flags_t flags);
0425 static __always_inline void *kasan_unpoison_vmalloc(const void *start,
0426                         unsigned long size,
0427                         kasan_vmalloc_flags_t flags)
0428 {
0429     if (kasan_enabled())
0430         return __kasan_unpoison_vmalloc(start, size, flags);
0431     return (void *)start;
0432 }
0433
0434 void __kasan_poison_vmalloc(const void *start, unsigned long size);
0435 static __always_inline void kasan_poison_vmalloc(const void *start,
0436                          unsigned long size)
0437 {
0438     if (kasan_enabled())
0439         __kasan_poison_vmalloc(start, size);
0440 }
0441
0442 #else /* CONFIG_KASAN_VMALLOC */
0443
0444 static inline void kasan_populate_early_vm_area_shadow(void *start,
0445                                unsigned long size) { }
0446 static inline int kasan_populate_vmalloc(unsigned long start,
0447                     unsigned long size)
0448 {
0449     return 0;
0450 }
0451 static inline void kasan_release_vmalloc(unsigned long start,
0452                      unsigned long end,
0453                      unsigned long free_region_start,
0454                      unsigned long free_region_end) { }
0455
0456 static inline void *kasan_unpoison_vmalloc(const void *start,
0457                        unsigned long size,
0458                        kasan_vmalloc_flags_t flags)
0459 {
0460     return (void *)start;
0461 }
0462 static inline void kasan_poison_vmalloc(const void *start, unsigned long size)
0463 { }
0464
0465 #endif /* CONFIG_KASAN_VMALLOC */
0466
0467 #if (defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)) && \
0468         !defined(CONFIG_KASAN_VMALLOC)
0469
0470 /*
0471  * These functions allocate and free shadow memory for kernel modules.
0472  * They are only required when KASAN_VMALLOC is not supported, as otherwise
0473  * shadow memory is allocated by the generic vmalloc handlers.
0474  */
0475 int kasan_alloc_module_shadow(void *addr, size_t size, gfp_t gfp_mask);
0476 void kasan_free_module_shadow(const struct vm_struct *vm);
0477
0478 #else /* (CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS) && !CONFIG_KASAN_VMALLOC */
0479
0480 static inline int kasan_alloc_module_shadow(void *addr, size_t size, gfp_t gfp_mask) { return 0; }
0481 static inline void kasan_free_module_shadow(const struct vm_struct *vm) {}
0482
0483 #endif /* (CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS) && !CONFIG_KASAN_VMALLOC */
0484
0485 #ifdef CONFIG_KASAN_INLINE
0486 void kasan_non_canonical_hook(unsigned long addr);
0487 #else /* CONFIG_KASAN_INLINE */
0488 static inline void kasan_non_canonical_hook(unsigned long addr) { }
0489 #endif /* CONFIG_KASAN_INLINE */
0490
0491 #endif /* LINUX_KASAN_H */