mm/kfence/kfence.h

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 /*
0003  * Kernel Electric-Fence (KFENCE). For more info please see
0004  * Documentation/dev-tools/kfence.rst.
0005  *
0006  * Copyright (C) 2020, Google LLC.
0007  */
0008
0009 #ifndef MM_KFENCE_KFENCE_H
0010 #define MM_KFENCE_KFENCE_H
0011
0012 #include <linux/mm.h>
0013 #include <linux/slab.h>
0014 #include <linux/spinlock.h>
0015 #include <linux/types.h>
0016
0017 #include "../slab.h" /* for struct kmem_cache */
0018
0019 /*
0020  * Get the canary byte pattern for @addr. Use a pattern that varies based on the
0021  * lower 3 bits of the address, to detect memory corruptions with higher
0022  * probability, where similar constants are used.
0023  */
0024 #define KFENCE_CANARY_PATTERN(addr) ((u8)0xaa ^ (u8)((unsigned long)(addr) & 0x7))
0025
0026 /* Maximum stack depth for reports. */
0027 #define KFENCE_STACK_DEPTH 64
0028
0029 /* KFENCE object states. */
0030 enum kfence_object_state {
0031     KFENCE_OBJECT_UNUSED,       /* Object is unused. */
0032     KFENCE_OBJECT_ALLOCATED,    /* Object is currently allocated. */
0033     KFENCE_OBJECT_FREED,        /* Object was allocated, and then freed. */
0034 };
0035
0036 /* Alloc/free tracking information. */
0037 struct kfence_track {
0038     pid_t pid;
0039     int cpu;
0040     u64 ts_nsec;
0041     int num_stack_entries;
0042     unsigned long stack_entries[KFENCE_STACK_DEPTH];
0043 };
0044
0045 /* KFENCE metadata per guarded allocation. */
0046 struct kfence_metadata {
0047     struct list_head list;      /* Freelist node; access under kfence_freelist_lock. */
0048     struct rcu_head rcu_head;   /* For delayed freeing. */
0049
0050     /*
0051      * Lock protecting below data; to ensure consistency of the below data,
0052      * since the following may execute concurrently: __kfence_alloc(),
0053      * __kfence_free(), kfence_handle_page_fault(). However, note that we
0054      * cannot grab the same metadata off the freelist twice, and multiple
0055      * __kfence_alloc() cannot run concurrently on the same metadata.
0056      */
0057     raw_spinlock_t lock;
0058
0059     /* The current state of the object; see above. */
0060     enum kfence_object_state state;
0061
0062     /*
0063      * Allocated object address; cannot be calculated from size, because of
0064      * alignment requirements.
0065      *
0066      * Invariant: ALIGN_DOWN(addr, PAGE_SIZE) is constant.
0067      */
0068     unsigned long addr;
0069
0070     /*
0071      * The size of the original allocation.
0072      */
0073     size_t size;
0074
0075     /*
0076      * The kmem_cache cache of the last allocation; NULL if never allocated
0077      * or the cache has already been destroyed.
0078      */
0079     struct kmem_cache *cache;
0080
0081     /*
0082      * In case of an invalid access, the page that was unprotected; we
0083      * optimistically only store one address.
0084      */
0085     unsigned long unprotected_page;
0086
0087     /* Allocation and free stack information. */
0088     struct kfence_track alloc_track;
0089     struct kfence_track free_track;
0090     /* For updating alloc_covered on frees. */
0091     u32 alloc_stack_hash;
0092 #ifdef CONFIG_MEMCG
0093     struct obj_cgroup *objcg;
0094 #endif
0095 };
0096
0097 extern struct kfence_metadata kfence_metadata[CONFIG_KFENCE_NUM_OBJECTS];
0098
0099 static inline struct kfence_metadata *addr_to_metadata(unsigned long addr)
0100 {
0101     long index;
0102
0103     /* The checks do not affect performance; only called from slow-paths. */
0104
0105     if (!is_kfence_address((void *)addr))
0106         return NULL;
0107
0108     /*
0109      * May be an invalid index if called with an address at the edge of
0110      * __kfence_pool, in which case we would report an "invalid access"
0111      * error.
0112      */
0113     index = (addr - (unsigned long)__kfence_pool) / (PAGE_SIZE * 2) - 1;
0114     if (index < 0 || index >= CONFIG_KFENCE_NUM_OBJECTS)
0115         return NULL;
0116
0117     return &kfence_metadata[index];
0118 }
0119
0120 /* KFENCE error types for report generation. */
0121 enum kfence_error_type {
0122     KFENCE_ERROR_OOB,       /* Detected a out-of-bounds access. */
0123     KFENCE_ERROR_UAF,       /* Detected a use-after-free access. */
0124     KFENCE_ERROR_CORRUPTION,    /* Detected a memory corruption on free. */
0125     KFENCE_ERROR_INVALID,       /* Invalid access of unknown type. */
0126     KFENCE_ERROR_INVALID_FREE,  /* Invalid free. */
0127 };
0128
0129 void kfence_report_error(unsigned long address, bool is_write, struct pt_regs *regs,
0130              const struct kfence_metadata *meta, enum kfence_error_type type);
0131
0132 void kfence_print_object(struct seq_file *seq, const struct kfence_metadata *meta);
0133
0134 #endif /* MM_KFENCE_KFENCE_H */