the-tree/mm/slab_common.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * Slab allocator functions that are independent of the allocator strategy
0004  *
0005  * (C) 2012 Christoph Lameter <cl@linux.com>
0006  */
0007 #include <linux/slab.h>
0008
0009 #include <linux/mm.h>
0010 #include <linux/poison.h>
0011 #include <linux/interrupt.h>
0012 #include <linux/memory.h>
0013 #include <linux/cache.h>
0014 #include <linux/compiler.h>
0015 #include <linux/kfence.h>
0016 #include <linux/module.h>
0017 #include <linux/cpu.h>
0018 #include <linux/uaccess.h>
0019 #include <linux/seq_file.h>
0020 #include <linux/proc_fs.h>
0021 #include <linux/debugfs.h>
0022 #include <linux/kasan.h>
0023 #include <asm/cacheflush.h>
0024 #include <asm/tlbflush.h>
0025 #include <asm/page.h>
0026 #include <linux/memcontrol.h>
0027 #include <linux/stackdepot.h>
0028
0029 #include "internal.h"
0030 #include "slab.h"
0031
0032 #define CREATE_TRACE_POINTS
0033 #include <trace/events/kmem.h>
0034
0035 enum slab_state slab_state;
0036 LIST_HEAD(slab_caches);
0037 DEFINE_MUTEX(slab_mutex);
0038 struct kmem_cache *kmem_cache;
0039
0040 static LIST_HEAD(slab_caches_to_rcu_destroy);
0041 static void slab_caches_to_rcu_destroy_workfn(struct work_struct *work);
0042 static DECLARE_WORK(slab_caches_to_rcu_destroy_work,
0043             slab_caches_to_rcu_destroy_workfn);
0044
0045 /*
0046  * Set of flags that will prevent slab merging
0047  */
0048 #define SLAB_NEVER_MERGE (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \
0049         SLAB_TRACE | SLAB_TYPESAFE_BY_RCU | SLAB_NOLEAKTRACE | \
0050         SLAB_FAILSLAB | kasan_never_merge())
0051
0052 #define SLAB_MERGE_SAME (SLAB_RECLAIM_ACCOUNT | SLAB_CACHE_DMA | \
0053              SLAB_CACHE_DMA32 | SLAB_ACCOUNT)
0054
0055 /*
0056  * Merge control. If this is set then no merging of slab caches will occur.
0057  */
0058 static bool slab_nomerge = !IS_ENABLED(CONFIG_SLAB_MERGE_DEFAULT);
0059
0060 static int __init setup_slab_nomerge(char *str)
0061 {
0062     slab_nomerge = true;
0063     return 1;
0064 }
0065
0066 static int __init setup_slab_merge(char *str)
0067 {
0068     slab_nomerge = false;
0069     return 1;
0070 }
0071
0072 #ifdef CONFIG_SLUB
0073 __setup_param("slub_nomerge", slub_nomerge, setup_slab_nomerge, 0);
0074 __setup_param("slub_merge", slub_merge, setup_slab_merge, 0);
0075 #endif
0076
0077 __setup("slab_nomerge", setup_slab_nomerge);
0078 __setup("slab_merge", setup_slab_merge);
0079
0080 /*
0081  * Determine the size of a slab object
0082  */
0083 unsigned int kmem_cache_size(struct kmem_cache *s)
0084 {
0085     return s->object_size;
0086 }
0087 EXPORT_SYMBOL(kmem_cache_size);
0088
0089 #ifdef CONFIG_DEBUG_VM
0090 static int kmem_cache_sanity_check(const char *name, unsigned int size)
0091 {
0092     if (!name || in_interrupt() || size > KMALLOC_MAX_SIZE) {
0093         pr_err("kmem_cache_create(%s) integrity check failed\n", name);
0094         return -EINVAL;
0095     }
0096
0097     WARN_ON(strchr(name, ' ')); /* It confuses parsers */
0098     return 0;
0099 }
0100 #else
0101 static inline int kmem_cache_sanity_check(const char *name, unsigned int size)
0102 {
0103     return 0;
0104 }
0105 #endif
0106
0107 /*
0108  * Figure out what the alignment of the objects will be given a set of
0109  * flags, a user specified alignment and the size of the objects.
0110  */
0111 static unsigned int calculate_alignment(slab_flags_t flags,
0112         unsigned int align, unsigned int size)
0113 {
0114     /*
0115      * If the user wants hardware cache aligned objects then follow that
0116      * suggestion if the object is sufficiently large.
0117      *
0118      * The hardware cache alignment cannot override the specified
0119      * alignment though. If that is greater then use it.
0120      */
0121     if (flags & SLAB_HWCACHE_ALIGN) {
0122         unsigned int ralign;
0123
0124         ralign = cache_line_size();
0125         while (size <= ralign / 2)
0126             ralign /= 2;
0127         align = max(align, ralign);
0128     }
0129
0130     align = max(align, arch_slab_minalign());
0131
0132     return ALIGN(align, sizeof(void *));
0133 }
0134
0135 /*
0136  * Find a mergeable slab cache
0137  */
0138 int slab_unmergeable(struct kmem_cache *s)
0139 {
0140     if (slab_nomerge || (s->flags & SLAB_NEVER_MERGE))
0141         return 1;
0142
0143     if (s->ctor)
0144         return 1;
0145
0146     if (s->usersize)
0147         return 1;
0148
0149     /*
0150      * We may have set a slab to be unmergeable during bootstrap.
0151      */
0152     if (s->refcount < 0)
0153         return 1;
0154
0155     return 0;
0156 }
0157
0158 struct kmem_cache *find_mergeable(unsigned int size, unsigned int align,
0159         slab_flags_t flags, const char *name, void (*ctor)(void *))
0160 {
0161     struct kmem_cache *s;
0162
0163     if (slab_nomerge)
0164         return NULL;
0165
0166     if (ctor)
0167         return NULL;
0168
0169     size = ALIGN(size, sizeof(void *));
0170     align = calculate_alignment(flags, align, size);
0171     size = ALIGN(size, align);
0172     flags = kmem_cache_flags(size, flags, name);
0173
0174     if (flags & SLAB_NEVER_MERGE)
0175         return NULL;
0176
0177     list_for_each_entry_reverse(s, &slab_caches, list) {
0178         if (slab_unmergeable(s))
0179             continue;
0180
0181         if (size > s->size)
0182             continue;
0183
0184         if ((flags & SLAB_MERGE_SAME) != (s->flags & SLAB_MERGE_SAME))
0185             continue;
0186         /*
0187          * Check if alignment is compatible.
0188          * Courtesy of Adrian Drzewiecki
0189          */
0190         if ((s->size & ~(align - 1)) != s->size)
0191             continue;
0192
0193         if (s->size - size >= sizeof(void *))
0194             continue;
0195
0196         if (IS_ENABLED(CONFIG_SLAB) && align &&
0197             (align > s->align || s->align % align))
0198             continue;
0199
0200         return s;
0201     }
0202     return NULL;
0203 }
0204
0205 static struct kmem_cache *create_cache(const char *name,
0206         unsigned int object_size, unsigned int align,
0207         slab_flags_t flags, unsigned int useroffset,
0208         unsigned int usersize, void (*ctor)(void *),
0209         struct kmem_cache *root_cache)
0210 {
0211     struct kmem_cache *s;
0212     int err;
0213
0214     if (WARN_ON(useroffset + usersize > object_size))
0215         useroffset = usersize = 0;
0216
0217     err = -ENOMEM;
0218     s = kmem_cache_zalloc(kmem_cache, GFP_KERNEL);
0219     if (!s)
0220         goto out;
0221
0222     s->name = name;
0223     s->size = s->object_size = object_size;
0224     s->align = align;
0225     s->ctor = ctor;
0226     s->useroffset = useroffset;
0227     s->usersize = usersize;
0228
0229     err = __kmem_cache_create(s, flags);
0230     if (err)
0231         goto out_free_cache;
0232
0233     s->refcount = 1;
0234     list_add(&s->list, &slab_caches);
0235 out:
0236     if (err)
0237         return ERR_PTR(err);
0238     return s;
0239
0240 out_free_cache:
0241     kmem_cache_free(kmem_cache, s);
0242     goto out;
0243 }
0244
0245 /**
0246  * kmem_cache_create_usercopy - Create a cache with a region suitable
0247  * for copying to userspace
0248  * @name: A string which is used in /proc/slabinfo to identify this cache.
0249  * @size: The size of objects to be created in this cache.
0250  * @align: The required alignment for the objects.
0251  * @flags: SLAB flags
0252  * @useroffset: Usercopy region offset
0253  * @usersize: Usercopy region size
0254  * @ctor: A constructor for the objects.
0255  *
0256  * Cannot be called within a interrupt, but can be interrupted.
0257  * The @ctor is run when new pages are allocated by the cache.
0258  *
0259  * The flags are
0260  *
0261  * %SLAB_POISON - Poison the slab with a known test pattern (a5a5a5a5)
0262  * to catch references to uninitialised memory.
0263  *
0264  * %SLAB_RED_ZONE - Insert `Red` zones around the allocated memory to check
0265  * for buffer overruns.
0266  *
0267  * %SLAB_HWCACHE_ALIGN - Align the objects in this cache to a hardware
0268  * cacheline.  This can be beneficial if you're counting cycles as closely
0269  * as davem.
0270  *
0271  * Return: a pointer to the cache on success, NULL on failure.
0272  */
0273 struct kmem_cache *
0274 kmem_cache_create_usercopy(const char *name,
0275           unsigned int size, unsigned int align,
0276           slab_flags_t flags,
0277           unsigned int useroffset, unsigned int usersize,
0278           void (*ctor)(void *))
0279 {
0280     struct kmem_cache *s = NULL;
0281     const char *cache_name;
0282     int err;
0283
0284 #ifdef CONFIG_SLUB_DEBUG
0285     /*
0286      * If no slub_debug was enabled globally, the static key is not yet
0287      * enabled by setup_slub_debug(). Enable it if the cache is being
0288      * created with any of the debugging flags passed explicitly.
0289      * It's also possible that this is the first cache created with
0290      * SLAB_STORE_USER and we should init stack_depot for it.
0291      */
0292     if (flags & SLAB_DEBUG_FLAGS)
0293         static_branch_enable(&slub_debug_enabled);
0294     if (flags & SLAB_STORE_USER)
0295         stack_depot_init();
0296 #endif
0297
0298     mutex_lock(&slab_mutex);
0299
0300     err = kmem_cache_sanity_check(name, size);
0301     if (err) {
0302         goto out_unlock;
0303     }
0304
0305     /* Refuse requests with allocator specific flags */
0306     if (flags & ~SLAB_FLAGS_PERMITTED) {
0307         err = -EINVAL;
0308         goto out_unlock;
0309     }
0310
0311     /*
0312      * Some allocators will constraint the set of valid flags to a subset
0313      * of all flags. We expect them to define CACHE_CREATE_MASK in this
0314      * case, and we'll just provide them with a sanitized version of the
0315      * passed flags.
0316      */
0317     flags &= CACHE_CREATE_MASK;
0318
0319     /* Fail closed on bad usersize of useroffset values. */
0320     if (WARN_ON(!usersize && useroffset) ||
0321         WARN_ON(size < usersize || size - usersize < useroffset))
0322         usersize = useroffset = 0;
0323
0324     if (!usersize)
0325         s = __kmem_cache_alias(name, size, align, flags, ctor);
0326     if (s)
0327         goto out_unlock;
0328
0329     cache_name = kstrdup_const(name, GFP_KERNEL);
0330     if (!cache_name) {
0331         err = -ENOMEM;
0332         goto out_unlock;
0333     }
0334
0335     s = create_cache(cache_name, size,
0336              calculate_alignment(flags, align, size),
0337              flags, useroffset, usersize, ctor, NULL);
0338     if (IS_ERR(s)) {
0339         err = PTR_ERR(s);
0340         kfree_const(cache_name);
0341     }
0342
0343 out_unlock:
0344     mutex_unlock(&slab_mutex);
0345
0346     if (err) {
0347         if (flags & SLAB_PANIC)
0348             panic("%s: Failed to create slab '%s'. Error %d\n",
0349                 __func__, name, err);
0350         else {
0351             pr_warn("%s(%s) failed with error %d\n",
0352                 __func__, name, err);
0353             dump_stack();
0354         }
0355         return NULL;
0356     }
0357     return s;
0358 }
0359 EXPORT_SYMBOL(kmem_cache_create_usercopy);
0360
0361 /**
0362  * kmem_cache_create - Create a cache.
0363  * @name: A string which is used in /proc/slabinfo to identify this cache.
0364  * @size: The size of objects to be created in this cache.
0365  * @align: The required alignment for the objects.
0366  * @flags: SLAB flags
0367  * @ctor: A constructor for the objects.
0368  *
0369  * Cannot be called within a interrupt, but can be interrupted.
0370  * The @ctor is run when new pages are allocated by the cache.
0371  *
0372  * The flags are
0373  *
0374  * %SLAB_POISON - Poison the slab with a known test pattern (a5a5a5a5)
0375  * to catch references to uninitialised memory.
0376  *
0377  * %SLAB_RED_ZONE - Insert `Red` zones around the allocated memory to check
0378  * for buffer overruns.
0379  *
0380  * %SLAB_HWCACHE_ALIGN - Align the objects in this cache to a hardware
0381  * cacheline.  This can be beneficial if you're counting cycles as closely
0382  * as davem.
0383  *
0384  * Return: a pointer to the cache on success, NULL on failure.
0385  */
0386 struct kmem_cache *
0387 kmem_cache_create(const char *name, unsigned int size, unsigned int align,
0388         slab_flags_t flags, void (*ctor)(void *))
0389 {
0390     return kmem_cache_create_usercopy(name, size, align, flags, 0, 0,
0391                       ctor);
0392 }
0393 EXPORT_SYMBOL(kmem_cache_create);
0394
0395 #ifdef SLAB_SUPPORTS_SYSFS
0396 /*
0397  * For a given kmem_cache, kmem_cache_destroy() should only be called
0398  * once or there will be a use-after-free problem. The actual deletion
0399  * and release of the kobject does not need slab_mutex or cpu_hotplug_lock
0400  * protection. So they are now done without holding those locks.
0401  *
0402  * Note that there will be a slight delay in the deletion of sysfs files
0403  * if kmem_cache_release() is called indrectly from a work function.
0404  */
0405 static void kmem_cache_release(struct kmem_cache *s)
0406 {
0407     sysfs_slab_unlink(s);
0408     sysfs_slab_release(s);
0409 }
0410 #else
0411 static void kmem_cache_release(struct kmem_cache *s)
0412 {
0413     slab_kmem_cache_release(s);
0414 }
0415 #endif
0416
0417 static void slab_caches_to_rcu_destroy_workfn(struct work_struct *work)
0418 {
0419     LIST_HEAD(to_destroy);
0420     struct kmem_cache *s, *s2;
0421
0422     /*
0423      * On destruction, SLAB_TYPESAFE_BY_RCU kmem_caches are put on the
0424      * @slab_caches_to_rcu_destroy list.  The slab pages are freed
0425      * through RCU and the associated kmem_cache are dereferenced
0426      * while freeing the pages, so the kmem_caches should be freed only
0427      * after the pending RCU operations are finished.  As rcu_barrier()
0428      * is a pretty slow operation, we batch all pending destructions
0429      * asynchronously.
0430      */
0431     mutex_lock(&slab_mutex);
0432     list_splice_init(&slab_caches_to_rcu_destroy, &to_destroy);
0433     mutex_unlock(&slab_mutex);
0434
0435     if (list_empty(&to_destroy))
0436         return;
0437
0438     rcu_barrier();
0439
0440     list_for_each_entry_safe(s, s2, &to_destroy, list) {
0441         debugfs_slab_release(s);
0442         kfence_shutdown_cache(s);
0443         kmem_cache_release(s);
0444     }
0445 }
0446
0447 static int shutdown_cache(struct kmem_cache *s)
0448 {
0449     /* free asan quarantined objects */
0450     kasan_cache_shutdown(s);
0451
0452     if (__kmem_cache_shutdown(s) != 0)
0453         return -EBUSY;
0454
0455     list_del(&s->list);
0456
0457     if (s->flags & SLAB_TYPESAFE_BY_RCU) {
0458         list_add_tail(&s->list, &slab_caches_to_rcu_destroy);
0459         schedule_work(&slab_caches_to_rcu_destroy_work);
0460     } else {
0461         kfence_shutdown_cache(s);
0462         debugfs_slab_release(s);
0463     }
0464
0465     return 0;
0466 }
0467
0468 void slab_kmem_cache_release(struct kmem_cache *s)
0469 {
0470     __kmem_cache_release(s);
0471     kfree_const(s->name);
0472     kmem_cache_free(kmem_cache, s);
0473 }
0474
0475 void kmem_cache_destroy(struct kmem_cache *s)
0476 {
0477     int refcnt;
0478     bool rcu_set;
0479
0480     if (unlikely(!s) || !kasan_check_byte(s))
0481         return;
0482
0483     cpus_read_lock();
0484     mutex_lock(&slab_mutex);
0485
0486     rcu_set = s->flags & SLAB_TYPESAFE_BY_RCU;
0487
0488     refcnt = --s->refcount;
0489     if (refcnt)
0490         goto out_unlock;
0491
0492     WARN(shutdown_cache(s),
0493          "%s %s: Slab cache still has objects when called from %pS",
0494          __func__, s->name, (void *)_RET_IP_);
0495 out_unlock:
0496     mutex_unlock(&slab_mutex);
0497     cpus_read_unlock();
0498     if (!refcnt && !rcu_set)
0499         kmem_cache_release(s);
0500 }
0501 EXPORT_SYMBOL(kmem_cache_destroy);
0502
0503 /**
0504  * kmem_cache_shrink - Shrink a cache.
0505  * @cachep: The cache to shrink.
0506  *
0507  * Releases as many slabs as possible for a cache.
0508  * To help debugging, a zero exit status indicates all slabs were released.
0509  *
0510  * Return: %0 if all slabs were released, non-zero otherwise
0511  */
0512 int kmem_cache_shrink(struct kmem_cache *cachep)
0513 {
0514     int ret;
0515
0516
0517     kasan_cache_shrink(cachep);
0518     ret = __kmem_cache_shrink(cachep);
0519
0520     return ret;
0521 }
0522 EXPORT_SYMBOL(kmem_cache_shrink);
0523
0524 bool slab_is_available(void)
0525 {
0526     return slab_state >= UP;
0527 }
0528
0529 #ifdef CONFIG_PRINTK
0530 /**
0531  * kmem_valid_obj - does the pointer reference a valid slab object?
0532  * @object: pointer to query.
0533  *
0534  * Return: %true if the pointer is to a not-yet-freed object from
0535  * kmalloc() or kmem_cache_alloc(), either %true or %false if the pointer
0536  * is to an already-freed object, and %false otherwise.
0537  */
0538 bool kmem_valid_obj(void *object)
0539 {
0540     struct folio *folio;
0541
0542     /* Some arches consider ZERO_SIZE_PTR to be a valid address. */
0543     if (object < (void *)PAGE_SIZE || !virt_addr_valid(object))
0544         return false;
0545     folio = virt_to_folio(object);
0546     return folio_test_slab(folio);
0547 }
0548 EXPORT_SYMBOL_GPL(kmem_valid_obj);
0549
0550 static void kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab)
0551 {
0552     if (__kfence_obj_info(kpp, object, slab))
0553         return;
0554     __kmem_obj_info(kpp, object, slab);
0555 }
0556
0557 /**
0558  * kmem_dump_obj - Print available slab provenance information
0559  * @object: slab object for which to find provenance information.
0560  *
0561  * This function uses pr_cont(), so that the caller is expected to have
0562  * printed out whatever preamble is appropriate.  The provenance information
0563  * depends on the type of object and on how much debugging is enabled.
0564  * For a slab-cache object, the fact that it is a slab object is printed,
0565  * and, if available, the slab name, return address, and stack trace from
0566  * the allocation and last free path of that object.
0567  *
0568  * This function will splat if passed a pointer to a non-slab object.
0569  * If you are not sure what type of object you have, you should instead
0570  * use mem_dump_obj().
0571  */
0572 void kmem_dump_obj(void *object)
0573 {
0574     char *cp = IS_ENABLED(CONFIG_MMU) ? "" : "/vmalloc";
0575     int i;
0576     struct slab *slab;
0577     unsigned long ptroffset;
0578     struct kmem_obj_info kp = { };
0579
0580     if (WARN_ON_ONCE(!virt_addr_valid(object)))
0581         return;
0582     slab = virt_to_slab(object);
0583     if (WARN_ON_ONCE(!slab)) {
0584         pr_cont(" non-slab memory.\n");
0585         return;
0586     }
0587     kmem_obj_info(&kp, object, slab);
0588     if (kp.kp_slab_cache)
0589         pr_cont(" slab%s %s", cp, kp.kp_slab_cache->name);
0590     else
0591         pr_cont(" slab%s", cp);
0592     if (is_kfence_address(object))
0593         pr_cont(" (kfence)");
0594     if (kp.kp_objp)
0595         pr_cont(" start %px", kp.kp_objp);
0596     if (kp.kp_data_offset)
0597         pr_cont(" data offset %lu", kp.kp_data_offset);
0598     if (kp.kp_objp) {
0599         ptroffset = ((char *)object - (char *)kp.kp_objp) - kp.kp_data_offset;
0600         pr_cont(" pointer offset %lu", ptroffset);
0601     }
0602     if (kp.kp_slab_cache && kp.kp_slab_cache->usersize)
0603         pr_cont(" size %u", kp.kp_slab_cache->usersize);
0604     if (kp.kp_ret)
0605         pr_cont(" allocated at %pS\n", kp.kp_ret);
0606     else
0607         pr_cont("\n");
0608     for (i = 0; i < ARRAY_SIZE(kp.kp_stack); i++) {
0609         if (!kp.kp_stack[i])
0610             break;
0611         pr_info("    %pS\n", kp.kp_stack[i]);
0612     }
0613
0614     if (kp.kp_free_stack[0])
0615         pr_cont(" Free path:\n");
0616
0617     for (i = 0; i < ARRAY_SIZE(kp.kp_free_stack); i++) {
0618         if (!kp.kp_free_stack[i])
0619             break;
0620         pr_info("    %pS\n", kp.kp_free_stack[i]);
0621     }
0622
0623 }
0624 EXPORT_SYMBOL_GPL(kmem_dump_obj);
0625 #endif
0626
0627 #ifndef CONFIG_SLOB
0628 /* Create a cache during boot when no slab services are available yet */
0629 void __init create_boot_cache(struct kmem_cache *s, const char *name,
0630         unsigned int size, slab_flags_t flags,
0631         unsigned int useroffset, unsigned int usersize)
0632 {
0633     int err;
0634     unsigned int align = ARCH_KMALLOC_MINALIGN;
0635
0636     s->name = name;
0637     s->size = s->object_size = size;
0638
0639     /*
0640      * For power of two sizes, guarantee natural alignment for kmalloc
0641      * caches, regardless of SL*B debugging options.
0642      */
0643     if (is_power_of_2(size))
0644         align = max(align, size);
0645     s->align = calculate_alignment(flags, align, size);
0646
0647     s->useroffset = useroffset;
0648     s->usersize = usersize;
0649
0650     err = __kmem_cache_create(s, flags);
0651
0652     if (err)
0653         panic("Creation of kmalloc slab %s size=%u failed. Reason %d\n",
0654                     name, size, err);
0655
0656     s->refcount = -1;   /* Exempt from merging for now */
0657 }
0658
0659 struct kmem_cache *__init create_kmalloc_cache(const char *name,
0660         unsigned int size, slab_flags_t flags,
0661         unsigned int useroffset, unsigned int usersize)
0662 {
0663     struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
0664
0665     if (!s)
0666         panic("Out of memory when creating slab %s\n", name);
0667
0668     create_boot_cache(s, name, size, flags, useroffset, usersize);
0669     kasan_cache_create_kmalloc(s);
0670     list_add(&s->list, &slab_caches);
0671     s->refcount = 1;
0672     return s;
0673 }
0674
0675 struct kmem_cache *
0676 kmalloc_caches[NR_KMALLOC_TYPES][KMALLOC_SHIFT_HIGH + 1] __ro_after_init =
0677 { /* initialization for https://bugs.llvm.org/show_bug.cgi?id=42570 */ };
0678 EXPORT_SYMBOL(kmalloc_caches);
0679
0680 /*
0681  * Conversion table for small slabs sizes / 8 to the index in the
0682  * kmalloc array. This is necessary for slabs < 192 since we have non power
0683  * of two cache sizes there. The size of larger slabs can be determined using
0684  * fls.
0685  */
0686 static u8 size_index[24] __ro_after_init = {
0687     3,  /* 8 */
0688     4,  /* 16 */
0689     5,  /* 24 */
0690     5,  /* 32 */
0691     6,  /* 40 */
0692     6,  /* 48 */
0693     6,  /* 56 */
0694     6,  /* 64 */
0695     1,  /* 72 */
0696     1,  /* 80 */
0697     1,  /* 88 */
0698     1,  /* 96 */
0699     7,  /* 104 */
0700     7,  /* 112 */
0701     7,  /* 120 */
0702     7,  /* 128 */
0703     2,  /* 136 */
0704     2,  /* 144 */
0705     2,  /* 152 */
0706     2,  /* 160 */
0707     2,  /* 168 */
0708     2,  /* 176 */
0709     2,  /* 184 */
0710     2   /* 192 */
0711 };
0712
0713 static inline unsigned int size_index_elem(unsigned int bytes)
0714 {
0715     return (bytes - 1) / 8;
0716 }
0717
0718 /*
0719  * Find the kmem_cache structure that serves a given size of
0720  * allocation
0721  */
0722 struct kmem_cache *kmalloc_slab(size_t size, gfp_t flags)
0723 {
0724     unsigned int index;
0725
0726     if (size <= 192) {
0727         if (!size)
0728             return ZERO_SIZE_PTR;
0729
0730         index = size_index[size_index_elem(size)];
0731     } else {
0732         if (WARN_ON_ONCE(size > KMALLOC_MAX_CACHE_SIZE))
0733             return NULL;
0734         index = fls(size - 1);
0735     }
0736
0737     return kmalloc_caches[kmalloc_type(flags)][index];
0738 }
0739
0740 #ifdef CONFIG_ZONE_DMA
0741 #define KMALLOC_DMA_NAME(sz)    .name[KMALLOC_DMA] = "dma-kmalloc-" #sz,
0742 #else
0743 #define KMALLOC_DMA_NAME(sz)
0744 #endif
0745
0746 #ifdef CONFIG_MEMCG_KMEM
0747 #define KMALLOC_CGROUP_NAME(sz) .name[KMALLOC_CGROUP] = "kmalloc-cg-" #sz,
0748 #else
0749 #define KMALLOC_CGROUP_NAME(sz)
0750 #endif
0751
0752 #define INIT_KMALLOC_INFO(__size, __short_size)         \
0753 {                               \
0754     .name[KMALLOC_NORMAL]  = "kmalloc-" #__short_size,  \
0755     .name[KMALLOC_RECLAIM] = "kmalloc-rcl-" #__short_size,  \
0756     KMALLOC_CGROUP_NAME(__short_size)           \
0757     KMALLOC_DMA_NAME(__short_size)              \
0758     .size = __size,                     \
0759 }
0760
0761 /*
0762  * kmalloc_info[] is to make slub_debug=,kmalloc-xx option work at boot time.
0763  * kmalloc_index() supports up to 2^25=32MB, so the final entry of the table is
0764  * kmalloc-32M.
0765  */
0766 const struct kmalloc_info_struct kmalloc_info[] __initconst = {
0767     INIT_KMALLOC_INFO(0, 0),
0768     INIT_KMALLOC_INFO(96, 96),
0769     INIT_KMALLOC_INFO(192, 192),
0770     INIT_KMALLOC_INFO(8, 8),
0771     INIT_KMALLOC_INFO(16, 16),
0772     INIT_KMALLOC_INFO(32, 32),
0773     INIT_KMALLOC_INFO(64, 64),
0774     INIT_KMALLOC_INFO(128, 128),
0775     INIT_KMALLOC_INFO(256, 256),
0776     INIT_KMALLOC_INFO(512, 512),
0777     INIT_KMALLOC_INFO(1024, 1k),
0778     INIT_KMALLOC_INFO(2048, 2k),
0779     INIT_KMALLOC_INFO(4096, 4k),
0780     INIT_KMALLOC_INFO(8192, 8k),
0781     INIT_KMALLOC_INFO(16384, 16k),
0782     INIT_KMALLOC_INFO(32768, 32k),
0783     INIT_KMALLOC_INFO(65536, 64k),
0784     INIT_KMALLOC_INFO(131072, 128k),
0785     INIT_KMALLOC_INFO(262144, 256k),
0786     INIT_KMALLOC_INFO(524288, 512k),
0787     INIT_KMALLOC_INFO(1048576, 1M),
0788     INIT_KMALLOC_INFO(2097152, 2M),
0789     INIT_KMALLOC_INFO(4194304, 4M),
0790     INIT_KMALLOC_INFO(8388608, 8M),
0791     INIT_KMALLOC_INFO(16777216, 16M),
0792     INIT_KMALLOC_INFO(33554432, 32M)
0793 };
0794
0795 /*
0796  * Patch up the size_index table if we have strange large alignment
0797  * requirements for the kmalloc array. This is only the case for
0798  * MIPS it seems. The standard arches will not generate any code here.
0799  *
0800  * Largest permitted alignment is 256 bytes due to the way we
0801  * handle the index determination for the smaller caches.
0802  *
0803  * Make sure that nothing crazy happens if someone starts tinkering
0804  * around with ARCH_KMALLOC_MINALIGN
0805  */
0806 void __init setup_kmalloc_cache_index_table(void)
0807 {
0808     unsigned int i;
0809
0810     BUILD_BUG_ON(KMALLOC_MIN_SIZE > 256 ||
0811         !is_power_of_2(KMALLOC_MIN_SIZE));
0812
0813     for (i = 8; i < KMALLOC_MIN_SIZE; i += 8) {
0814         unsigned int elem = size_index_elem(i);
0815
0816         if (elem >= ARRAY_SIZE(size_index))
0817             break;
0818         size_index[elem] = KMALLOC_SHIFT_LOW;
0819     }
0820
0821     if (KMALLOC_MIN_SIZE >= 64) {
0822         /*
0823          * The 96 byte sized cache is not used if the alignment
0824          * is 64 byte.
0825          */
0826         for (i = 64 + 8; i <= 96; i += 8)
0827             size_index[size_index_elem(i)] = 7;
0828
0829     }
0830
0831     if (KMALLOC_MIN_SIZE >= 128) {
0832         /*
0833          * The 192 byte sized cache is not used if the alignment
0834          * is 128 byte. Redirect kmalloc to use the 256 byte cache
0835          * instead.
0836          */
0837         for (i = 128 + 8; i <= 192; i += 8)
0838             size_index[size_index_elem(i)] = 8;
0839     }
0840 }
0841
0842 static void __init
0843 new_kmalloc_cache(int idx, enum kmalloc_cache_type type, slab_flags_t flags)
0844 {
0845     if (type == KMALLOC_RECLAIM) {
0846         flags |= SLAB_RECLAIM_ACCOUNT;
0847     } else if (IS_ENABLED(CONFIG_MEMCG_KMEM) && (type == KMALLOC_CGROUP)) {
0848         if (mem_cgroup_kmem_disabled()) {
0849             kmalloc_caches[type][idx] = kmalloc_caches[KMALLOC_NORMAL][idx];
0850             return;
0851         }
0852         flags |= SLAB_ACCOUNT;
0853     } else if (IS_ENABLED(CONFIG_ZONE_DMA) && (type == KMALLOC_DMA)) {
0854         flags |= SLAB_CACHE_DMA;
0855     }
0856
0857     kmalloc_caches[type][idx] = create_kmalloc_cache(
0858                     kmalloc_info[idx].name[type],
0859                     kmalloc_info[idx].size, flags, 0,
0860                     kmalloc_info[idx].size);
0861
0862     /*
0863      * If CONFIG_MEMCG_KMEM is enabled, disable cache merging for
0864      * KMALLOC_NORMAL caches.
0865      */
0866     if (IS_ENABLED(CONFIG_MEMCG_KMEM) && (type == KMALLOC_NORMAL))
0867         kmalloc_caches[type][idx]->refcount = -1;
0868 }
0869
0870 /*
0871  * Create the kmalloc array. Some of the regular kmalloc arrays
0872  * may already have been created because they were needed to
0873  * enable allocations for slab creation.
0874  */
0875 void __init create_kmalloc_caches(slab_flags_t flags)
0876 {
0877     int i;
0878     enum kmalloc_cache_type type;
0879
0880     /*
0881      * Including KMALLOC_CGROUP if CONFIG_MEMCG_KMEM defined
0882      */
0883     for (type = KMALLOC_NORMAL; type < NR_KMALLOC_TYPES; type++) {
0884         for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) {
0885             if (!kmalloc_caches[type][i])
0886                 new_kmalloc_cache(i, type, flags);
0887
0888             /*
0889              * Caches that are not of the two-to-the-power-of size.
0890              * These have to be created immediately after the
0891              * earlier power of two caches
0892              */
0893             if (KMALLOC_MIN_SIZE <= 32 && i == 6 &&
0894                     !kmalloc_caches[type][1])
0895                 new_kmalloc_cache(1, type, flags);
0896             if (KMALLOC_MIN_SIZE <= 64 && i == 7 &&
0897                     !kmalloc_caches[type][2])
0898                 new_kmalloc_cache(2, type, flags);
0899         }
0900     }
0901
0902     /* Kmalloc array is now usable */
0903     slab_state = UP;
0904 }
0905 #endif /* !CONFIG_SLOB */
0906
0907 gfp_t kmalloc_fix_flags(gfp_t flags)
0908 {
0909     gfp_t invalid_mask = flags & GFP_SLAB_BUG_MASK;
0910
0911     flags &= ~GFP_SLAB_BUG_MASK;
0912     pr_warn("Unexpected gfp: %#x (%pGg). Fixing up to gfp: %#x (%pGg). Fix your code!\n",
0913             invalid_mask, &invalid_mask, flags, &flags);
0914     dump_stack();
0915
0916     return flags;
0917 }
0918
0919 /*
0920  * To avoid unnecessary overhead, we pass through large allocation requests
0921  * directly to the page allocator. We use __GFP_COMP, because we will need to
0922  * know the allocation order to free the pages properly in kfree.
0923  */
0924 void *kmalloc_order(size_t size, gfp_t flags, unsigned int order)
0925 {
0926     void *ret = NULL;
0927     struct page *page;
0928
0929     if (unlikely(flags & GFP_SLAB_BUG_MASK))
0930         flags = kmalloc_fix_flags(flags);
0931
0932     flags |= __GFP_COMP;
0933     page = alloc_pages(flags, order);
0934     if (likely(page)) {
0935         ret = page_address(page);
0936         mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE_B,
0937                       PAGE_SIZE << order);
0938     }
0939     ret = kasan_kmalloc_large(ret, size, flags);
0940     /* As ret might get tagged, call kmemleak hook after KASAN. */
0941     kmemleak_alloc(ret, size, 1, flags);
0942     return ret;
0943 }
0944 EXPORT_SYMBOL(kmalloc_order);
0945
0946 #ifdef CONFIG_TRACING
0947 void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order)
0948 {
0949     void *ret = kmalloc_order(size, flags, order);
0950     trace_kmalloc(_RET_IP_, ret, NULL, size, PAGE_SIZE << order, flags);
0951     return ret;
0952 }
0953 EXPORT_SYMBOL(kmalloc_order_trace);
0954 #endif
0955
0956 #ifdef CONFIG_SLAB_FREELIST_RANDOM
0957 /* Randomize a generic freelist */
0958 static void freelist_randomize(struct rnd_state *state, unsigned int *list,
0959                    unsigned int count)
0960 {
0961     unsigned int rand;
0962     unsigned int i;
0963
0964     for (i = 0; i < count; i++)
0965         list[i] = i;
0966
0967     /* Fisher-Yates shuffle */
0968     for (i = count - 1; i > 0; i--) {
0969         rand = prandom_u32_state(state);
0970         rand %= (i + 1);
0971         swap(list[i], list[rand]);
0972     }
0973 }
0974
0975 /* Create a random sequence per cache */
0976 int cache_random_seq_create(struct kmem_cache *cachep, unsigned int count,
0977                     gfp_t gfp)
0978 {
0979     struct rnd_state state;
0980
0981     if (count < 2 || cachep->random_seq)
0982         return 0;
0983
0984     cachep->random_seq = kcalloc(count, sizeof(unsigned int), gfp);
0985     if (!cachep->random_seq)
0986         return -ENOMEM;
0987
0988     /* Get best entropy at this stage of boot */
0989     prandom_seed_state(&state, get_random_long());
0990
0991     freelist_randomize(&state, cachep->random_seq, count);
0992     return 0;
0993 }
0994
0995 /* Destroy the per-cache random freelist sequence */
0996 void cache_random_seq_destroy(struct kmem_cache *cachep)
0997 {
0998     kfree(cachep->random_seq);
0999     cachep->random_seq = NULL;
1000 }
1001 #endif /* CONFIG_SLAB_FREELIST_RANDOM */
1002
1003 #if defined(CONFIG_SLAB) || defined(CONFIG_SLUB_DEBUG)
1004 #ifdef CONFIG_SLAB
1005 #define SLABINFO_RIGHTS (0600)
1006 #else
1007 #define SLABINFO_RIGHTS (0400)
1008 #endif
1009
1010 static void print_slabinfo_header(struct seq_file *m)
1011 {
1012     /*
1013      * Output format version, so at least we can change it
1014      * without _too_ many complaints.
1015      */
1016 #ifdef CONFIG_DEBUG_SLAB
1017     seq_puts(m, "slabinfo - version: 2.1 (statistics)\n");
1018 #else
1019     seq_puts(m, "slabinfo - version: 2.1\n");
1020 #endif
1021     seq_puts(m, "# name            <active_objs> <num_objs> <objsize> <objperslab> <pagesperslab>");
1022     seq_puts(m, " : tunables <limit> <batchcount> <sharedfactor>");
1023     seq_puts(m, " : slabdata <active_slabs> <num_slabs> <sharedavail>");
1024 #ifdef CONFIG_DEBUG_SLAB
1025     seq_puts(m, " : globalstat <listallocs> <maxobjs> <grown> <reaped> <error> <maxfreeable> <nodeallocs> <remotefrees> <alienoverflow>");
1026     seq_puts(m, " : cpustat <allochit> <allocmiss> <freehit> <freemiss>");
1027 #endif
1028     seq_putc(m, '\n');
1029 }
1030
1031 static void *slab_start(struct seq_file *m, loff_t *pos)
1032 {
1033     mutex_lock(&slab_mutex);
1034     return seq_list_start(&slab_caches, *pos);
1035 }
1036
1037 static void *slab_next(struct seq_file *m, void *p, loff_t *pos)
1038 {
1039     return seq_list_next(p, &slab_caches, pos);
1040 }
1041
1042 static void slab_stop(struct seq_file *m, void *p)
1043 {
1044     mutex_unlock(&slab_mutex);
1045 }
1046
1047 static void cache_show(struct kmem_cache *s, struct seq_file *m)
1048 {
1049     struct slabinfo sinfo;
1050
1051     memset(&sinfo, 0, sizeof(sinfo));
1052     get_slabinfo(s, &sinfo);
1053
1054     seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d",
1055            s->name, sinfo.active_objs, sinfo.num_objs, s->size,
1056            sinfo.objects_per_slab, (1 << sinfo.cache_order));
1057
1058     seq_printf(m, " : tunables %4u %4u %4u",
1059            sinfo.limit, sinfo.batchcount, sinfo.shared);
1060     seq_printf(m, " : slabdata %6lu %6lu %6lu",
1061            sinfo.active_slabs, sinfo.num_slabs, sinfo.shared_avail);
1062     slabinfo_show_stats(m, s);
1063     seq_putc(m, '\n');
1064 }
1065
1066 static int slab_show(struct seq_file *m, void *p)
1067 {
1068     struct kmem_cache *s = list_entry(p, struct kmem_cache, list);
1069
1070     if (p == slab_caches.next)
1071         print_slabinfo_header(m);
1072     cache_show(s, m);
1073     return 0;
1074 }
1075
1076 void dump_unreclaimable_slab(void)
1077 {
1078     struct kmem_cache *s;
1079     struct slabinfo sinfo;
1080
1081     /*
1082      * Here acquiring slab_mutex is risky since we don't prefer to get
1083      * sleep in oom path. But, without mutex hold, it may introduce a
1084      * risk of crash.
1085      * Use mutex_trylock to protect the list traverse, dump nothing
1086      * without acquiring the mutex.
1087      */
1088     if (!mutex_trylock(&slab_mutex)) {
1089         pr_warn("excessive unreclaimable slab but cannot dump stats\n");
1090         return;
1091     }
1092
1093     pr_info("Unreclaimable slab info:\n");
1094     pr_info("Name                      Used          Total\n");
1095
1096     list_for_each_entry(s, &slab_caches, list) {
1097         if (s->flags & SLAB_RECLAIM_ACCOUNT)
1098             continue;
1099
1100         get_slabinfo(s, &sinfo);
1101
1102         if (sinfo.num_objs > 0)
1103             pr_info("%-17s %10luKB %10luKB\n", s->name,
1104                 (sinfo.active_objs * s->size) / 1024,
1105                 (sinfo.num_objs * s->size) / 1024);
1106     }
1107     mutex_unlock(&slab_mutex);
1108 }
1109
1110 /*
1111  * slabinfo_op - iterator that generates /proc/slabinfo
1112  *
1113  * Output layout:
1114  * cache-name
1115  * num-active-objs
1116  * total-objs
1117  * object size
1118  * num-active-slabs
1119  * total-slabs
1120  * num-pages-per-slab
1121  * + further values on SMP and with statistics enabled
1122  */
1123 static const struct seq_operations slabinfo_op = {
1124     .start = slab_start,
1125     .next = slab_next,
1126     .stop = slab_stop,
1127     .show = slab_show,
1128 };
1129
1130 static int slabinfo_open(struct inode *inode, struct file *file)
1131 {
1132     return seq_open(file, &slabinfo_op);
1133 }
1134
1135 static const struct proc_ops slabinfo_proc_ops = {
1136     .proc_flags = PROC_ENTRY_PERMANENT,
1137     .proc_open  = slabinfo_open,
1138     .proc_read  = seq_read,
1139     .proc_write = slabinfo_write,
1140     .proc_lseek = seq_lseek,
1141     .proc_release   = seq_release,
1142 };
1143
1144 static int __init slab_proc_init(void)
1145 {
1146     proc_create("slabinfo", SLABINFO_RIGHTS, NULL, &slabinfo_proc_ops);
1147     return 0;
1148 }
1149 module_init(slab_proc_init);
1150
1151 #endif /* CONFIG_SLAB || CONFIG_SLUB_DEBUG */
1152
1153 static __always_inline void *__do_krealloc(const void *p, size_t new_size,
1154                        gfp_t flags)
1155 {
1156     void *ret;
1157     size_t ks;
1158
1159     /* Don't use instrumented ksize to allow precise KASAN poisoning. */
1160     if (likely(!ZERO_OR_NULL_PTR(p))) {
1161         if (!kasan_check_byte(p))
1162             return NULL;
1163         ks = kfence_ksize(p) ?: __ksize(p);
1164     } else
1165         ks = 0;
1166
1167     /* If the object still fits, repoison it precisely. */
1168     if (ks >= new_size) {
1169         p = kasan_krealloc((void *)p, new_size, flags);
1170         return (void *)p;
1171     }
1172
1173     ret = kmalloc_track_caller(new_size, flags);
1174     if (ret && p) {
1175         /* Disable KASAN checks as the object's redzone is accessed. */
1176         kasan_disable_current();
1177         memcpy(ret, kasan_reset_tag(p), ks);
1178         kasan_enable_current();
1179     }
1180
1181     return ret;
1182 }
1183
1184 /**
1185  * krealloc - reallocate memory. The contents will remain unchanged.
1186  * @p: object to reallocate memory for.
1187  * @new_size: how many bytes of memory are required.
1188  * @flags: the type of memory to allocate.
1189  *
1190  * The contents of the object pointed to are preserved up to the
1191  * lesser of the new and old sizes (__GFP_ZERO flag is effectively ignored).
1192  * If @p is %NULL, krealloc() behaves exactly like kmalloc().  If @new_size
1193  * is 0 and @p is not a %NULL pointer, the object pointed to is freed.
1194  *
1195  * Return: pointer to the allocated memory or %NULL in case of error
1196  */
1197 void *krealloc(const void *p, size_t new_size, gfp_t flags)
1198 {
1199     void *ret;
1200
1201     if (unlikely(!new_size)) {
1202         kfree(p);
1203         return ZERO_SIZE_PTR;
1204     }
1205
1206     ret = __do_krealloc(p, new_size, flags);
1207     if (ret && kasan_reset_tag(p) != kasan_reset_tag(ret))
1208         kfree(p);
1209
1210     return ret;
1211 }
1212 EXPORT_SYMBOL(krealloc);
1213
1214 /**
1215  * kfree_sensitive - Clear sensitive information in memory before freeing
1216  * @p: object to free memory of
1217  *
1218  * The memory of the object @p points to is zeroed before freed.
1219  * If @p is %NULL, kfree_sensitive() does nothing.
1220  *
1221  * Note: this function zeroes the whole allocated buffer which can be a good
1222  * deal bigger than the requested buffer size passed to kmalloc(). So be
1223  * careful when using this function in performance sensitive code.
1224  */
1225 void kfree_sensitive(const void *p)
1226 {
1227     size_t ks;
1228     void *mem = (void *)p;
1229
1230     ks = ksize(mem);
1231     if (ks)
1232         memzero_explicit(mem, ks);
1233     kfree(mem);
1234 }
1235 EXPORT_SYMBOL(kfree_sensitive);
1236
1237 /**
1238  * ksize - get the actual amount of memory allocated for a given object
1239  * @objp: Pointer to the object
1240  *
1241  * kmalloc may internally round up allocations and return more memory
1242  * than requested. ksize() can be used to determine the actual amount of
1243  * memory allocated. The caller may use this additional memory, even though
1244  * a smaller amount of memory was initially specified with the kmalloc call.
1245  * The caller must guarantee that objp points to a valid object previously
1246  * allocated with either kmalloc() or kmem_cache_alloc(). The object
1247  * must not be freed during the duration of the call.
1248  *
1249  * Return: size of the actual memory used by @objp in bytes
1250  */
1251 size_t ksize(const void *objp)
1252 {
1253     size_t size;
1254
1255     /*
1256      * We need to first check that the pointer to the object is valid, and
1257      * only then unpoison the memory. The report printed from ksize() is
1258      * more useful, then when it's printed later when the behaviour could
1259      * be undefined due to a potential use-after-free or double-free.
1260      *
1261      * We use kasan_check_byte(), which is supported for the hardware
1262      * tag-based KASAN mode, unlike kasan_check_read/write().
1263      *
1264      * If the pointed to memory is invalid, we return 0 to avoid users of
1265      * ksize() writing to and potentially corrupting the memory region.
1266      *
1267      * We want to perform the check before __ksize(), to avoid potentially
1268      * crashing in __ksize() due to accessing invalid metadata.
1269      */
1270     if (unlikely(ZERO_OR_NULL_PTR(objp)) || !kasan_check_byte(objp))
1271         return 0;
1272
1273     size = kfence_ksize(objp) ?: __ksize(objp);
1274     /*
1275      * We assume that ksize callers could use whole allocated area,
1276      * so we need to unpoison this area.
1277      */
1278     kasan_unpoison_range(objp, size);
1279     return size;
1280 }
1281 EXPORT_SYMBOL(ksize);
1282
1283 /* Tracepoints definitions. */
1284 EXPORT_TRACEPOINT_SYMBOL(kmalloc);
1285 EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc);
1286 EXPORT_TRACEPOINT_SYMBOL(kmalloc_node);
1287 EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc_node);
1288 EXPORT_TRACEPOINT_SYMBOL(kfree);
1289 EXPORT_TRACEPOINT_SYMBOL(kmem_cache_free);
1290
1291 int should_failslab(struct kmem_cache *s, gfp_t gfpflags)
1292 {
1293     if (__should_failslab(s, gfpflags))
1294         return -ENOMEM;
1295     return 0;
1296 }
1297 ALLOW_ERROR_INJECTION(should_failslab, ERRNO);