Back to home page

LXR

 
 

    


0001 /*
0002  * SLUB: A slab allocator that limits cache line use instead of queuing
0003  * objects in per cpu and per node lists.
0004  *
0005  * The allocator synchronizes using per slab locks or atomic operatios
0006  * and only uses a centralized lock to manage a pool of partial slabs.
0007  *
0008  * (C) 2007 SGI, Christoph Lameter
0009  * (C) 2011 Linux Foundation, Christoph Lameter
0010  */
0011 
0012 #include <linux/mm.h>
0013 #include <linux/swap.h> /* struct reclaim_state */
0014 #include <linux/module.h>
0015 #include <linux/bit_spinlock.h>
0016 #include <linux/interrupt.h>
0017 #include <linux/bitops.h>
0018 #include <linux/slab.h>
0019 #include "slab.h"
0020 #include <linux/proc_fs.h>
0021 #include <linux/notifier.h>
0022 #include <linux/seq_file.h>
0023 #include <linux/kasan.h>
0024 #include <linux/kmemcheck.h>
0025 #include <linux/cpu.h>
0026 #include <linux/cpuset.h>
0027 #include <linux/mempolicy.h>
0028 #include <linux/ctype.h>
0029 #include <linux/debugobjects.h>
0030 #include <linux/kallsyms.h>
0031 #include <linux/memory.h>
0032 #include <linux/math64.h>
0033 #include <linux/fault-inject.h>
0034 #include <linux/stacktrace.h>
0035 #include <linux/prefetch.h>
0036 #include <linux/memcontrol.h>
0037 
0038 #include <trace/events/kmem.h>
0039 
0040 #include "internal.h"
0041 
0042 /*
0043  * Lock order:
0044  *   1. slab_mutex (Global Mutex)
0045  *   2. node->list_lock
0046  *   3. slab_lock(page) (Only on some arches and for debugging)
0047  *
0048  *   slab_mutex
0049  *
0050  *   The role of the slab_mutex is to protect the list of all the slabs
0051  *   and to synchronize major metadata changes to slab cache structures.
0052  *
0053  *   The slab_lock is only used for debugging and on arches that do not
0054  *   have the ability to do a cmpxchg_double. It only protects the second
0055  *   double word in the page struct. Meaning
0056  *  A. page->freelist   -> List of object free in a page
0057  *  B. page->counters   -> Counters of objects
0058  *  C. page->frozen     -> frozen state
0059  *
0060  *   If a slab is frozen then it is exempt from list management. It is not
0061  *   on any list. The processor that froze the slab is the one who can
0062  *   perform list operations on the page. Other processors may put objects
0063  *   onto the freelist but the processor that froze the slab is the only
0064  *   one that can retrieve the objects from the page's freelist.
0065  *
0066  *   The list_lock protects the partial and full list on each node and
0067  *   the partial slab counter. If taken then no new slabs may be added or
0068  *   removed from the lists nor make the number of partial slabs be modified.
0069  *   (Note that the total number of slabs is an atomic value that may be
0070  *   modified without taking the list lock).
0071  *
0072  *   The list_lock is a centralized lock and thus we avoid taking it as
0073  *   much as possible. As long as SLUB does not have to handle partial
0074  *   slabs, operations can continue without any centralized lock. F.e.
0075  *   allocating a long series of objects that fill up slabs does not require
0076  *   the list lock.
0077  *   Interrupts are disabled during allocation and deallocation in order to
0078  *   make the slab allocator safe to use in the context of an irq. In addition
0079  *   interrupts are disabled to ensure that the processor does not change
0080  *   while handling per_cpu slabs, due to kernel preemption.
0081  *
0082  * SLUB assigns one slab for allocation to each processor.
0083  * Allocations only occur from these slabs called cpu slabs.
0084  *
0085  * Slabs with free elements are kept on a partial list and during regular
0086  * operations no list for full slabs is used. If an object in a full slab is
0087  * freed then the slab will show up again on the partial lists.
0088  * We track full slabs for debugging purposes though because otherwise we
0089  * cannot scan all objects.
0090  *
0091  * Slabs are freed when they become empty. Teardown and setup is
0092  * minimal so we rely on the page allocators per cpu caches for
0093  * fast frees and allocs.
0094  *
0095  * Overloading of page flags that are otherwise used for LRU management.
0096  *
0097  * PageActive       The slab is frozen and exempt from list processing.
0098  *          This means that the slab is dedicated to a purpose
0099  *          such as satisfying allocations for a specific
0100  *          processor. Objects may be freed in the slab while
0101  *          it is frozen but slab_free will then skip the usual
0102  *          list operations. It is up to the processor holding
0103  *          the slab to integrate the slab into the slab lists
0104  *          when the slab is no longer needed.
0105  *
0106  *          One use of this flag is to mark slabs that are
0107  *          used for allocations. Then such a slab becomes a cpu
0108  *          slab. The cpu slab may be equipped with an additional
0109  *          freelist that allows lockless access to
0110  *          free objects in addition to the regular freelist
0111  *          that requires the slab lock.
0112  *
0113  * PageError        Slab requires special handling due to debug
0114  *          options set. This moves slab handling out of
0115  *          the fast path and disables lockless freelists.
0116  */
0117 
0118 static inline int kmem_cache_debug(struct kmem_cache *s)
0119 {
0120 #ifdef CONFIG_SLUB_DEBUG
0121     return unlikely(s->flags & SLAB_DEBUG_FLAGS);
0122 #else
0123     return 0;
0124 #endif
0125 }
0126 
0127 void *fixup_red_left(struct kmem_cache *s, void *p)
0128 {
0129     if (kmem_cache_debug(s) && s->flags & SLAB_RED_ZONE)
0130         p += s->red_left_pad;
0131 
0132     return p;
0133 }
0134 
0135 static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s)
0136 {
0137 #ifdef CONFIG_SLUB_CPU_PARTIAL
0138     return !kmem_cache_debug(s);
0139 #else
0140     return false;
0141 #endif
0142 }
0143 
0144 /*
0145  * Issues still to be resolved:
0146  *
0147  * - Support PAGE_ALLOC_DEBUG. Should be easy to do.
0148  *
0149  * - Variable sizing of the per node arrays
0150  */
0151 
0152 /* Enable to test recovery from slab corruption on boot */
0153 #undef SLUB_RESILIENCY_TEST
0154 
0155 /* Enable to log cmpxchg failures */
0156 #undef SLUB_DEBUG_CMPXCHG
0157 
0158 /*
0159  * Mininum number of partial slabs. These will be left on the partial
0160  * lists even if they are empty. kmem_cache_shrink may reclaim them.
0161  */
0162 #define MIN_PARTIAL 5
0163 
0164 /*
0165  * Maximum number of desirable partial slabs.
0166  * The existence of more partial slabs makes kmem_cache_shrink
0167  * sort the partial list by the number of objects in use.
0168  */
0169 #define MAX_PARTIAL 10
0170 
0171 #define DEBUG_DEFAULT_FLAGS (SLAB_CONSISTENCY_CHECKS | SLAB_RED_ZONE | \
0172                 SLAB_POISON | SLAB_STORE_USER)
0173 
0174 /*
0175  * These debug flags cannot use CMPXCHG because there might be consistency
0176  * issues when checking or reading debug information
0177  */
0178 #define SLAB_NO_CMPXCHG (SLAB_CONSISTENCY_CHECKS | SLAB_STORE_USER | \
0179                 SLAB_TRACE)
0180 
0181 
0182 /*
0183  * Debugging flags that require metadata to be stored in the slab.  These get
0184  * disabled when slub_debug=O is used and a cache's min order increases with
0185  * metadata.
0186  */
0187 #define DEBUG_METADATA_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER)
0188 
0189 #define OO_SHIFT    16
0190 #define OO_MASK     ((1 << OO_SHIFT) - 1)
0191 #define MAX_OBJS_PER_PAGE   32767 /* since page.objects is u15 */
0192 
0193 /* Internal SLUB flags */
0194 #define __OBJECT_POISON     0x80000000UL /* Poison object */
0195 #define __CMPXCHG_DOUBLE    0x40000000UL /* Use cmpxchg_double */
0196 
0197 /*
0198  * Tracking user of a slab.
0199  */
0200 #define TRACK_ADDRS_COUNT 16
0201 struct track {
0202     unsigned long addr; /* Called from address */
0203 #ifdef CONFIG_STACKTRACE
0204     unsigned long addrs[TRACK_ADDRS_COUNT]; /* Called from address */
0205 #endif
0206     int cpu;        /* Was running on cpu */
0207     int pid;        /* Pid context */
0208     unsigned long when; /* When did the operation occur */
0209 };
0210 
0211 enum track_item { TRACK_ALLOC, TRACK_FREE };
0212 
0213 #ifdef CONFIG_SYSFS
0214 static int sysfs_slab_add(struct kmem_cache *);
0215 static int sysfs_slab_alias(struct kmem_cache *, const char *);
0216 static void memcg_propagate_slab_attrs(struct kmem_cache *s);
0217 #else
0218 static inline int sysfs_slab_add(struct kmem_cache *s) { return 0; }
0219 static inline int sysfs_slab_alias(struct kmem_cache *s, const char *p)
0220                             { return 0; }
0221 static inline void memcg_propagate_slab_attrs(struct kmem_cache *s) { }
0222 #endif
0223 
0224 static inline void stat(const struct kmem_cache *s, enum stat_item si)
0225 {
0226 #ifdef CONFIG_SLUB_STATS
0227     /*
0228      * The rmw is racy on a preemptible kernel but this is acceptable, so
0229      * avoid this_cpu_add()'s irq-disable overhead.
0230      */
0231     raw_cpu_inc(s->cpu_slab->stat[si]);
0232 #endif
0233 }
0234 
0235 /********************************************************************
0236  *          Core slab cache functions
0237  *******************************************************************/
0238 
0239 static inline void *get_freepointer(struct kmem_cache *s, void *object)
0240 {
0241     return *(void **)(object + s->offset);
0242 }
0243 
0244 static void prefetch_freepointer(const struct kmem_cache *s, void *object)
0245 {
0246     prefetch(object + s->offset);
0247 }
0248 
0249 static inline void *get_freepointer_safe(struct kmem_cache *s, void *object)
0250 {
0251     void *p;
0252 
0253     if (!debug_pagealloc_enabled())
0254         return get_freepointer(s, object);
0255 
0256     probe_kernel_read(&p, (void **)(object + s->offset), sizeof(p));
0257     return p;
0258 }
0259 
0260 static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp)
0261 {
0262     *(void **)(object + s->offset) = fp;
0263 }
0264 
0265 /* Loop over all objects in a slab */
0266 #define for_each_object(__p, __s, __addr, __objects) \
0267     for (__p = fixup_red_left(__s, __addr); \
0268         __p < (__addr) + (__objects) * (__s)->size; \
0269         __p += (__s)->size)
0270 
0271 #define for_each_object_idx(__p, __idx, __s, __addr, __objects) \
0272     for (__p = fixup_red_left(__s, __addr), __idx = 1; \
0273         __idx <= __objects; \
0274         __p += (__s)->size, __idx++)
0275 
0276 /* Determine object index from a given position */
0277 static inline int slab_index(void *p, struct kmem_cache *s, void *addr)
0278 {
0279     return (p - addr) / s->size;
0280 }
0281 
0282 static inline int order_objects(int order, unsigned long size, int reserved)
0283 {
0284     return ((PAGE_SIZE << order) - reserved) / size;
0285 }
0286 
0287 static inline struct kmem_cache_order_objects oo_make(int order,
0288         unsigned long size, int reserved)
0289 {
0290     struct kmem_cache_order_objects x = {
0291         (order << OO_SHIFT) + order_objects(order, size, reserved)
0292     };
0293 
0294     return x;
0295 }
0296 
0297 static inline int oo_order(struct kmem_cache_order_objects x)
0298 {
0299     return x.x >> OO_SHIFT;
0300 }
0301 
0302 static inline int oo_objects(struct kmem_cache_order_objects x)
0303 {
0304     return x.x & OO_MASK;
0305 }
0306 
0307 /*
0308  * Per slab locking using the pagelock
0309  */
0310 static __always_inline void slab_lock(struct page *page)
0311 {
0312     VM_BUG_ON_PAGE(PageTail(page), page);
0313     bit_spin_lock(PG_locked, &page->flags);
0314 }
0315 
0316 static __always_inline void slab_unlock(struct page *page)
0317 {
0318     VM_BUG_ON_PAGE(PageTail(page), page);
0319     __bit_spin_unlock(PG_locked, &page->flags);
0320 }
0321 
0322 static inline void set_page_slub_counters(struct page *page, unsigned long counters_new)
0323 {
0324     struct page tmp;
0325     tmp.counters = counters_new;
0326     /*
0327      * page->counters can cover frozen/inuse/objects as well
0328      * as page->_refcount.  If we assign to ->counters directly
0329      * we run the risk of losing updates to page->_refcount, so
0330      * be careful and only assign to the fields we need.
0331      */
0332     page->frozen  = tmp.frozen;
0333     page->inuse   = tmp.inuse;
0334     page->objects = tmp.objects;
0335 }
0336 
0337 /* Interrupts must be disabled (for the fallback code to work right) */
0338 static inline bool __cmpxchg_double_slab(struct kmem_cache *s, struct page *page,
0339         void *freelist_old, unsigned long counters_old,
0340         void *freelist_new, unsigned long counters_new,
0341         const char *n)
0342 {
0343     VM_BUG_ON(!irqs_disabled());
0344 #if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
0345     defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
0346     if (s->flags & __CMPXCHG_DOUBLE) {
0347         if (cmpxchg_double(&page->freelist, &page->counters,
0348                    freelist_old, counters_old,
0349                    freelist_new, counters_new))
0350             return true;
0351     } else
0352 #endif
0353     {
0354         slab_lock(page);
0355         if (page->freelist == freelist_old &&
0356                     page->counters == counters_old) {
0357             page->freelist = freelist_new;
0358             set_page_slub_counters(page, counters_new);
0359             slab_unlock(page);
0360             return true;
0361         }
0362         slab_unlock(page);
0363     }
0364 
0365     cpu_relax();
0366     stat(s, CMPXCHG_DOUBLE_FAIL);
0367 
0368 #ifdef SLUB_DEBUG_CMPXCHG
0369     pr_info("%s %s: cmpxchg double redo ", n, s->name);
0370 #endif
0371 
0372     return false;
0373 }
0374 
0375 static inline bool cmpxchg_double_slab(struct kmem_cache *s, struct page *page,
0376         void *freelist_old, unsigned long counters_old,
0377         void *freelist_new, unsigned long counters_new,
0378         const char *n)
0379 {
0380 #if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
0381     defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
0382     if (s->flags & __CMPXCHG_DOUBLE) {
0383         if (cmpxchg_double(&page->freelist, &page->counters,
0384                    freelist_old, counters_old,
0385                    freelist_new, counters_new))
0386             return true;
0387     } else
0388 #endif
0389     {
0390         unsigned long flags;
0391 
0392         local_irq_save(flags);
0393         slab_lock(page);
0394         if (page->freelist == freelist_old &&
0395                     page->counters == counters_old) {
0396             page->freelist = freelist_new;
0397             set_page_slub_counters(page, counters_new);
0398             slab_unlock(page);
0399             local_irq_restore(flags);
0400             return true;
0401         }
0402         slab_unlock(page);
0403         local_irq_restore(flags);
0404     }
0405 
0406     cpu_relax();
0407     stat(s, CMPXCHG_DOUBLE_FAIL);
0408 
0409 #ifdef SLUB_DEBUG_CMPXCHG
0410     pr_info("%s %s: cmpxchg double redo ", n, s->name);
0411 #endif
0412 
0413     return false;
0414 }
0415 
0416 #ifdef CONFIG_SLUB_DEBUG
0417 /*
0418  * Determine a map of object in use on a page.
0419  *
0420  * Node listlock must be held to guarantee that the page does
0421  * not vanish from under us.
0422  */
0423 static void get_map(struct kmem_cache *s, struct page *page, unsigned long *map)
0424 {
0425     void *p;
0426     void *addr = page_address(page);
0427 
0428     for (p = page->freelist; p; p = get_freepointer(s, p))
0429         set_bit(slab_index(p, s, addr), map);
0430 }
0431 
0432 static inline int size_from_object(struct kmem_cache *s)
0433 {
0434     if (s->flags & SLAB_RED_ZONE)
0435         return s->size - s->red_left_pad;
0436 
0437     return s->size;
0438 }
0439 
0440 static inline void *restore_red_left(struct kmem_cache *s, void *p)
0441 {
0442     if (s->flags & SLAB_RED_ZONE)
0443         p -= s->red_left_pad;
0444 
0445     return p;
0446 }
0447 
0448 /*
0449  * Debug settings:
0450  */
0451 #if defined(CONFIG_SLUB_DEBUG_ON)
0452 static int slub_debug = DEBUG_DEFAULT_FLAGS;
0453 #else
0454 static int slub_debug;
0455 #endif
0456 
0457 static char *slub_debug_slabs;
0458 static int disable_higher_order_debug;
0459 
0460 /*
0461  * slub is about to manipulate internal object metadata.  This memory lies
0462  * outside the range of the allocated object, so accessing it would normally
0463  * be reported by kasan as a bounds error.  metadata_access_enable() is used
0464  * to tell kasan that these accesses are OK.
0465  */
0466 static inline void metadata_access_enable(void)
0467 {
0468     kasan_disable_current();
0469 }
0470 
0471 static inline void metadata_access_disable(void)
0472 {
0473     kasan_enable_current();
0474 }
0475 
0476 /*
0477  * Object debugging
0478  */
0479 
0480 /* Verify that a pointer has an address that is valid within a slab page */
0481 static inline int check_valid_pointer(struct kmem_cache *s,
0482                 struct page *page, void *object)
0483 {
0484     void *base;
0485 
0486     if (!object)
0487         return 1;
0488 
0489     base = page_address(page);
0490     object = restore_red_left(s, object);
0491     if (object < base || object >= base + page->objects * s->size ||
0492         (object - base) % s->size) {
0493         return 0;
0494     }
0495 
0496     return 1;
0497 }
0498 
0499 static void print_section(char *level, char *text, u8 *addr,
0500               unsigned int length)
0501 {
0502     metadata_access_enable();
0503     print_hex_dump(level, text, DUMP_PREFIX_ADDRESS, 16, 1, addr,
0504             length, 1);
0505     metadata_access_disable();
0506 }
0507 
0508 static struct track *get_track(struct kmem_cache *s, void *object,
0509     enum track_item alloc)
0510 {
0511     struct track *p;
0512 
0513     if (s->offset)
0514         p = object + s->offset + sizeof(void *);
0515     else
0516         p = object + s->inuse;
0517 
0518     return p + alloc;
0519 }
0520 
0521 static void set_track(struct kmem_cache *s, void *object,
0522             enum track_item alloc, unsigned long addr)
0523 {
0524     struct track *p = get_track(s, object, alloc);
0525 
0526     if (addr) {
0527 #ifdef CONFIG_STACKTRACE
0528         struct stack_trace trace;
0529         int i;
0530 
0531         trace.nr_entries = 0;
0532         trace.max_entries = TRACK_ADDRS_COUNT;
0533         trace.entries = p->addrs;
0534         trace.skip = 3;
0535         metadata_access_enable();
0536         save_stack_trace(&trace);
0537         metadata_access_disable();
0538 
0539         /* See rant in lockdep.c */
0540         if (trace.nr_entries != 0 &&
0541             trace.entries[trace.nr_entries - 1] == ULONG_MAX)
0542             trace.nr_entries--;
0543 
0544         for (i = trace.nr_entries; i < TRACK_ADDRS_COUNT; i++)
0545             p->addrs[i] = 0;
0546 #endif
0547         p->addr = addr;
0548         p->cpu = smp_processor_id();
0549         p->pid = current->pid;
0550         p->when = jiffies;
0551     } else
0552         memset(p, 0, sizeof(struct track));
0553 }
0554 
0555 static void init_tracking(struct kmem_cache *s, void *object)
0556 {
0557     if (!(s->flags & SLAB_STORE_USER))
0558         return;
0559 
0560     set_track(s, object, TRACK_FREE, 0UL);
0561     set_track(s, object, TRACK_ALLOC, 0UL);
0562 }
0563 
0564 static void print_track(const char *s, struct track *t)
0565 {
0566     if (!t->addr)
0567         return;
0568 
0569     pr_err("INFO: %s in %pS age=%lu cpu=%u pid=%d\n",
0570            s, (void *)t->addr, jiffies - t->when, t->cpu, t->pid);
0571 #ifdef CONFIG_STACKTRACE
0572     {
0573         int i;
0574         for (i = 0; i < TRACK_ADDRS_COUNT; i++)
0575             if (t->addrs[i])
0576                 pr_err("\t%pS\n", (void *)t->addrs[i]);
0577             else
0578                 break;
0579     }
0580 #endif
0581 }
0582 
0583 static void print_tracking(struct kmem_cache *s, void *object)
0584 {
0585     if (!(s->flags & SLAB_STORE_USER))
0586         return;
0587 
0588     print_track("Allocated", get_track(s, object, TRACK_ALLOC));
0589     print_track("Freed", get_track(s, object, TRACK_FREE));
0590 }
0591 
0592 static void print_page_info(struct page *page)
0593 {
0594     pr_err("INFO: Slab 0x%p objects=%u used=%u fp=0x%p flags=0x%04lx\n",
0595            page, page->objects, page->inuse, page->freelist, page->flags);
0596 
0597 }
0598 
0599 static void slab_bug(struct kmem_cache *s, char *fmt, ...)
0600 {
0601     struct va_format vaf;
0602     va_list args;
0603 
0604     va_start(args, fmt);
0605     vaf.fmt = fmt;
0606     vaf.va = &args;
0607     pr_err("=============================================================================\n");
0608     pr_err("BUG %s (%s): %pV\n", s->name, print_tainted(), &vaf);
0609     pr_err("-----------------------------------------------------------------------------\n\n");
0610 
0611     add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
0612     va_end(args);
0613 }
0614 
0615 static void slab_fix(struct kmem_cache *s, char *fmt, ...)
0616 {
0617     struct va_format vaf;
0618     va_list args;
0619 
0620     va_start(args, fmt);
0621     vaf.fmt = fmt;
0622     vaf.va = &args;
0623     pr_err("FIX %s: %pV\n", s->name, &vaf);
0624     va_end(args);
0625 }
0626 
0627 static void print_trailer(struct kmem_cache *s, struct page *page, u8 *p)
0628 {
0629     unsigned int off;   /* Offset of last byte */
0630     u8 *addr = page_address(page);
0631 
0632     print_tracking(s, p);
0633 
0634     print_page_info(page);
0635 
0636     pr_err("INFO: Object 0x%p @offset=%tu fp=0x%p\n\n",
0637            p, p - addr, get_freepointer(s, p));
0638 
0639     if (s->flags & SLAB_RED_ZONE)
0640         print_section(KERN_ERR, "Redzone ", p - s->red_left_pad,
0641                   s->red_left_pad);
0642     else if (p > addr + 16)
0643         print_section(KERN_ERR, "Bytes b4 ", p - 16, 16);
0644 
0645     print_section(KERN_ERR, "Object ", p,
0646               min_t(unsigned long, s->object_size, PAGE_SIZE));
0647     if (s->flags & SLAB_RED_ZONE)
0648         print_section(KERN_ERR, "Redzone ", p + s->object_size,
0649             s->inuse - s->object_size);
0650 
0651     if (s->offset)
0652         off = s->offset + sizeof(void *);
0653     else
0654         off = s->inuse;
0655 
0656     if (s->flags & SLAB_STORE_USER)
0657         off += 2 * sizeof(struct track);
0658 
0659     off += kasan_metadata_size(s);
0660 
0661     if (off != size_from_object(s))
0662         /* Beginning of the filler is the free pointer */
0663         print_section(KERN_ERR, "Padding ", p + off,
0664                   size_from_object(s) - off);
0665 
0666     dump_stack();
0667 }
0668 
0669 void object_err(struct kmem_cache *s, struct page *page,
0670             u8 *object, char *reason)
0671 {
0672     slab_bug(s, "%s", reason);
0673     print_trailer(s, page, object);
0674 }
0675 
0676 static void slab_err(struct kmem_cache *s, struct page *page,
0677             const char *fmt, ...)
0678 {
0679     va_list args;
0680     char buf[100];
0681 
0682     va_start(args, fmt);
0683     vsnprintf(buf, sizeof(buf), fmt, args);
0684     va_end(args);
0685     slab_bug(s, "%s", buf);
0686     print_page_info(page);
0687     dump_stack();
0688 }
0689 
0690 static void init_object(struct kmem_cache *s, void *object, u8 val)
0691 {
0692     u8 *p = object;
0693 
0694     if (s->flags & SLAB_RED_ZONE)
0695         memset(p - s->red_left_pad, val, s->red_left_pad);
0696 
0697     if (s->flags & __OBJECT_POISON) {
0698         memset(p, POISON_FREE, s->object_size - 1);
0699         p[s->object_size - 1] = POISON_END;
0700     }
0701 
0702     if (s->flags & SLAB_RED_ZONE)
0703         memset(p + s->object_size, val, s->inuse - s->object_size);
0704 }
0705 
0706 static void restore_bytes(struct kmem_cache *s, char *message, u8 data,
0707                         void *from, void *to)
0708 {
0709     slab_fix(s, "Restoring 0x%p-0x%p=0x%x\n", from, to - 1, data);
0710     memset(from, data, to - from);
0711 }
0712 
0713 static int check_bytes_and_report(struct kmem_cache *s, struct page *page,
0714             u8 *object, char *what,
0715             u8 *start, unsigned int value, unsigned int bytes)
0716 {
0717     u8 *fault;
0718     u8 *end;
0719 
0720     metadata_access_enable();
0721     fault = memchr_inv(start, value, bytes);
0722     metadata_access_disable();
0723     if (!fault)
0724         return 1;
0725 
0726     end = start + bytes;
0727     while (end > fault && end[-1] == value)
0728         end--;
0729 
0730     slab_bug(s, "%s overwritten", what);
0731     pr_err("INFO: 0x%p-0x%p. First byte 0x%x instead of 0x%x\n",
0732                     fault, end - 1, fault[0], value);
0733     print_trailer(s, page, object);
0734 
0735     restore_bytes(s, what, value, fault, end);
0736     return 0;
0737 }
0738 
0739 /*
0740  * Object layout:
0741  *
0742  * object address
0743  *  Bytes of the object to be managed.
0744  *  If the freepointer may overlay the object then the free
0745  *  pointer is the first word of the object.
0746  *
0747  *  Poisoning uses 0x6b (POISON_FREE) and the last byte is
0748  *  0xa5 (POISON_END)
0749  *
0750  * object + s->object_size
0751  *  Padding to reach word boundary. This is also used for Redzoning.
0752  *  Padding is extended by another word if Redzoning is enabled and
0753  *  object_size == inuse.
0754  *
0755  *  We fill with 0xbb (RED_INACTIVE) for inactive objects and with
0756  *  0xcc (RED_ACTIVE) for objects in use.
0757  *
0758  * object + s->inuse
0759  *  Meta data starts here.
0760  *
0761  *  A. Free pointer (if we cannot overwrite object on free)
0762  *  B. Tracking data for SLAB_STORE_USER
0763  *  C. Padding to reach required alignment boundary or at mininum
0764  *      one word if debugging is on to be able to detect writes
0765  *      before the word boundary.
0766  *
0767  *  Padding is done using 0x5a (POISON_INUSE)
0768  *
0769  * object + s->size
0770  *  Nothing is used beyond s->size.
0771  *
0772  * If slabcaches are merged then the object_size and inuse boundaries are mostly
0773  * ignored. And therefore no slab options that rely on these boundaries
0774  * may be used with merged slabcaches.
0775  */
0776 
0777 static int check_pad_bytes(struct kmem_cache *s, struct page *page, u8 *p)
0778 {
0779     unsigned long off = s->inuse;   /* The end of info */
0780 
0781     if (s->offset)
0782         /* Freepointer is placed after the object. */
0783         off += sizeof(void *);
0784 
0785     if (s->flags & SLAB_STORE_USER)
0786         /* We also have user information there */
0787         off += 2 * sizeof(struct track);
0788 
0789     off += kasan_metadata_size(s);
0790 
0791     if (size_from_object(s) == off)
0792         return 1;
0793 
0794     return check_bytes_and_report(s, page, p, "Object padding",
0795             p + off, POISON_INUSE, size_from_object(s) - off);
0796 }
0797 
0798 /* Check the pad bytes at the end of a slab page */
0799 static int slab_pad_check(struct kmem_cache *s, struct page *page)
0800 {
0801     u8 *start;
0802     u8 *fault;
0803     u8 *end;
0804     int length;
0805     int remainder;
0806 
0807     if (!(s->flags & SLAB_POISON))
0808         return 1;
0809 
0810     start = page_address(page);
0811     length = (PAGE_SIZE << compound_order(page)) - s->reserved;
0812     end = start + length;
0813     remainder = length % s->size;
0814     if (!remainder)
0815         return 1;
0816 
0817     metadata_access_enable();
0818     fault = memchr_inv(end - remainder, POISON_INUSE, remainder);
0819     metadata_access_disable();
0820     if (!fault)
0821         return 1;
0822     while (end > fault && end[-1] == POISON_INUSE)
0823         end--;
0824 
0825     slab_err(s, page, "Padding overwritten. 0x%p-0x%p", fault, end - 1);
0826     print_section(KERN_ERR, "Padding ", end - remainder, remainder);
0827 
0828     restore_bytes(s, "slab padding", POISON_INUSE, end - remainder, end);
0829     return 0;
0830 }
0831 
0832 static int check_object(struct kmem_cache *s, struct page *page,
0833                     void *object, u8 val)
0834 {
0835     u8 *p = object;
0836     u8 *endobject = object + s->object_size;
0837 
0838     if (s->flags & SLAB_RED_ZONE) {
0839         if (!check_bytes_and_report(s, page, object, "Redzone",
0840             object - s->red_left_pad, val, s->red_left_pad))
0841             return 0;
0842 
0843         if (!check_bytes_and_report(s, page, object, "Redzone",
0844             endobject, val, s->inuse - s->object_size))
0845             return 0;
0846     } else {
0847         if ((s->flags & SLAB_POISON) && s->object_size < s->inuse) {
0848             check_bytes_and_report(s, page, p, "Alignment padding",
0849                 endobject, POISON_INUSE,
0850                 s->inuse - s->object_size);
0851         }
0852     }
0853 
0854     if (s->flags & SLAB_POISON) {
0855         if (val != SLUB_RED_ACTIVE && (s->flags & __OBJECT_POISON) &&
0856             (!check_bytes_and_report(s, page, p, "Poison", p,
0857                     POISON_FREE, s->object_size - 1) ||
0858              !check_bytes_and_report(s, page, p, "Poison",
0859                 p + s->object_size - 1, POISON_END, 1)))
0860             return 0;
0861         /*
0862          * check_pad_bytes cleans up on its own.
0863          */
0864         check_pad_bytes(s, page, p);
0865     }
0866 
0867     if (!s->offset && val == SLUB_RED_ACTIVE)
0868         /*
0869          * Object and freepointer overlap. Cannot check
0870          * freepointer while object is allocated.
0871          */
0872         return 1;
0873 
0874     /* Check free pointer validity */
0875     if (!check_valid_pointer(s, page, get_freepointer(s, p))) {
0876         object_err(s, page, p, "Freepointer corrupt");
0877         /*
0878          * No choice but to zap it and thus lose the remainder
0879          * of the free objects in this slab. May cause
0880          * another error because the object count is now wrong.
0881          */
0882         set_freepointer(s, p, NULL);
0883         return 0;
0884     }
0885     return 1;
0886 }
0887 
0888 static int check_slab(struct kmem_cache *s, struct page *page)
0889 {
0890     int maxobj;
0891 
0892     VM_BUG_ON(!irqs_disabled());
0893 
0894     if (!PageSlab(page)) {
0895         slab_err(s, page, "Not a valid slab page");
0896         return 0;
0897     }
0898 
0899     maxobj = order_objects(compound_order(page), s->size, s->reserved);
0900     if (page->objects > maxobj) {
0901         slab_err(s, page, "objects %u > max %u",
0902             page->objects, maxobj);
0903         return 0;
0904     }
0905     if (page->inuse > page->objects) {
0906         slab_err(s, page, "inuse %u > max %u",
0907             page->inuse, page->objects);
0908         return 0;
0909     }
0910     /* Slab_pad_check fixes things up after itself */
0911     slab_pad_check(s, page);
0912     return 1;
0913 }
0914 
0915 /*
0916  * Determine if a certain object on a page is on the freelist. Must hold the
0917  * slab lock to guarantee that the chains are in a consistent state.
0918  */
0919 static int on_freelist(struct kmem_cache *s, struct page *page, void *search)
0920 {
0921     int nr = 0;
0922     void *fp;
0923     void *object = NULL;
0924     int max_objects;
0925 
0926     fp = page->freelist;
0927     while (fp && nr <= page->objects) {
0928         if (fp == search)
0929             return 1;
0930         if (!check_valid_pointer(s, page, fp)) {
0931             if (object) {
0932                 object_err(s, page, object,
0933                     "Freechain corrupt");
0934                 set_freepointer(s, object, NULL);
0935             } else {
0936                 slab_err(s, page, "Freepointer corrupt");
0937                 page->freelist = NULL;
0938                 page->inuse = page->objects;
0939                 slab_fix(s, "Freelist cleared");
0940                 return 0;
0941             }
0942             break;
0943         }
0944         object = fp;
0945         fp = get_freepointer(s, object);
0946         nr++;
0947     }
0948 
0949     max_objects = order_objects(compound_order(page), s->size, s->reserved);
0950     if (max_objects > MAX_OBJS_PER_PAGE)
0951         max_objects = MAX_OBJS_PER_PAGE;
0952 
0953     if (page->objects != max_objects) {
0954         slab_err(s, page, "Wrong number of objects. Found %d but should be %d",
0955              page->objects, max_objects);
0956         page->objects = max_objects;
0957         slab_fix(s, "Number of objects adjusted.");
0958     }
0959     if (page->inuse != page->objects - nr) {
0960         slab_err(s, page, "Wrong object count. Counter is %d but counted were %d",
0961              page->inuse, page->objects - nr);
0962         page->inuse = page->objects - nr;
0963         slab_fix(s, "Object count adjusted.");
0964     }
0965     return search == NULL;
0966 }
0967 
0968 static void trace(struct kmem_cache *s, struct page *page, void *object,
0969                                 int alloc)
0970 {
0971     if (s->flags & SLAB_TRACE) {
0972         pr_info("TRACE %s %s 0x%p inuse=%d fp=0x%p\n",
0973             s->name,
0974             alloc ? "alloc" : "free",
0975             object, page->inuse,
0976             page->freelist);
0977 
0978         if (!alloc)
0979             print_section(KERN_INFO, "Object ", (void *)object,
0980                     s->object_size);
0981 
0982         dump_stack();
0983     }
0984 }
0985 
0986 /*
0987  * Tracking of fully allocated slabs for debugging purposes.
0988  */
0989 static void add_full(struct kmem_cache *s,
0990     struct kmem_cache_node *n, struct page *page)
0991 {
0992     if (!(s->flags & SLAB_STORE_USER))
0993         return;
0994 
0995     lockdep_assert_held(&n->list_lock);
0996     list_add(&page->lru, &n->full);
0997 }
0998 
0999 static void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, struct page *page)
1000 {
1001     if (!(s->flags & SLAB_STORE_USER))
1002         return;
1003 
1004     lockdep_assert_held(&n->list_lock);
1005     list_del(&page->lru);
1006 }
1007 
1008 /* Tracking of the number of slabs for debugging purposes */
1009 static inline unsigned long slabs_node(struct kmem_cache *s, int node)
1010 {
1011     struct kmem_cache_node *n = get_node(s, node);
1012 
1013     return atomic_long_read(&n->nr_slabs);
1014 }
1015 
1016 static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
1017 {
1018     return atomic_long_read(&n->nr_slabs);
1019 }
1020 
1021 static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects)
1022 {
1023     struct kmem_cache_node *n = get_node(s, node);
1024 
1025     /*
1026      * May be called early in order to allocate a slab for the
1027      * kmem_cache_node structure. Solve the chicken-egg
1028      * dilemma by deferring the increment of the count during
1029      * bootstrap (see early_kmem_cache_node_alloc).
1030      */
1031     if (likely(n)) {
1032         atomic_long_inc(&n->nr_slabs);
1033         atomic_long_add(objects, &n->total_objects);
1034     }
1035 }
1036 static inline void dec_slabs_node(struct kmem_cache *s, int node, int objects)
1037 {
1038     struct kmem_cache_node *n = get_node(s, node);
1039 
1040     atomic_long_dec(&n->nr_slabs);
1041     atomic_long_sub(objects, &n->total_objects);
1042 }
1043 
1044 /* Object debug checks for alloc/free paths */
1045 static void setup_object_debug(struct kmem_cache *s, struct page *page,
1046                                 void *object)
1047 {
1048     if (!(s->flags & (SLAB_STORE_USER|SLAB_RED_ZONE|__OBJECT_POISON)))
1049         return;
1050 
1051     init_object(s, object, SLUB_RED_INACTIVE);
1052     init_tracking(s, object);
1053 }
1054 
1055 static inline int alloc_consistency_checks(struct kmem_cache *s,
1056                     struct page *page,
1057                     void *object, unsigned long addr)
1058 {
1059     if (!check_slab(s, page))
1060         return 0;
1061 
1062     if (!check_valid_pointer(s, page, object)) {
1063         object_err(s, page, object, "Freelist Pointer check fails");
1064         return 0;
1065     }
1066 
1067     if (!check_object(s, page, object, SLUB_RED_INACTIVE))
1068         return 0;
1069 
1070     return 1;
1071 }
1072 
1073 static noinline int alloc_debug_processing(struct kmem_cache *s,
1074                     struct page *page,
1075                     void *object, unsigned long addr)
1076 {
1077     if (s->flags & SLAB_CONSISTENCY_CHECKS) {
1078         if (!alloc_consistency_checks(s, page, object, addr))
1079             goto bad;
1080     }
1081 
1082     /* Success perform special debug activities for allocs */
1083     if (s->flags & SLAB_STORE_USER)
1084         set_track(s, object, TRACK_ALLOC, addr);
1085     trace(s, page, object, 1);
1086     init_object(s, object, SLUB_RED_ACTIVE);
1087     return 1;
1088 
1089 bad:
1090     if (PageSlab(page)) {
1091         /*
1092          * If this is a slab page then lets do the best we can
1093          * to avoid issues in the future. Marking all objects
1094          * as used avoids touching the remaining objects.
1095          */
1096         slab_fix(s, "Marking all objects used");
1097         page->inuse = page->objects;
1098         page->freelist = NULL;
1099     }
1100     return 0;
1101 }
1102 
1103 static inline int free_consistency_checks(struct kmem_cache *s,
1104         struct page *page, void *object, unsigned long addr)
1105 {
1106     if (!check_valid_pointer(s, page, object)) {
1107         slab_err(s, page, "Invalid object pointer 0x%p", object);
1108         return 0;
1109     }
1110 
1111     if (on_freelist(s, page, object)) {
1112         object_err(s, page, object, "Object already free");
1113         return 0;
1114     }
1115 
1116     if (!check_object(s, page, object, SLUB_RED_ACTIVE))
1117         return 0;
1118 
1119     if (unlikely(s != page->slab_cache)) {
1120         if (!PageSlab(page)) {
1121             slab_err(s, page, "Attempt to free object(0x%p) outside of slab",
1122                  object);
1123         } else if (!page->slab_cache) {
1124             pr_err("SLUB <none>: no slab for object 0x%p.\n",
1125                    object);
1126             dump_stack();
1127         } else
1128             object_err(s, page, object,
1129                     "page slab pointer corrupt.");
1130         return 0;
1131     }
1132     return 1;
1133 }
1134 
1135 /* Supports checking bulk free of a constructed freelist */
1136 static noinline int free_debug_processing(
1137     struct kmem_cache *s, struct page *page,
1138     void *head, void *tail, int bulk_cnt,
1139     unsigned long addr)
1140 {
1141     struct kmem_cache_node *n = get_node(s, page_to_nid(page));
1142     void *object = head;
1143     int cnt = 0;
1144     unsigned long uninitialized_var(flags);
1145     int ret = 0;
1146 
1147     spin_lock_irqsave(&n->list_lock, flags);
1148     slab_lock(page);
1149 
1150     if (s->flags & SLAB_CONSISTENCY_CHECKS) {
1151         if (!check_slab(s, page))
1152             goto out;
1153     }
1154 
1155 next_object:
1156     cnt++;
1157 
1158     if (s->flags & SLAB_CONSISTENCY_CHECKS) {
1159         if (!free_consistency_checks(s, page, object, addr))
1160             goto out;
1161     }
1162 
1163     if (s->flags & SLAB_STORE_USER)
1164         set_track(s, object, TRACK_FREE, addr);
1165     trace(s, page, object, 0);
1166     /* Freepointer not overwritten by init_object(), SLAB_POISON moved it */
1167     init_object(s, object, SLUB_RED_INACTIVE);
1168 
1169     /* Reached end of constructed freelist yet? */
1170     if (object != tail) {
1171         object = get_freepointer(s, object);
1172         goto next_object;
1173     }
1174     ret = 1;
1175 
1176 out:
1177     if (cnt != bulk_cnt)
1178         slab_err(s, page, "Bulk freelist count(%d) invalid(%d)\n",
1179              bulk_cnt, cnt);
1180 
1181     slab_unlock(page);
1182     spin_unlock_irqrestore(&n->list_lock, flags);
1183     if (!ret)
1184         slab_fix(s, "Object at 0x%p not freed", object);
1185     return ret;
1186 }
1187 
1188 static int __init setup_slub_debug(char *str)
1189 {
1190     slub_debug = DEBUG_DEFAULT_FLAGS;
1191     if (*str++ != '=' || !*str)
1192         /*
1193          * No options specified. Switch on full debugging.
1194          */
1195         goto out;
1196 
1197     if (*str == ',')
1198         /*
1199          * No options but restriction on slabs. This means full
1200          * debugging for slabs matching a pattern.
1201          */
1202         goto check_slabs;
1203 
1204     slub_debug = 0;
1205     if (*str == '-')
1206         /*
1207          * Switch off all debugging measures.
1208          */
1209         goto out;
1210 
1211     /*
1212      * Determine which debug features should be switched on
1213      */
1214     for (; *str && *str != ','; str++) {
1215         switch (tolower(*str)) {
1216         case 'f':
1217             slub_debug |= SLAB_CONSISTENCY_CHECKS;
1218             break;
1219         case 'z':
1220             slub_debug |= SLAB_RED_ZONE;
1221             break;
1222         case 'p':
1223             slub_debug |= SLAB_POISON;
1224             break;
1225         case 'u':
1226             slub_debug |= SLAB_STORE_USER;
1227             break;
1228         case 't':
1229             slub_debug |= SLAB_TRACE;
1230             break;
1231         case 'a':
1232             slub_debug |= SLAB_FAILSLAB;
1233             break;
1234         case 'o':
1235             /*
1236              * Avoid enabling debugging on caches if its minimum
1237              * order would increase as a result.
1238              */
1239             disable_higher_order_debug = 1;
1240             break;
1241         default:
1242             pr_err("slub_debug option '%c' unknown. skipped\n",
1243                    *str);
1244         }
1245     }
1246 
1247 check_slabs:
1248     if (*str == ',')
1249         slub_debug_slabs = str + 1;
1250 out:
1251     return 1;
1252 }
1253 
1254 __setup("slub_debug", setup_slub_debug);
1255 
1256 unsigned long kmem_cache_flags(unsigned long object_size,
1257     unsigned long flags, const char *name,
1258     void (*ctor)(void *))
1259 {
1260     /*
1261      * Enable debugging if selected on the kernel commandline.
1262      */
1263     if (slub_debug && (!slub_debug_slabs || (name &&
1264         !strncmp(slub_debug_slabs, name, strlen(slub_debug_slabs)))))
1265         flags |= slub_debug;
1266 
1267     return flags;
1268 }
1269 #else /* !CONFIG_SLUB_DEBUG */
1270 static inline void setup_object_debug(struct kmem_cache *s,
1271             struct page *page, void *object) {}
1272 
1273 static inline int alloc_debug_processing(struct kmem_cache *s,
1274     struct page *page, void *object, unsigned long addr) { return 0; }
1275 
1276 static inline int free_debug_processing(
1277     struct kmem_cache *s, struct page *page,
1278     void *head, void *tail, int bulk_cnt,
1279     unsigned long addr) { return 0; }
1280 
1281 static inline int slab_pad_check(struct kmem_cache *s, struct page *page)
1282             { return 1; }
1283 static inline int check_object(struct kmem_cache *s, struct page *page,
1284             void *object, u8 val) { return 1; }
1285 static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n,
1286                     struct page *page) {}
1287 static inline void remove_full(struct kmem_cache *s, struct kmem_cache_node *n,
1288                     struct page *page) {}
1289 unsigned long kmem_cache_flags(unsigned long object_size,
1290     unsigned long flags, const char *name,
1291     void (*ctor)(void *))
1292 {
1293     return flags;
1294 }
1295 #define slub_debug 0
1296 
1297 #define disable_higher_order_debug 0
1298 
1299 static inline unsigned long slabs_node(struct kmem_cache *s, int node)
1300                             { return 0; }
1301 static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
1302                             { return 0; }
1303 static inline void inc_slabs_node(struct kmem_cache *s, int node,
1304                             int objects) {}
1305 static inline void dec_slabs_node(struct kmem_cache *s, int node,
1306                             int objects) {}
1307 
1308 #endif /* CONFIG_SLUB_DEBUG */
1309 
1310 /*
1311  * Hooks for other subsystems that check memory allocations. In a typical
1312  * production configuration these hooks all should produce no code at all.
1313  */
1314 static inline void kmalloc_large_node_hook(void *ptr, size_t size, gfp_t flags)
1315 {
1316     kmemleak_alloc(ptr, size, 1, flags);
1317     kasan_kmalloc_large(ptr, size, flags);
1318 }
1319 
1320 static inline void kfree_hook(const void *x)
1321 {
1322     kmemleak_free(x);
1323     kasan_kfree_large(x);
1324 }
1325 
1326 static inline void *slab_free_hook(struct kmem_cache *s, void *x)
1327 {
1328     void *freeptr;
1329 
1330     kmemleak_free_recursive(x, s->flags);
1331 
1332     /*
1333      * Trouble is that we may no longer disable interrupts in the fast path
1334      * So in order to make the debug calls that expect irqs to be
1335      * disabled we need to disable interrupts temporarily.
1336      */
1337 #if defined(CONFIG_KMEMCHECK) || defined(CONFIG_LOCKDEP)
1338     {
1339         unsigned long flags;
1340 
1341         local_irq_save(flags);
1342         kmemcheck_slab_free(s, x, s->object_size);
1343         debug_check_no_locks_freed(x, s->object_size);
1344         local_irq_restore(flags);
1345     }
1346 #endif
1347     if (!(s->flags & SLAB_DEBUG_OBJECTS))
1348         debug_check_no_obj_freed(x, s->object_size);
1349 
1350     freeptr = get_freepointer(s, x);
1351     /*
1352      * kasan_slab_free() may put x into memory quarantine, delaying its
1353      * reuse. In this case the object's freelist pointer is changed.
1354      */
1355     kasan_slab_free(s, x);
1356     return freeptr;
1357 }
1358 
1359 static inline void slab_free_freelist_hook(struct kmem_cache *s,
1360                        void *head, void *tail)
1361 {
1362 /*
1363  * Compiler cannot detect this function can be removed if slab_free_hook()
1364  * evaluates to nothing.  Thus, catch all relevant config debug options here.
1365  */
1366 #if defined(CONFIG_KMEMCHECK) ||        \
1367     defined(CONFIG_LOCKDEP) ||      \
1368     defined(CONFIG_DEBUG_KMEMLEAK) ||   \
1369     defined(CONFIG_DEBUG_OBJECTS_FREE) ||   \
1370     defined(CONFIG_KASAN)
1371 
1372     void *object = head;
1373     void *tail_obj = tail ? : head;
1374     void *freeptr;
1375 
1376     do {
1377         freeptr = slab_free_hook(s, object);
1378     } while ((object != tail_obj) && (object = freeptr));
1379 #endif
1380 }
1381 
1382 static void setup_object(struct kmem_cache *s, struct page *page,
1383                 void *object)
1384 {
1385     setup_object_debug(s, page, object);
1386     kasan_init_slab_obj(s, object);
1387     if (unlikely(s->ctor)) {
1388         kasan_unpoison_object_data(s, object);
1389         s->ctor(object);
1390         kasan_poison_object_data(s, object);
1391     }
1392 }
1393 
1394 /*
1395  * Slab allocation and freeing
1396  */
1397 static inline struct page *alloc_slab_page(struct kmem_cache *s,
1398         gfp_t flags, int node, struct kmem_cache_order_objects oo)
1399 {
1400     struct page *page;
1401     int order = oo_order(oo);
1402 
1403     flags |= __GFP_NOTRACK;
1404 
1405     if (node == NUMA_NO_NODE)
1406         page = alloc_pages(flags, order);
1407     else
1408         page = __alloc_pages_node(node, flags, order);
1409 
1410     if (page && memcg_charge_slab(page, flags, order, s)) {
1411         __free_pages(page, order);
1412         page = NULL;
1413     }
1414 
1415     return page;
1416 }
1417 
1418 #ifdef CONFIG_SLAB_FREELIST_RANDOM
1419 /* Pre-initialize the random sequence cache */
1420 static int init_cache_random_seq(struct kmem_cache *s)
1421 {
1422     int err;
1423     unsigned long i, count = oo_objects(s->oo);
1424 
1425     /* Bailout if already initialised */
1426     if (s->random_seq)
1427         return 0;
1428 
1429     err = cache_random_seq_create(s, count, GFP_KERNEL);
1430     if (err) {
1431         pr_err("SLUB: Unable to initialize free list for %s\n",
1432             s->name);
1433         return err;
1434     }
1435 
1436     /* Transform to an offset on the set of pages */
1437     if (s->random_seq) {
1438         for (i = 0; i < count; i++)
1439             s->random_seq[i] *= s->size;
1440     }
1441     return 0;
1442 }
1443 
1444 /* Initialize each random sequence freelist per cache */
1445 static void __init init_freelist_randomization(void)
1446 {
1447     struct kmem_cache *s;
1448 
1449     mutex_lock(&slab_mutex);
1450 
1451     list_for_each_entry(s, &slab_caches, list)
1452         init_cache_random_seq(s);
1453 
1454     mutex_unlock(&slab_mutex);
1455 }
1456 
1457 /* Get the next entry on the pre-computed freelist randomized */
1458 static void *next_freelist_entry(struct kmem_cache *s, struct page *page,
1459                 unsigned long *pos, void *start,
1460                 unsigned long page_limit,
1461                 unsigned long freelist_count)
1462 {
1463     unsigned int idx;
1464 
1465     /*
1466      * If the target page allocation failed, the number of objects on the
1467      * page might be smaller than the usual size defined by the cache.
1468      */
1469     do {
1470         idx = s->random_seq[*pos];
1471         *pos += 1;
1472         if (*pos >= freelist_count)
1473             *pos = 0;
1474     } while (unlikely(idx >= page_limit));
1475 
1476     return (char *)start + idx;
1477 }
1478 
1479 /* Shuffle the single linked freelist based on a random pre-computed sequence */
1480 static bool shuffle_freelist(struct kmem_cache *s, struct page *page)
1481 {
1482     void *start;
1483     void *cur;
1484     void *next;
1485     unsigned long idx, pos, page_limit, freelist_count;
1486 
1487     if (page->objects < 2 || !s->random_seq)
1488         return false;
1489 
1490     freelist_count = oo_objects(s->oo);
1491     pos = get_random_int() % freelist_count;
1492 
1493     page_limit = page->objects * s->size;
1494     start = fixup_red_left(s, page_address(page));
1495 
1496     /* First entry is used as the base of the freelist */
1497     cur = next_freelist_entry(s, page, &pos, start, page_limit,
1498                 freelist_count);
1499     page->freelist = cur;
1500 
1501     for (idx = 1; idx < page->objects; idx++) {
1502         setup_object(s, page, cur);
1503         next = next_freelist_entry(s, page, &pos, start, page_limit,
1504             freelist_count);
1505         set_freepointer(s, cur, next);
1506         cur = next;
1507     }
1508     setup_object(s, page, cur);
1509     set_freepointer(s, cur, NULL);
1510 
1511     return true;
1512 }
1513 #else
1514 static inline int init_cache_random_seq(struct kmem_cache *s)
1515 {
1516     return 0;
1517 }
1518 static inline void init_freelist_randomization(void) { }
1519 static inline bool shuffle_freelist(struct kmem_cache *s, struct page *page)
1520 {
1521     return false;
1522 }
1523 #endif /* CONFIG_SLAB_FREELIST_RANDOM */
1524 
1525 static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
1526 {
1527     struct page *page;
1528     struct kmem_cache_order_objects oo = s->oo;
1529     gfp_t alloc_gfp;
1530     void *start, *p;
1531     int idx, order;
1532     bool shuffle;
1533 
1534     flags &= gfp_allowed_mask;
1535 
1536     if (gfpflags_allow_blocking(flags))
1537         local_irq_enable();
1538 
1539     flags |= s->allocflags;
1540 
1541     /*
1542      * Let the initial higher-order allocation fail under memory pressure
1543      * so we fall-back to the minimum order allocation.
1544      */
1545     alloc_gfp = (flags | __GFP_NOWARN | __GFP_NORETRY) & ~__GFP_NOFAIL;
1546     if ((alloc_gfp & __GFP_DIRECT_RECLAIM) && oo_order(oo) > oo_order(s->min))
1547         alloc_gfp = (alloc_gfp | __GFP_NOMEMALLOC) & ~(__GFP_RECLAIM|__GFP_NOFAIL);
1548 
1549     page = alloc_slab_page(s, alloc_gfp, node, oo);
1550     if (unlikely(!page)) {
1551         oo = s->min;
1552         alloc_gfp = flags;
1553         /*
1554          * Allocation may have failed due to fragmentation.
1555          * Try a lower order alloc if possible
1556          */
1557         page = alloc_slab_page(s, alloc_gfp, node, oo);
1558         if (unlikely(!page))
1559             goto out;
1560         stat(s, ORDER_FALLBACK);
1561     }
1562 
1563     if (kmemcheck_enabled &&
1564         !(s->flags & (SLAB_NOTRACK | DEBUG_DEFAULT_FLAGS))) {
1565         int pages = 1 << oo_order(oo);
1566 
1567         kmemcheck_alloc_shadow(page, oo_order(oo), alloc_gfp, node);
1568 
1569         /*
1570          * Objects from caches that have a constructor don't get
1571          * cleared when they're allocated, so we need to do it here.
1572          */
1573         if (s->ctor)
1574             kmemcheck_mark_uninitialized_pages(page, pages);
1575         else
1576             kmemcheck_mark_unallocated_pages(page, pages);
1577     }
1578 
1579     page->objects = oo_objects(oo);
1580 
1581     order = compound_order(page);
1582     page->slab_cache = s;
1583     __SetPageSlab(page);
1584     if (page_is_pfmemalloc(page))
1585         SetPageSlabPfmemalloc(page);
1586 
1587     start = page_address(page);
1588 
1589     if (unlikely(s->flags & SLAB_POISON))
1590         memset(start, POISON_INUSE, PAGE_SIZE << order);
1591 
1592     kasan_poison_slab(page);
1593 
1594     shuffle = shuffle_freelist(s, page);
1595 
1596     if (!shuffle) {
1597         for_each_object_idx(p, idx, s, start, page->objects) {
1598             setup_object(s, page, p);
1599             if (likely(idx < page->objects))
1600                 set_freepointer(s, p, p + s->size);
1601             else
1602                 set_freepointer(s, p, NULL);
1603         }
1604         page->freelist = fixup_red_left(s, start);
1605     }
1606 
1607     page->inuse = page->objects;
1608     page->frozen = 1;
1609 
1610 out:
1611     if (gfpflags_allow_blocking(flags))
1612         local_irq_disable();
1613     if (!page)
1614         return NULL;
1615 
1616     mod_zone_page_state(page_zone(page),
1617         (s->flags & SLAB_RECLAIM_ACCOUNT) ?
1618         NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
1619         1 << oo_order(oo));
1620 
1621     inc_slabs_node(s, page_to_nid(page), page->objects);
1622 
1623     return page;
1624 }
1625 
1626 static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
1627 {
1628     if (unlikely(flags & GFP_SLAB_BUG_MASK)) {
1629         gfp_t invalid_mask = flags & GFP_SLAB_BUG_MASK;
1630         flags &= ~GFP_SLAB_BUG_MASK;
1631         pr_warn("Unexpected gfp: %#x (%pGg). Fixing up to gfp: %#x (%pGg). Fix your code!\n",
1632                 invalid_mask, &invalid_mask, flags, &flags);
1633     }
1634 
1635     return allocate_slab(s,
1636         flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node);
1637 }
1638 
1639 static void __free_slab(struct kmem_cache *s, struct page *page)
1640 {
1641     int order = compound_order(page);
1642     int pages = 1 << order;
1643 
1644     if (s->flags & SLAB_CONSISTENCY_CHECKS) {
1645         void *p;
1646 
1647         slab_pad_check(s, page);
1648         for_each_object(p, s, page_address(page),
1649                         page->objects)
1650             check_object(s, page, p, SLUB_RED_INACTIVE);
1651     }
1652 
1653     kmemcheck_free_shadow(page, compound_order(page));
1654 
1655     mod_zone_page_state(page_zone(page),
1656         (s->flags & SLAB_RECLAIM_ACCOUNT) ?
1657         NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
1658         -pages);
1659 
1660     __ClearPageSlabPfmemalloc(page);
1661     __ClearPageSlab(page);
1662 
1663     page_mapcount_reset(page);
1664     if (current->reclaim_state)
1665         current->reclaim_state->reclaimed_slab += pages;
1666     memcg_uncharge_slab(page, order, s);
1667     __free_pages(page, order);
1668 }
1669 
1670 #define need_reserve_slab_rcu                       \
1671     (sizeof(((struct page *)NULL)->lru) < sizeof(struct rcu_head))
1672 
1673 static void rcu_free_slab(struct rcu_head *h)
1674 {
1675     struct page *page;
1676 
1677     if (need_reserve_slab_rcu)
1678         page = virt_to_head_page(h);
1679     else
1680         page = container_of((struct list_head *)h, struct page, lru);
1681 
1682     __free_slab(page->slab_cache, page);
1683 }
1684 
1685 static void free_slab(struct kmem_cache *s, struct page *page)
1686 {
1687     if (unlikely(s->flags & SLAB_DESTROY_BY_RCU)) {
1688         struct rcu_head *head;
1689 
1690         if (need_reserve_slab_rcu) {
1691             int order = compound_order(page);
1692             int offset = (PAGE_SIZE << order) - s->reserved;
1693 
1694             VM_BUG_ON(s->reserved != sizeof(*head));
1695             head = page_address(page) + offset;
1696         } else {
1697             head = &page->rcu_head;
1698         }
1699 
1700         call_rcu(head, rcu_free_slab);
1701     } else
1702         __free_slab(s, page);
1703 }
1704 
1705 static void discard_slab(struct kmem_cache *s, struct page *page)
1706 {
1707     dec_slabs_node(s, page_to_nid(page), page->objects);
1708     free_slab(s, page);
1709 }
1710 
1711 /*
1712  * Management of partially allocated slabs.
1713  */
1714 static inline void
1715 __add_partial(struct kmem_cache_node *n, struct page *page, int tail)
1716 {
1717     n->nr_partial++;
1718     if (tail == DEACTIVATE_TO_TAIL)
1719         list_add_tail(&page->lru, &n->partial);
1720     else
1721         list_add(&page->lru, &n->partial);
1722 }
1723 
1724 static inline void add_partial(struct kmem_cache_node *n,
1725                 struct page *page, int tail)
1726 {
1727     lockdep_assert_held(&n->list_lock);
1728     __add_partial(n, page, tail);
1729 }
1730 
1731 static inline void remove_partial(struct kmem_cache_node *n,
1732                     struct page *page)
1733 {
1734     lockdep_assert_held(&n->list_lock);
1735     list_del(&page->lru);
1736     n->nr_partial--;
1737 }
1738 
1739 /*
1740  * Remove slab from the partial list, freeze it and
1741  * return the pointer to the freelist.
1742  *
1743  * Returns a list of objects or NULL if it fails.
1744  */
1745 static inline void *acquire_slab(struct kmem_cache *s,
1746         struct kmem_cache_node *n, struct page *page,
1747         int mode, int *objects)
1748 {
1749     void *freelist;
1750     unsigned long counters;
1751     struct page new;
1752 
1753     lockdep_assert_held(&n->list_lock);
1754 
1755     /*
1756      * Zap the freelist and set the frozen bit.
1757      * The old freelist is the list of objects for the
1758      * per cpu allocation list.
1759      */
1760     freelist = page->freelist;
1761     counters = page->counters;
1762     new.counters = counters;
1763     *objects = new.objects - new.inuse;
1764     if (mode) {
1765         new.inuse = page->objects;
1766         new.freelist = NULL;
1767     } else {
1768         new.freelist = freelist;
1769     }
1770 
1771     VM_BUG_ON(new.frozen);
1772     new.frozen = 1;
1773 
1774     if (!__cmpxchg_double_slab(s, page,
1775             freelist, counters,
1776             new.freelist, new.counters,
1777             "acquire_slab"))
1778         return NULL;
1779 
1780     remove_partial(n, page);
1781     WARN_ON(!freelist);
1782     return freelist;
1783 }
1784 
1785 static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);
1786 static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);
1787 
1788 /*
1789  * Try to allocate a partial slab from a specific node.
1790  */
1791 static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n,
1792                 struct kmem_cache_cpu *c, gfp_t flags)
1793 {
1794     struct page *page, *page2;
1795     void *object = NULL;
1796     int available = 0;
1797     int objects;
1798 
1799     /*
1800      * Racy check. If we mistakenly see no partial slabs then we
1801      * just allocate an empty slab. If we mistakenly try to get a
1802      * partial slab and there is none available then get_partials()
1803      * will return NULL.
1804      */
1805     if (!n || !n->nr_partial)
1806         return NULL;
1807 
1808     spin_lock(&n->list_lock);
1809     list_for_each_entry_safe(page, page2, &n->partial, lru) {
1810         void *t;
1811 
1812         if (!pfmemalloc_match(page, flags))
1813             continue;
1814 
1815         t = acquire_slab(s, n, page, object == NULL, &objects);
1816         if (!t)
1817             break;
1818 
1819         available += objects;
1820         if (!object) {
1821             c->page = page;
1822             stat(s, ALLOC_FROM_PARTIAL);
1823             object = t;
1824         } else {
1825             put_cpu_partial(s, page, 0);
1826             stat(s, CPU_PARTIAL_NODE);
1827         }
1828         if (!kmem_cache_has_cpu_partial(s)
1829             || available > s->cpu_partial / 2)
1830             break;
1831 
1832     }
1833     spin_unlock(&n->list_lock);
1834     return object;
1835 }
1836 
1837 /*
1838  * Get a page from somewhere. Search in increasing NUMA distances.
1839  */
1840 static void *get_any_partial(struct kmem_cache *s, gfp_t flags,
1841         struct kmem_cache_cpu *c)
1842 {
1843 #ifdef CONFIG_NUMA
1844     struct zonelist *zonelist;
1845     struct zoneref *z;
1846     struct zone *zone;
1847     enum zone_type high_zoneidx = gfp_zone(flags);
1848     void *object;
1849     unsigned int cpuset_mems_cookie;
1850 
1851     /*
1852      * The defrag ratio allows a configuration of the tradeoffs between
1853      * inter node defragmentation and node local allocations. A lower
1854      * defrag_ratio increases the tendency to do local allocations
1855      * instead of attempting to obtain partial slabs from other nodes.
1856      *
1857      * If the defrag_ratio is set to 0 then kmalloc() always
1858      * returns node local objects. If the ratio is higher then kmalloc()
1859      * may return off node objects because partial slabs are obtained
1860      * from other nodes and filled up.
1861      *
1862      * If /sys/kernel/slab/xx/remote_node_defrag_ratio is set to 100
1863      * (which makes defrag_ratio = 1000) then every (well almost)
1864      * allocation will first attempt to defrag slab caches on other nodes.
1865      * This means scanning over all nodes to look for partial slabs which
1866      * may be expensive if we do it every time we are trying to find a slab
1867      * with available objects.
1868      */
1869     if (!s->remote_node_defrag_ratio ||
1870             get_cycles() % 1024 > s->remote_node_defrag_ratio)
1871         return NULL;
1872 
1873     do {
1874         cpuset_mems_cookie = read_mems_allowed_begin();
1875         zonelist = node_zonelist(mempolicy_slab_node(), flags);
1876         for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) {
1877             struct kmem_cache_node *n;
1878 
1879             n = get_node(s, zone_to_nid(zone));
1880 
1881             if (n && cpuset_zone_allowed(zone, flags) &&
1882                     n->nr_partial > s->min_partial) {
1883                 object = get_partial_node(s, n, c, flags);
1884                 if (object) {
1885                     /*
1886                      * Don't check read_mems_allowed_retry()
1887                      * here - if mems_allowed was updated in
1888                      * parallel, that was a harmless race
1889                      * between allocation and the cpuset
1890                      * update
1891                      */
1892                     return object;
1893                 }
1894             }
1895         }
1896     } while (read_mems_allowed_retry(cpuset_mems_cookie));
1897 #endif
1898     return NULL;
1899 }
1900 
1901 /*
1902  * Get a partial page, lock it and return it.
1903  */
1904 static void *get_partial(struct kmem_cache *s, gfp_t flags, int node,
1905         struct kmem_cache_cpu *c)
1906 {
1907     void *object;
1908     int searchnode = node;
1909 
1910     if (node == NUMA_NO_NODE)
1911         searchnode = numa_mem_id();
1912     else if (!node_present_pages(node))
1913         searchnode = node_to_mem_node(node);
1914 
1915     object = get_partial_node(s, get_node(s, searchnode), c, flags);
1916     if (object || node != NUMA_NO_NODE)
1917         return object;
1918 
1919     return get_any_partial(s, flags, c);
1920 }
1921 
1922 #ifdef CONFIG_PREEMPT
1923 /*
1924  * Calculate the next globally unique transaction for disambiguiation
1925  * during cmpxchg. The transactions start with the cpu number and are then
1926  * incremented by CONFIG_NR_CPUS.
1927  */
1928 #define TID_STEP  roundup_pow_of_two(CONFIG_NR_CPUS)
1929 #else
1930 /*
1931  * No preemption supported therefore also no need to check for
1932  * different cpus.
1933  */
1934 #define TID_STEP 1
1935 #endif
1936 
1937 static inline unsigned long next_tid(unsigned long tid)
1938 {
1939     return tid + TID_STEP;
1940 }
1941 
1942 static inline unsigned int tid_to_cpu(unsigned long tid)
1943 {
1944     return tid % TID_STEP;
1945 }
1946 
1947 static inline unsigned long tid_to_event(unsigned long tid)
1948 {
1949     return tid / TID_STEP;
1950 }
1951 
1952 static inline unsigned int init_tid(int cpu)
1953 {
1954     return cpu;
1955 }
1956 
1957 static inline void note_cmpxchg_failure(const char *n,
1958         const struct kmem_cache *s, unsigned long tid)
1959 {
1960 #ifdef SLUB_DEBUG_CMPXCHG
1961     unsigned long actual_tid = __this_cpu_read(s->cpu_slab->tid);
1962 
1963     pr_info("%s %s: cmpxchg redo ", n, s->name);
1964 
1965 #ifdef CONFIG_PREEMPT
1966     if (tid_to_cpu(tid) != tid_to_cpu(actual_tid))
1967         pr_warn("due to cpu change %d -> %d\n",
1968             tid_to_cpu(tid), tid_to_cpu(actual_tid));
1969     else
1970 #endif
1971     if (tid_to_event(tid) != tid_to_event(actual_tid))
1972         pr_warn("due to cpu running other code. Event %ld->%ld\n",
1973             tid_to_event(tid), tid_to_event(actual_tid));
1974     else
1975         pr_warn("for unknown reason: actual=%lx was=%lx target=%lx\n",
1976             actual_tid, tid, next_tid(tid));
1977 #endif
1978     stat(s, CMPXCHG_DOUBLE_CPU_FAIL);
1979 }
1980 
1981 static void init_kmem_cache_cpus(struct kmem_cache *s)
1982 {
1983     int cpu;
1984 
1985     for_each_possible_cpu(cpu)
1986         per_cpu_ptr(s->cpu_slab, cpu)->tid = init_tid(cpu);
1987 }
1988 
1989 /*
1990  * Remove the cpu slab
1991  */
1992 static void deactivate_slab(struct kmem_cache *s, struct page *page,
1993                 void *freelist)
1994 {
1995     enum slab_modes { M_NONE, M_PARTIAL, M_FULL, M_FREE };
1996     struct kmem_cache_node *n = get_node(s, page_to_nid(page));
1997     int lock = 0;
1998     enum slab_modes l = M_NONE, m = M_NONE;
1999     void *nextfree;
2000     int tail = DEACTIVATE_TO_HEAD;
2001     struct page new;
2002     struct page old;
2003 
2004     if (page->freelist) {
2005         stat(s, DEACTIVATE_REMOTE_FREES);
2006         tail = DEACTIVATE_TO_TAIL;
2007     }
2008 
2009     /*
2010      * Stage one: Free all available per cpu objects back
2011      * to the page freelist while it is still frozen. Leave the
2012      * last one.
2013      *
2014      * There is no need to take the list->lock because the page
2015      * is still frozen.
2016      */
2017     while (freelist && (nextfree = get_freepointer(s, freelist))) {
2018         void *prior;
2019         unsigned long counters;
2020 
2021         do {
2022             prior = page->freelist;
2023             counters = page->counters;
2024             set_freepointer(s, freelist, prior);
2025             new.counters = counters;
2026             new.inuse--;
2027             VM_BUG_ON(!new.frozen);
2028 
2029         } while (!__cmpxchg_double_slab(s, page,
2030             prior, counters,
2031             freelist, new.counters,
2032             "drain percpu freelist"));
2033 
2034         freelist = nextfree;
2035     }
2036 
2037     /*
2038      * Stage two: Ensure that the page is unfrozen while the
2039      * list presence reflects the actual number of objects
2040      * during unfreeze.
2041      *
2042      * We setup the list membership and then perform a cmpxchg
2043      * with the count. If there is a mismatch then the page
2044      * is not unfrozen but the page is on the wrong list.
2045      *
2046      * Then we restart the process which may have to remove
2047      * the page from the list that we just put it on again
2048      * because the number of objects in the slab may have
2049      * changed.
2050      */
2051 redo:
2052 
2053     old.freelist = page->freelist;
2054     old.counters = page->counters;
2055     VM_BUG_ON(!old.frozen);
2056 
2057     /* Determine target state of the slab */
2058     new.counters = old.counters;
2059     if (freelist) {
2060         new.inuse--;
2061         set_freepointer(s, freelist, old.freelist);
2062         new.freelist = freelist;
2063     } else
2064         new.freelist = old.freelist;
2065 
2066     new.frozen = 0;
2067 
2068     if (!new.inuse && n->nr_partial >= s->min_partial)
2069         m = M_FREE;
2070     else if (new.freelist) {
2071         m = M_PARTIAL;
2072         if (!lock) {
2073             lock = 1;
2074             /*
2075              * Taking the spinlock removes the possiblity
2076              * that acquire_slab() will see a slab page that
2077              * is frozen
2078              */
2079             spin_lock(&n->list_lock);
2080         }
2081     } else {
2082         m = M_FULL;
2083         if (kmem_cache_debug(s) && !lock) {
2084             lock = 1;
2085             /*
2086              * This also ensures that the scanning of full
2087              * slabs from diagnostic functions will not see
2088              * any frozen slabs.
2089              */
2090             spin_lock(&n->list_lock);
2091         }
2092     }
2093 
2094     if (l != m) {
2095 
2096         if (l == M_PARTIAL)
2097 
2098             remove_partial(n, page);
2099 
2100         else if (l == M_FULL)
2101 
2102             remove_full(s, n, page);
2103 
2104         if (m == M_PARTIAL) {
2105 
2106             add_partial(n, page, tail);
2107             stat(s, tail);
2108 
2109         } else if (m == M_FULL) {
2110 
2111             stat(s, DEACTIVATE_FULL);
2112             add_full(s, n, page);
2113 
2114         }
2115     }
2116 
2117     l = m;
2118     if (!__cmpxchg_double_slab(s, page,
2119                 old.freelist, old.counters,
2120                 new.freelist, new.counters,
2121                 "unfreezing slab"))
2122         goto redo;
2123 
2124     if (lock)
2125         spin_unlock(&n->list_lock);
2126 
2127     if (m == M_FREE) {
2128         stat(s, DEACTIVATE_EMPTY);
2129         discard_slab(s, page);
2130         stat(s, FREE_SLAB);
2131     }
2132 }
2133 
2134 /*
2135  * Unfreeze all the cpu partial slabs.
2136  *
2137  * This function must be called with interrupts disabled
2138  * for the cpu using c (or some other guarantee must be there
2139  * to guarantee no concurrent accesses).
2140  */
2141 static void unfreeze_partials(struct kmem_cache *s,
2142         struct kmem_cache_cpu *c)
2143 {
2144 #ifdef CONFIG_SLUB_CPU_PARTIAL
2145     struct kmem_cache_node *n = NULL, *n2 = NULL;
2146     struct page *page, *discard_page = NULL;
2147 
2148     while ((page = c->partial)) {
2149         struct page new;
2150         struct page old;
2151 
2152         c->partial = page->next;
2153 
2154         n2 = get_node(s, page_to_nid(page));
2155         if (n != n2) {
2156             if (n)
2157                 spin_unlock(&n->list_lock);
2158 
2159             n = n2;
2160             spin_lock(&n->list_lock);
2161         }
2162 
2163         do {
2164 
2165             old.freelist = page->freelist;
2166             old.counters = page->counters;
2167             VM_BUG_ON(!old.frozen);
2168 
2169             new.counters = old.counters;
2170             new.freelist = old.freelist;
2171 
2172             new.frozen = 0;
2173 
2174         } while (!__cmpxchg_double_slab(s, page,
2175                 old.freelist, old.counters,
2176                 new.freelist, new.counters,
2177                 "unfreezing slab"));
2178 
2179         if (unlikely(!new.inuse && n->nr_partial >= s->min_partial)) {
2180             page->next = discard_page;
2181             discard_page = page;
2182         } else {
2183             add_partial(n, page, DEACTIVATE_TO_TAIL);
2184             stat(s, FREE_ADD_PARTIAL);
2185         }
2186     }
2187 
2188     if (n)
2189         spin_unlock(&n->list_lock);
2190 
2191     while (discard_page) {
2192         page = discard_page;
2193         discard_page = discard_page->next;
2194 
2195         stat(s, DEACTIVATE_EMPTY);
2196         discard_slab(s, page);
2197         stat(s, FREE_SLAB);
2198     }
2199 #endif
2200 }
2201 
2202 /*
2203  * Put a page that was just frozen (in __slab_free) into a partial page
2204  * slot if available. This is done without interrupts disabled and without
2205  * preemption disabled. The cmpxchg is racy and may put the partial page
2206  * onto a random cpus partial slot.
2207  *
2208  * If we did not find a slot then simply move all the partials to the
2209  * per node partial list.
2210  */
2211 static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain)
2212 {
2213 #ifdef CONFIG_SLUB_CPU_PARTIAL
2214     struct page *oldpage;
2215     int pages;
2216     int pobjects;
2217 
2218     preempt_disable();
2219     do {
2220         pages = 0;
2221         pobjects = 0;
2222         oldpage = this_cpu_read(s->cpu_slab->partial);
2223 
2224         if (oldpage) {
2225             pobjects = oldpage->pobjects;
2226             pages = oldpage->pages;
2227             if (drain && pobjects > s->cpu_partial) {
2228                 unsigned long flags;
2229                 /*
2230                  * partial array is full. Move the existing
2231                  * set to the per node partial list.
2232                  */
2233                 local_irq_save(flags);
2234                 unfreeze_partials(s, this_cpu_ptr(s->cpu_slab));
2235                 local_irq_restore(flags);
2236                 oldpage = NULL;
2237                 pobjects = 0;
2238                 pages = 0;
2239                 stat(s, CPU_PARTIAL_DRAIN);
2240             }
2241         }
2242 
2243         pages++;
2244         pobjects += page->objects - page->inuse;
2245 
2246         page->pages = pages;
2247         page->pobjects = pobjects;
2248         page->next = oldpage;
2249 
2250     } while (this_cpu_cmpxchg(s->cpu_slab->partial, oldpage, page)
2251                                 != oldpage);
2252     if (unlikely(!s->cpu_partial)) {
2253         unsigned long flags;
2254 
2255         local_irq_save(flags);
2256         unfreeze_partials(s, this_cpu_ptr(s->cpu_slab));
2257         local_irq_restore(flags);
2258     }
2259     preempt_enable();
2260 #endif
2261 }
2262 
2263 static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
2264 {
2265     stat(s, CPUSLAB_FLUSH);
2266     deactivate_slab(s, c->page, c->freelist);
2267 
2268     c->tid = next_tid(c->tid);
2269     c->page = NULL;
2270     c->freelist = NULL;
2271 }
2272 
2273 /*
2274  * Flush cpu slab.
2275  *
2276  * Called from IPI handler with interrupts disabled.
2277  */
2278 static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu)
2279 {
2280     struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
2281 
2282     if (likely(c)) {
2283         if (c->page)
2284             flush_slab(s, c);
2285 
2286         unfreeze_partials(s, c);
2287     }
2288 }
2289 
2290 static void flush_cpu_slab(void *d)
2291 {
2292     struct kmem_cache *s = d;
2293 
2294     __flush_cpu_slab(s, smp_processor_id());
2295 }
2296 
2297 static bool has_cpu_slab(int cpu, void *info)
2298 {
2299     struct kmem_cache *s = info;
2300     struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
2301 
2302     return c->page || c->partial;
2303 }
2304 
2305 static void flush_all(struct kmem_cache *s)
2306 {
2307     on_each_cpu_cond(has_cpu_slab, flush_cpu_slab, s, 1, GFP_ATOMIC);
2308 }
2309 
2310 /*
2311  * Use the cpu notifier to insure that the cpu slabs are flushed when
2312  * necessary.
2313  */
2314 static int slub_cpu_dead(unsigned int cpu)
2315 {
2316     struct kmem_cache *s;
2317     unsigned long flags;
2318 
2319     mutex_lock(&slab_mutex);
2320     list_for_each_entry(s, &slab_caches, list) {
2321         local_irq_save(flags);
2322         __flush_cpu_slab(s, cpu);
2323         local_irq_restore(flags);
2324     }
2325     mutex_unlock(&slab_mutex);
2326     return 0;
2327 }
2328 
2329 /*
2330  * Check if the objects in a per cpu structure fit numa
2331  * locality expectations.
2332  */
2333 static inline int node_match(struct page *page, int node)
2334 {
2335 #ifdef CONFIG_NUMA
2336     if (!page || (node != NUMA_NO_NODE && page_to_nid(page) != node))
2337         return 0;
2338 #endif
2339     return 1;
2340 }
2341 
2342 #ifdef CONFIG_SLUB_DEBUG
2343 static int count_free(struct page *page)
2344 {
2345     return page->objects - page->inuse;
2346 }
2347 
2348 static inline unsigned long node_nr_objs(struct kmem_cache_node *n)
2349 {
2350     return atomic_long_read(&n->total_objects);
2351 }
2352 #endif /* CONFIG_SLUB_DEBUG */
2353 
2354 #if defined(CONFIG_SLUB_DEBUG) || defined(CONFIG_SYSFS)
2355 static unsigned long count_partial(struct kmem_cache_node *n,
2356                     int (*get_count)(struct page *))
2357 {
2358     unsigned long flags;
2359     unsigned long x = 0;
2360     struct page *page;
2361 
2362     spin_lock_irqsave(&n->list_lock, flags);
2363     list_for_each_entry(page, &n->partial, lru)
2364         x += get_count(page);
2365     spin_unlock_irqrestore(&n->list_lock, flags);
2366     return x;
2367 }
2368 #endif /* CONFIG_SLUB_DEBUG || CONFIG_SYSFS */
2369 
2370 static noinline void
2371 slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid)
2372 {
2373 #ifdef CONFIG_SLUB_DEBUG
2374     static DEFINE_RATELIMIT_STATE(slub_oom_rs, DEFAULT_RATELIMIT_INTERVAL,
2375                       DEFAULT_RATELIMIT_BURST);
2376     int node;
2377     struct kmem_cache_node *n;
2378 
2379     if ((gfpflags & __GFP_NOWARN) || !__ratelimit(&slub_oom_rs))
2380         return;
2381 
2382     pr_warn("SLUB: Unable to allocate memory on node %d, gfp=%#x(%pGg)\n",
2383         nid, gfpflags, &gfpflags);
2384     pr_warn("  cache: %s, object size: %d, buffer size: %d, default order: %d, min order: %d\n",
2385         s->name, s->object_size, s->size, oo_order(s->oo),
2386         oo_order(s->min));
2387 
2388     if (oo_order(s->min) > get_order(s->object_size))
2389         pr_warn("  %s debugging increased min order, use slub_debug=O to disable.\n",
2390             s->name);
2391 
2392     for_each_kmem_cache_node(s, node, n) {
2393         unsigned long nr_slabs;
2394         unsigned long nr_objs;
2395         unsigned long nr_free;
2396 
2397         nr_free  = count_partial(n, count_free);
2398         nr_slabs = node_nr_slabs(n);
2399         nr_objs  = node_nr_objs(n);
2400 
2401         pr_warn("  node %d: slabs: %ld, objs: %ld, free: %ld\n",
2402             node, nr_slabs, nr_objs, nr_free);
2403     }
2404 #endif
2405 }
2406 
2407 static inline void *new_slab_objects(struct kmem_cache *s, gfp_t flags,
2408             int node, struct kmem_cache_cpu **pc)
2409 {
2410     void *freelist;
2411     struct kmem_cache_cpu *c = *pc;
2412     struct page *page;
2413 
2414     freelist = get_partial(s, flags, node, c);
2415 
2416     if (freelist)
2417         return freelist;
2418 
2419     page = new_slab(s, flags, node);
2420     if (page) {
2421         c = raw_cpu_ptr(s->cpu_slab);
2422         if (c->page)
2423             flush_slab(s, c);
2424 
2425         /*
2426          * No other reference to the page yet so we can
2427          * muck around with it freely without cmpxchg
2428          */
2429         freelist = page->freelist;
2430         page->freelist = NULL;
2431 
2432         stat(s, ALLOC_SLAB);
2433         c->page = page;
2434         *pc = c;
2435     } else
2436         freelist = NULL;
2437 
2438     return freelist;
2439 }
2440 
2441 static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags)
2442 {
2443     if (unlikely(PageSlabPfmemalloc(page)))
2444         return gfp_pfmemalloc_allowed(gfpflags);
2445 
2446     return true;
2447 }
2448 
2449 /*
2450  * Check the page->freelist of a page and either transfer the freelist to the
2451  * per cpu freelist or deactivate the page.
2452  *
2453  * The page is still frozen if the return value is not NULL.
2454  *
2455  * If this function returns NULL then the page has been unfrozen.
2456  *
2457  * This function must be called with interrupt disabled.
2458  */
2459 static inline void *get_freelist(struct kmem_cache *s, struct page *page)
2460 {
2461     struct page new;
2462     unsigned long counters;
2463     void *freelist;
2464 
2465     do {
2466         freelist = page->freelist;
2467         counters = page->counters;
2468 
2469         new.counters = counters;
2470         VM_BUG_ON(!new.frozen);
2471 
2472         new.inuse = page->objects;
2473         new.frozen = freelist != NULL;
2474 
2475     } while (!__cmpxchg_double_slab(s, page,
2476         freelist, counters,
2477         NULL, new.counters,
2478         "get_freelist"));
2479 
2480     return freelist;
2481 }
2482 
2483 /*
2484  * Slow path. The lockless freelist is empty or we need to perform
2485  * debugging duties.
2486  *
2487  * Processing is still very fast if new objects have been freed to the
2488  * regular freelist. In that case we simply take over the regular freelist
2489  * as the lockless freelist and zap the regular freelist.
2490  *
2491  * If that is not working then we fall back to the partial lists. We take the
2492  * first element of the freelist as the object to allocate now and move the
2493  * rest of the freelist to the lockless freelist.
2494  *
2495  * And if we were unable to get a new slab from the partial slab lists then
2496  * we need to allocate a new slab. This is the slowest path since it involves
2497  * a call to the page allocator and the setup of a new slab.
2498  *
2499  * Version of __slab_alloc to use when we know that interrupts are
2500  * already disabled (which is the case for bulk allocation).
2501  */
2502 static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
2503               unsigned long addr, struct kmem_cache_cpu *c)
2504 {
2505     void *freelist;
2506     struct page *page;
2507 
2508     page = c->page;
2509     if (!page)
2510         goto new_slab;
2511 redo:
2512 
2513     if (unlikely(!node_match(page, node))) {
2514         int searchnode = node;
2515 
2516         if (node != NUMA_NO_NODE && !node_present_pages(node))
2517             searchnode = node_to_mem_node(node);
2518 
2519         if (unlikely(!node_match(page, searchnode))) {
2520             stat(s, ALLOC_NODE_MISMATCH);
2521             deactivate_slab(s, page, c->freelist);
2522             c->page = NULL;
2523             c->freelist = NULL;
2524             goto new_slab;
2525         }
2526     }
2527 
2528     /*
2529      * By rights, we should be searching for a slab page that was
2530      * PFMEMALLOC but right now, we are losing the pfmemalloc
2531      * information when the page leaves the per-cpu allocator
2532      */
2533     if (unlikely(!pfmemalloc_match(page, gfpflags))) {
2534         deactivate_slab(s, page, c->freelist);
2535         c->page = NULL;
2536         c->freelist = NULL;
2537         goto new_slab;
2538     }
2539 
2540     /* must check again c->freelist in case of cpu migration or IRQ */
2541     freelist = c->freelist;
2542     if (freelist)
2543         goto load_freelist;
2544 
2545     freelist = get_freelist(s, page);
2546 
2547     if (!freelist) {
2548         c->page = NULL;
2549         stat(s, DEACTIVATE_BYPASS);
2550         goto new_slab;
2551     }
2552 
2553     stat(s, ALLOC_REFILL);
2554 
2555 load_freelist:
2556     /*
2557      * freelist is pointing to the list of objects to be used.
2558      * page is pointing to the page from which the objects are obtained.
2559      * That page must be frozen for per cpu allocations to work.
2560      */
2561     VM_BUG_ON(!c->page->frozen);
2562     c->freelist = get_freepointer(s, freelist);
2563     c->tid = next_tid(c->tid);
2564     return freelist;
2565 
2566 new_slab:
2567 
2568     if (c->partial) {
2569         page = c->page = c->partial;
2570         c->partial = page->next;
2571         stat(s, CPU_PARTIAL_ALLOC);
2572         c->freelist = NULL;
2573         goto redo;
2574     }
2575 
2576     freelist = new_slab_objects(s, gfpflags, node, &c);
2577 
2578     if (unlikely(!freelist)) {
2579         slab_out_of_memory(s, gfpflags, node);
2580         return NULL;
2581     }
2582 
2583     page = c->page;
2584     if (likely(!kmem_cache_debug(s) && pfmemalloc_match(page, gfpflags)))
2585         goto load_freelist;
2586 
2587     /* Only entered in the debug case */
2588     if (kmem_cache_debug(s) &&
2589             !alloc_debug_processing(s, page, freelist, addr))
2590         goto new_slab;  /* Slab failed checks. Next slab needed */
2591 
2592     deactivate_slab(s, page, get_freepointer(s, freelist));
2593     c->page = NULL;
2594     c->freelist = NULL;
2595     return freelist;
2596 }
2597 
2598 /*
2599  * Another one that disabled interrupt and compensates for possible
2600  * cpu changes by refetching the per cpu area pointer.
2601  */
2602 static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
2603               unsigned long addr, struct kmem_cache_cpu *c)
2604 {
2605     void *p;
2606     unsigned long flags;
2607 
2608     local_irq_save(flags);
2609 #ifdef CONFIG_PREEMPT
2610     /*
2611      * We may have been preempted and rescheduled on a different
2612      * cpu before disabling interrupts. Need to reload cpu area
2613      * pointer.
2614      */
2615     c = this_cpu_ptr(s->cpu_slab);
2616 #endif
2617 
2618     p = ___slab_alloc(s, gfpflags, node, addr, c);
2619     local_irq_restore(flags);
2620     return p;
2621 }
2622 
2623 /*
2624  * Inlined fastpath so that allocation functions (kmalloc, kmem_cache_alloc)
2625  * have the fastpath folded into their functions. So no function call
2626  * overhead for requests that can be satisfied on the fastpath.
2627  *
2628  * The fastpath works by first checking if the lockless freelist can be used.
2629  * If not then __slab_alloc is called for slow processing.
2630  *
2631  * Otherwise we can simply pick the next object from the lockless free list.
2632  */
2633 static __always_inline void *slab_alloc_node(struct kmem_cache *s,
2634         gfp_t gfpflags, int node, unsigned long addr)
2635 {
2636     void *object;
2637     struct kmem_cache_cpu *c;
2638     struct page *page;
2639     unsigned long tid;
2640 
2641     s = slab_pre_alloc_hook(s, gfpflags);
2642     if (!s)
2643         return NULL;
2644 redo:
2645     /*
2646      * Must read kmem_cache cpu data via this cpu ptr. Preemption is
2647      * enabled. We may switch back and forth between cpus while
2648      * reading from one cpu area. That does not matter as long
2649      * as we end up on the original cpu again when doing the cmpxchg.
2650      *
2651      * We should guarantee that tid and kmem_cache are retrieved on
2652      * the same cpu. It could be different if CONFIG_PREEMPT so we need
2653      * to check if it is matched or not.
2654      */
2655     do {
2656         tid = this_cpu_read(s->cpu_slab->tid);
2657         c = raw_cpu_ptr(s->cpu_slab);
2658     } while (IS_ENABLED(CONFIG_PREEMPT) &&
2659          unlikely(tid != READ_ONCE(c->tid)));
2660 
2661     /*
2662      * Irqless object alloc/free algorithm used here depends on sequence
2663      * of fetching cpu_slab's data. tid should be fetched before anything
2664      * on c to guarantee that object and page associated with previous tid
2665      * won't be used with current tid. If we fetch tid first, object and
2666      * page could be one associated with next tid and our alloc/free
2667      * request will be failed. In this case, we will retry. So, no problem.
2668      */
2669     barrier();
2670 
2671     /*
2672      * The transaction ids are globally unique per cpu and per operation on
2673      * a per cpu queue. Thus they can be guarantee that the cmpxchg_double
2674      * occurs on the right processor and that there was no operation on the
2675      * linked list in between.
2676      */
2677 
2678     object = c->freelist;
2679     page = c->page;
2680     if (unlikely(!object || !node_match(page, node))) {
2681         object = __slab_alloc(s, gfpflags, node, addr, c);
2682         stat(s, ALLOC_SLOWPATH);
2683     } else {
2684         void *next_object = get_freepointer_safe(s, object);
2685 
2686         /*
2687          * The cmpxchg will only match if there was no additional
2688          * operation and if we are on the right processor.
2689          *
2690          * The cmpxchg does the following atomically (without lock
2691          * semantics!)
2692          * 1. Relocate first pointer to the current per cpu area.
2693          * 2. Verify that tid and freelist have not been changed
2694          * 3. If they were not changed replace tid and freelist
2695          *
2696          * Since this is without lock semantics the protection is only
2697          * against code executing on this cpu *not* from access by
2698          * other cpus.
2699          */
2700         if (unlikely(!this_cpu_cmpxchg_double(
2701                 s->cpu_slab->freelist, s->cpu_slab->tid,
2702                 object, tid,
2703                 next_object, next_tid(tid)))) {
2704 
2705             note_cmpxchg_failure("slab_alloc", s, tid);
2706             goto redo;
2707         }
2708         prefetch_freepointer(s, next_object);
2709         stat(s, ALLOC_FASTPATH);
2710     }
2711 
2712     if (unlikely(gfpflags & __GFP_ZERO) && object)
2713         memset(object, 0, s->object_size);
2714 
2715     slab_post_alloc_hook(s, gfpflags, 1, &object);
2716 
2717     return object;
2718 }
2719 
2720 static __always_inline void *slab_alloc(struct kmem_cache *s,
2721         gfp_t gfpflags, unsigned long addr)
2722 {
2723     return slab_alloc_node(s, gfpflags, NUMA_NO_NODE, addr);
2724 }
2725 
2726 void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags)
2727 {
2728     void *ret = slab_alloc(s, gfpflags, _RET_IP_);
2729 
2730     trace_kmem_cache_alloc(_RET_IP_, ret, s->object_size,
2731                 s->size, gfpflags);
2732 
2733     return ret;
2734 }
2735 EXPORT_SYMBOL(kmem_cache_alloc);
2736 
2737 #ifdef CONFIG_TRACING
2738 void *kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size)
2739 {
2740     void *ret = slab_alloc(s, gfpflags, _RET_IP_);
2741     trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags);
2742     kasan_kmalloc(s, ret, size, gfpflags);
2743     return ret;
2744 }
2745 EXPORT_SYMBOL(kmem_cache_alloc_trace);
2746 #endif
2747 
2748 #ifdef CONFIG_NUMA
2749 void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node)
2750 {
2751     void *ret = slab_alloc_node(s, gfpflags, node, _RET_IP_);
2752 
2753     trace_kmem_cache_alloc_node(_RET_IP_, ret,
2754                     s->object_size, s->size, gfpflags, node);
2755 
2756     return ret;
2757 }
2758 EXPORT_SYMBOL(kmem_cache_alloc_node);
2759 
2760 #ifdef CONFIG_TRACING
2761 void *kmem_cache_alloc_node_trace(struct kmem_cache *s,
2762                     gfp_t gfpflags,
2763                     int node, size_t size)
2764 {
2765     void *ret = slab_alloc_node(s, gfpflags, node, _RET_IP_);
2766 
2767     trace_kmalloc_node(_RET_IP_, ret,
2768                size, s->size, gfpflags, node);
2769 
2770     kasan_kmalloc(s, ret, size, gfpflags);
2771     return ret;
2772 }
2773 EXPORT_SYMBOL(kmem_cache_alloc_node_trace);
2774 #endif
2775 #endif
2776 
2777 /*
2778  * Slow path handling. This may still be called frequently since objects
2779  * have a longer lifetime than the cpu slabs in most processing loads.
2780  *
2781  * So we still attempt to reduce cache line usage. Just take the slab
2782  * lock and free the item. If there is no additional partial page
2783  * handling required then we can return immediately.
2784  */
2785 static void __slab_free(struct kmem_cache *s, struct page *page,
2786             void *head, void *tail, int cnt,
2787             unsigned long addr)
2788 
2789 {
2790     void *prior;
2791     int was_frozen;
2792     struct page new;
2793     unsigned long counters;
2794     struct kmem_cache_node *n = NULL;
2795     unsigned long uninitialized_var(flags);
2796 
2797     stat(s, FREE_SLOWPATH);
2798 
2799     if (kmem_cache_debug(s) &&
2800         !free_debug_processing(s, page, head, tail, cnt, addr))
2801         return;
2802 
2803     do {
2804         if (unlikely(n)) {
2805             spin_unlock_irqrestore(&n->list_lock, flags);
2806             n = NULL;
2807         }
2808         prior = page->freelist;
2809         counters = page->counters;
2810         set_freepointer(s, tail, prior);
2811         new.counters = counters;
2812         was_frozen = new.frozen;
2813         new.inuse -= cnt;
2814         if ((!new.inuse || !prior) && !was_frozen) {
2815 
2816             if (kmem_cache_has_cpu_partial(s) && !prior) {
2817 
2818                 /*
2819                  * Slab was on no list before and will be
2820                  * partially empty
2821                  * We can defer the list move and instead
2822                  * freeze it.
2823                  */
2824                 new.frozen = 1;
2825 
2826             } else { /* Needs to be taken off a list */
2827 
2828                 n = get_node(s, page_to_nid(page));
2829                 /*
2830                  * Speculatively acquire the list_lock.
2831                  * If the cmpxchg does not succeed then we may
2832                  * drop the list_lock without any processing.
2833                  *
2834                  * Otherwise the list_lock will synchronize with
2835                  * other processors updating the list of slabs.
2836                  */
2837                 spin_lock_irqsave(&n->list_lock, flags);
2838 
2839             }
2840         }
2841 
2842     } while (!cmpxchg_double_slab(s, page,
2843         prior, counters,
2844         head, new.counters,
2845         "__slab_free"));
2846 
2847     if (likely(!n)) {
2848 
2849         /*
2850          * If we just froze the page then put it onto the
2851          * per cpu partial list.
2852          */
2853         if (new.frozen && !was_frozen) {
2854             put_cpu_partial(s, page, 1);
2855             stat(s, CPU_PARTIAL_FREE);
2856         }
2857         /*
2858          * The list lock was not taken therefore no list
2859          * activity can be necessary.
2860          */
2861         if (was_frozen)
2862             stat(s, FREE_FROZEN);
2863         return;
2864     }
2865 
2866     if (unlikely(!new.inuse && n->nr_partial >= s->min_partial))
2867         goto slab_empty;
2868 
2869     /*
2870      * Objects left in the slab. If it was not on the partial list before
2871      * then add it.
2872      */
2873     if (!kmem_cache_has_cpu_partial(s) && unlikely(!prior)) {
2874         if (kmem_cache_debug(s))
2875             remove_full(s, n, page);
2876         add_partial(n, page, DEACTIVATE_TO_TAIL);
2877         stat(s, FREE_ADD_PARTIAL);
2878     }
2879     spin_unlock_irqrestore(&n->list_lock, flags);
2880     return;
2881 
2882 slab_empty:
2883     if (prior) {
2884         /*
2885          * Slab on the partial list.
2886          */
2887         remove_partial(n, page);
2888         stat(s, FREE_REMOVE_PARTIAL);
2889     } else {
2890         /* Slab must be on the full list */
2891         remove_full(s, n, page);
2892     }
2893 
2894     spin_unlock_irqrestore(&n->list_lock, flags);
2895     stat(s, FREE_SLAB);
2896     discard_slab(s, page);
2897 }
2898 
2899 /*
2900  * Fastpath with forced inlining to produce a kfree and kmem_cache_free that
2901  * can perform fastpath freeing without additional function calls.
2902  *
2903  * The fastpath is only possible if we are freeing to the current cpu slab
2904  * of this processor. This typically the case if we have just allocated
2905  * the item before.
2906  *
2907  * If fastpath is not possible then fall back to __slab_free where we deal
2908  * with all sorts of special processing.
2909  *
2910  * Bulk free of a freelist with several objects (all pointing to the
2911  * same page) possible by specifying head and tail ptr, plus objects
2912  * count (cnt). Bulk free indicated by tail pointer being set.
2913  */
2914 static __always_inline void do_slab_free(struct kmem_cache *s,
2915                 struct page *page, void *head, void *tail,
2916                 int cnt, unsigned long addr)
2917 {
2918     void *tail_obj = tail ? : head;
2919     struct kmem_cache_cpu *c;
2920     unsigned long tid;
2921 redo:
2922     /*
2923      * Determine the currently cpus per cpu slab.
2924      * The cpu may change afterward. However that does not matter since
2925      * data is retrieved via this pointer. If we are on the same cpu
2926      * during the cmpxchg then the free will succeed.
2927      */
2928     do {
2929         tid = this_cpu_read(s->cpu_slab->tid);
2930         c = raw_cpu_ptr(s->cpu_slab);
2931     } while (IS_ENABLED(CONFIG_PREEMPT) &&
2932          unlikely(tid != READ_ONCE(c->tid)));
2933 
2934     /* Same with comment on barrier() in slab_alloc_node() */
2935     barrier();
2936 
2937     if (likely(page == c->page)) {
2938         set_freepointer(s, tail_obj, c->freelist);
2939 
2940         if (unlikely(!this_cpu_cmpxchg_double(
2941                 s->cpu_slab->freelist, s->cpu_slab->tid,
2942                 c->freelist, tid,
2943                 head, next_tid(tid)))) {
2944 
2945             note_cmpxchg_failure("slab_free", s, tid);
2946             goto redo;
2947         }
2948         stat(s, FREE_FASTPATH);
2949     } else
2950         __slab_free(s, page, head, tail_obj, cnt, addr);
2951 
2952 }
2953 
2954 static __always_inline void slab_free(struct kmem_cache *s, struct page *page,
2955                       void *head, void *tail, int cnt,
2956                       unsigned long addr)
2957 {
2958     slab_free_freelist_hook(s, head, tail);
2959     /*
2960      * slab_free_freelist_hook() could have put the items into quarantine.
2961      * If so, no need to free them.
2962      */
2963     if (s->flags & SLAB_KASAN && !(s->flags & SLAB_DESTROY_BY_RCU))
2964         return;
2965     do_slab_free(s, page, head, tail, cnt, addr);
2966 }
2967 
2968 #ifdef CONFIG_KASAN
2969 void ___cache_free(struct kmem_cache *cache, void *x, unsigned long addr)
2970 {
2971     do_slab_free(cache, virt_to_head_page(x), x, NULL, 1, addr);
2972 }
2973 #endif
2974 
2975 void kmem_cache_free(struct kmem_cache *s, void *x)
2976 {
2977     s = cache_from_obj(s, x);
2978     if (!s)
2979         return;
2980     slab_free(s, virt_to_head_page(x), x, NULL, 1, _RET_IP_);
2981     trace_kmem_cache_free(_RET_IP_, x);
2982 }
2983 EXPORT_SYMBOL(kmem_cache_free);
2984 
2985 struct detached_freelist {
2986     struct page *page;
2987     void *tail;
2988     void *freelist;
2989     int cnt;
2990     struct kmem_cache *s;
2991 };
2992 
2993 /*
2994  * This function progressively scans the array with free objects (with
2995  * a limited look ahead) and extract objects belonging to the same
2996  * page.  It builds a detached freelist directly within the given
2997  * page/objects.  This can happen without any need for
2998  * synchronization, because the objects are owned by running process.
2999  * The freelist is build up as a single linked list in the objects.
3000  * The idea is, that this detached freelist can then be bulk
3001  * transferred to the real freelist(s), but only requiring a single
3002  * synchronization primitive.  Look ahead in the array is limited due
3003  * to performance reasons.
3004  */
3005 static inline
3006 int build_detached_freelist(struct kmem_cache *s, size_t size,
3007                 void **p, struct detached_freelist *df)
3008 {
3009     size_t first_skipped_index = 0;
3010     int lookahead = 3;
3011     void *object;
3012     struct page *page;
3013 
3014     /* Always re-init detached_freelist */
3015     df->page = NULL;
3016 
3017     do {
3018         object = p[--size];
3019         /* Do we need !ZERO_OR_NULL_PTR(object) here? (for kfree) */
3020     } while (!object && size);
3021 
3022     if (!object)
3023         return 0;
3024 
3025     page = virt_to_head_page(object);
3026     if (!s) {
3027         /* Handle kalloc'ed objects */
3028         if (unlikely(!PageSlab(page))) {
3029             BUG_ON(!PageCompound(page));
3030             kfree_hook(object);
3031             __free_pages(page, compound_order(page));
3032             p[size] = NULL; /* mark object processed */
3033             return size;
3034         }
3035         /* Derive kmem_cache from object */
3036         df->s = page->slab_cache;
3037     } else {
3038         df->s = cache_from_obj(s, object); /* Support for memcg */
3039     }
3040 
3041     /* Start new detached freelist */
3042     df->page = page;
3043     set_freepointer(df->s, object, NULL);
3044     df->tail = object;
3045     df->freelist = object;
3046     p[size] = NULL; /* mark object processed */
3047     df->cnt = 1;
3048 
3049     while (size) {
3050         object = p[--size];
3051         if (!object)
3052             continue; /* Skip processed objects */
3053 
3054         /* df->page is always set at this point */
3055         if (df->page == virt_to_head_page(object)) {
3056             /* Opportunity build freelist */
3057             set_freepointer(df->s, object, df->freelist);
3058             df->freelist = object;
3059             df->cnt++;
3060             p[size] = NULL; /* mark object processed */
3061 
3062             continue;
3063         }
3064 
3065         /* Limit look ahead search */
3066         if (!--lookahead)
3067             break;
3068 
3069         if (!first_skipped_index)
3070             first_skipped_index = size + 1;
3071     }
3072 
3073     return first_skipped_index;
3074 }
3075 
3076 /* Note that interrupts must be enabled when calling this function. */
3077 void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)
3078 {
3079     if (WARN_ON(!size))
3080         return;
3081 
3082     do {
3083         struct detached_freelist df;
3084 
3085         size = build_detached_freelist(s, size, p, &df);
3086         if (!df.page)
3087             continue;
3088 
3089         slab_free(df.s, df.page, df.freelist, df.tail, df.cnt,_RET_IP_);
3090     } while (likely(size));
3091 }
3092 EXPORT_SYMBOL(kmem_cache_free_bulk);
3093 
3094 /* Note that interrupts must be enabled when calling this function. */
3095 int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
3096               void **p)
3097 {
3098     struct kmem_cache_cpu *c;
3099     int i;
3100 
3101     /* memcg and kmem_cache debug support */
3102     s = slab_pre_alloc_hook(s, flags);
3103     if (unlikely(!s))
3104         return false;
3105     /*
3106      * Drain objects in the per cpu slab, while disabling local
3107      * IRQs, which protects against PREEMPT and interrupts
3108      * handlers invoking normal fastpath.
3109      */
3110     local_irq_disable();
3111     c = this_cpu_ptr(s->cpu_slab);
3112 
3113     for (i = 0; i < size; i++) {
3114         void *object = c->freelist;
3115 
3116         if (unlikely(!object)) {
3117             /*
3118              * Invoking slow path likely have side-effect
3119              * of re-populating per CPU c->freelist
3120              */
3121             p[i] = ___slab_alloc(s, flags, NUMA_NO_NODE,
3122                         _RET_IP_, c);
3123             if (unlikely(!p[i]))
3124                 goto error;
3125 
3126             c = this_cpu_ptr(s->cpu_slab);
3127             continue; /* goto for-loop */
3128         }
3129         c->freelist = get_freepointer(s, object);
3130         p[i] = object;
3131     }
3132     c->tid = next_tid(c->tid);
3133     local_irq_enable();
3134 
3135     /* Clear memory outside IRQ disabled fastpath loop */
3136     if (unlikely(flags & __GFP_ZERO)) {
3137         int j;
3138 
3139         for (j = 0; j < i; j++)
3140             memset(p[j], 0, s->object_size);
3141     }
3142 
3143     /* memcg and kmem_cache debug support */
3144     slab_post_alloc_hook(s, flags, size, p);
3145     return i;
3146 error:
3147     local_irq_enable();
3148     slab_post_alloc_hook(s, flags, i, p);
3149     __kmem_cache_free_bulk(s, i, p);
3150     return 0;
3151 }
3152 EXPORT_SYMBOL(kmem_cache_alloc_bulk);
3153 
3154 
3155 /*
3156  * Object placement in a slab is made very easy because we always start at
3157  * offset 0. If we tune the size of the object to the alignment then we can
3158  * get the required alignment by putting one properly sized object after
3159  * another.
3160  *
3161  * Notice that the allocation order determines the sizes of the per cpu
3162  * caches. Each processor has always one slab available for allocations.
3163  * Increasing the allocation order reduces the number of times that slabs
3164  * must be moved on and off the partial lists and is therefore a factor in
3165  * locking overhead.
3166  */
3167 
3168 /*
3169  * Mininum / Maximum order of slab pages. This influences locking overhead
3170  * and slab fragmentation. A higher order reduces the number of partial slabs
3171  * and increases the number of allocations possible without having to
3172  * take the list_lock.
3173  */
3174 static int slub_min_order;
3175 static int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;
3176 static int slub_min_objects;
3177 
3178 /*
3179  * Calculate the order of allocation given an slab object size.
3180  *
3181  * The order of allocation has significant impact on performance and other
3182  * system components. Generally order 0 allocations should be preferred since
3183  * order 0 does not cause fragmentation in the page allocator. Larger objects
3184  * be problematic to put into order 0 slabs because there may be too much
3185  * unused space left. We go to a higher order if more than 1/16th of the slab
3186  * would be wasted.
3187  *
3188  * In order to reach satisfactory performance we must ensure that a minimum
3189  * number of objects is in one slab. Otherwise we may generate too much
3190  * activity on the partial lists which requires taking the list_lock. This is
3191  * less a concern for large slabs though which are rarely used.
3192  *
3193  * slub_max_order specifies the order where we begin to stop considering the
3194  * number of objects in a slab as critical. If we reach slub_max_order then
3195  * we try to keep the page order as low as possible. So we accept more waste
3196  * of space in favor of a small page order.
3197  *
3198  * Higher order allocations also allow the placement of more objects in a
3199  * slab and thereby reduce object handling overhead. If the user has
3200  * requested a higher mininum order then we start with that one instead of
3201  * the smallest order which will fit the object.
3202  */
3203 static inline int slab_order(int size, int min_objects,
3204                 int max_order, int fract_leftover, int reserved)
3205 {
3206     int order;
3207     int rem;
3208     int min_order = slub_min_order;
3209 
3210     if (order_objects(min_order, size, reserved) > MAX_OBJS_PER_PAGE)
3211         return get_order(size * MAX_OBJS_PER_PAGE) - 1;
3212 
3213     for (order = max(min_order, get_order(min_objects * size + reserved));
3214             order <= max_order; order++) {
3215 
3216         unsigned long slab_size = PAGE_SIZE << order;
3217 
3218         rem = (slab_size - reserved) % size;
3219 
3220         if (rem <= slab_size / fract_leftover)
3221             break;
3222     }
3223 
3224     return order;
3225 }
3226 
3227 static inline int calculate_order(int size, int reserved)
3228 {
3229     int order;
3230     int min_objects;
3231     int fraction;
3232     int max_objects;
3233 
3234     /*
3235      * Attempt to find best configuration for a slab. This
3236      * works by first attempting to generate a layout with
3237      * the best configuration and backing off gradually.
3238      *
3239      * First we increase the acceptable waste in a slab. Then
3240      * we reduce the minimum objects required in a slab.
3241      */
3242     min_objects = slub_min_objects;
3243     if (!min_objects)
3244         min_objects = 4 * (fls(nr_cpu_ids) + 1);
3245     max_objects = order_objects(slub_max_order, size, reserved);
3246     min_objects = min(min_objects, max_objects);
3247 
3248     while (min_objects > 1) {
3249         fraction = 16;
3250         while (fraction >= 4) {
3251             order = slab_order(size, min_objects,
3252                     slub_max_order, fraction, reserved);
3253             if (order <= slub_max_order)
3254                 return order;
3255             fraction /= 2;
3256         }
3257         min_objects--;
3258     }
3259 
3260     /*
3261      * We were unable to place multiple objects in a slab. Now
3262      * lets see if we can place a single object there.
3263      */
3264     order = slab_order(size, 1, slub_max_order, 1, reserved);
3265     if (order <= slub_max_order)
3266         return order;
3267 
3268     /*
3269      * Doh this slab cannot be placed using slub_max_order.
3270      */
3271     order = slab_order(size, 1, MAX_ORDER, 1, reserved);
3272     if (order < MAX_ORDER)
3273         return order;
3274     return -ENOSYS;
3275 }
3276 
3277 static void
3278 init_kmem_cache_node(struct kmem_cache_node *n)
3279 {
3280     n->nr_partial = 0;
3281     spin_lock_init(&n->list_lock);
3282     INIT_LIST_HEAD(&n->partial);
3283 #ifdef CONFIG_SLUB_DEBUG
3284     atomic_long_set(&n->nr_slabs, 0);
3285     atomic_long_set(&n->total_objects, 0);
3286     INIT_LIST_HEAD(&n->full);
3287 #endif
3288 }
3289 
3290 static inline int alloc_kmem_cache_cpus(struct kmem_cache *s)
3291 {
3292     BUILD_BUG_ON(PERCPU_DYNAMIC_EARLY_SIZE <
3293             KMALLOC_SHIFT_HIGH * sizeof(struct kmem_cache_cpu));
3294 
3295     /*
3296      * Must align to double word boundary for the double cmpxchg
3297      * instructions to work; see __pcpu_double_call_return_bool().
3298      */
3299     s->cpu_slab = __alloc_percpu(sizeof(struct kmem_cache_cpu),
3300                      2 * sizeof(void *));
3301 
3302     if (!s->cpu_slab)
3303         return 0;
3304 
3305     init_kmem_cache_cpus(s);
3306 
3307     return 1;
3308 }
3309 
3310 static struct kmem_cache *kmem_cache_node;
3311 
3312 /*
3313  * No kmalloc_node yet so do it by hand. We know that this is the first
3314  * slab on the node for this slabcache. There are no concurrent accesses
3315  * possible.
3316  *
3317  * Note that this function only works on the kmem_cache_node
3318  * when allocating for the kmem_cache_node. This is used for bootstrapping
3319  * memory on a fresh node that has no slab structures yet.
3320  */
3321 static void early_kmem_cache_node_alloc(int node)
3322 {
3323     struct page *page;
3324     struct kmem_cache_node *n;
3325 
3326     BUG_ON(kmem_cache_node->size < sizeof(struct kmem_cache_node));
3327 
3328     page = new_slab(kmem_cache_node, GFP_NOWAIT, node);
3329 
3330     BUG_ON(!page);
3331     if (page_to_nid(page) != node) {
3332         pr_err("SLUB: Unable to allocate memory from node %d\n", node);
3333         pr_err("SLUB: Allocating a useless per node structure in order to be able to continue\n");
3334     }
3335 
3336     n = page->freelist;
3337     BUG_ON(!n);
3338     page->freelist = get_freepointer(kmem_cache_node, n);
3339     page->inuse = 1;
3340     page->frozen = 0;
3341     kmem_cache_node->node[node] = n;
3342 #ifdef CONFIG_SLUB_DEBUG
3343     init_object(kmem_cache_node, n, SLUB_RED_ACTIVE);
3344     init_tracking(kmem_cache_node, n);
3345 #endif
3346     kasan_kmalloc(kmem_cache_node, n, sizeof(struct kmem_cache_node),
3347               GFP_KERNEL);
3348     init_kmem_cache_node(n);
3349     inc_slabs_node(kmem_cache_node, node, page->objects);
3350 
3351     /*
3352      * No locks need to be taken here as it has just been
3353      * initialized and there is no concurrent access.
3354      */
3355     __add_partial(n, page, DEACTIVATE_TO_HEAD);
3356 }
3357 
3358 static void free_kmem_cache_nodes(struct kmem_cache *s)
3359 {
3360     int node;
3361     struct kmem_cache_node *n;
3362 
3363     for_each_kmem_cache_node(s, node, n) {
3364         kmem_cache_free(kmem_cache_node, n);
3365         s->node[node] = NULL;
3366     }
3367 }
3368 
3369 void __kmem_cache_release(struct kmem_cache *s)
3370 {
3371     cache_random_seq_destroy(s);
3372     free_percpu(s->cpu_slab);
3373     free_kmem_cache_nodes(s);
3374 }
3375 
3376 static int init_kmem_cache_nodes(struct kmem_cache *s)
3377 {
3378     int node;
3379 
3380     for_each_node_state(node, N_NORMAL_MEMORY) {
3381         struct kmem_cache_node *n;
3382 
3383         if (slab_state == DOWN) {
3384             early_kmem_cache_node_alloc(node);
3385             continue;
3386         }
3387         n = kmem_cache_alloc_node(kmem_cache_node,
3388                         GFP_KERNEL, node);
3389 
3390         if (!n) {
3391             free_kmem_cache_nodes(s);
3392             return 0;
3393         }
3394 
3395         s->node[node] = n;
3396         init_kmem_cache_node(n);
3397     }
3398     return 1;
3399 }
3400 
3401 static void set_min_partial(struct kmem_cache *s, unsigned long min)
3402 {
3403     if (min < MIN_PARTIAL)
3404         min = MIN_PARTIAL;
3405     else if (min > MAX_PARTIAL)
3406         min = MAX_PARTIAL;
3407     s->min_partial = min;
3408 }
3409 
3410 /*
3411  * calculate_sizes() determines the order and the distribution of data within
3412  * a slab object.
3413  */
3414 static int calculate_sizes(struct kmem_cache *s, int forced_order)
3415 {
3416     unsigned long flags = s->flags;
3417     size_t size = s->object_size;
3418     int order;
3419 
3420     /*
3421      * Round up object size to the next word boundary. We can only
3422      * place the free pointer at word boundaries and this determines
3423      * the possible location of the free pointer.
3424      */
3425     size = ALIGN(size, sizeof(void *));
3426 
3427 #ifdef CONFIG_SLUB_DEBUG
3428     /*
3429      * Determine if we can poison the object itself. If the user of
3430      * the slab may touch the object after free or before allocation
3431      * then we should never poison the object itself.
3432      */
3433     if ((flags & SLAB_POISON) && !(flags & SLAB_DESTROY_BY_RCU) &&
3434             !s->ctor)
3435         s->flags |= __OBJECT_POISON;
3436     else
3437         s->flags &= ~__OBJECT_POISON;
3438 
3439 
3440     /*
3441      * If we are Redzoning then check if there is some space between the
3442      * end of the object and the free pointer. If not then add an
3443      * additional word to have some bytes to store Redzone information.
3444      */
3445     if ((flags & SLAB_RED_ZONE) && size == s->object_size)
3446         size += sizeof(void *);
3447 #endif
3448 
3449     /*
3450      * With that we have determined the number of bytes in actual use
3451      * by the object. This is the potential offset to the free pointer.
3452      */
3453     s->inuse = size;
3454 
3455     if (((flags & (SLAB_DESTROY_BY_RCU | SLAB_POISON)) ||
3456         s->ctor)) {
3457         /*
3458          * Relocate free pointer after the object if it is not
3459          * permitted to overwrite the first word of the object on
3460          * kmem_cache_free.
3461          *
3462          * This is the case if we do RCU, have a constructor or
3463          * destructor or are poisoning the objects.
3464          */
3465         s->offset = size;
3466         size += sizeof(void *);
3467     }
3468 
3469 #ifdef CONFIG_SLUB_DEBUG
3470     if (flags & SLAB_STORE_USER)
3471         /*
3472          * Need to store information about allocs and frees after
3473          * the object.
3474          */
3475         size += 2 * sizeof(struct track);
3476 #endif
3477 
3478     kasan_cache_create(s, &size, &s->flags);
3479 #ifdef CONFIG_SLUB_DEBUG
3480     if (flags & SLAB_RED_ZONE) {
3481         /*
3482          * Add some empty padding so that we can catch
3483          * overwrites from earlier objects rather than let
3484          * tracking information or the free pointer be
3485          * corrupted if a user writes before the start
3486          * of the object.
3487          */
3488         size += sizeof(void *);
3489 
3490         s->red_left_pad = sizeof(void *);
3491         s->red_left_pad = ALIGN(s->red_left_pad, s->align);
3492         size += s->red_left_pad;
3493     }
3494 #endif
3495 
3496     /*
3497      * SLUB stores one object immediately after another beginning from
3498      * offset 0. In order to align the objects we have to simply size
3499      * each object to conform to the alignment.
3500      */
3501     size = ALIGN(size, s->align);
3502     s->size = size;
3503     if (forced_order >= 0)
3504         order = forced_order;
3505     else
3506         order = calculate_order(size, s->reserved);
3507 
3508     if (order < 0)
3509         return 0;
3510 
3511     s->allocflags = 0;
3512     if (order)
3513         s->allocflags |= __GFP_COMP;
3514 
3515     if (s->flags & SLAB_CACHE_DMA)
3516         s->allocflags |= GFP_DMA;
3517 
3518     if (s->flags & SLAB_RECLAIM_ACCOUNT)
3519         s->allocflags |= __GFP_RECLAIMABLE;
3520 
3521     /*
3522      * Determine the number of objects per slab
3523      */
3524     s->oo = oo_make(order, size, s->reserved);
3525     s->min = oo_make(get_order(size), size, s->reserved);
3526     if (oo_objects(s->oo) > oo_objects(s->max))
3527         s->max = s->oo;
3528 
3529     return !!oo_objects(s->oo);
3530 }
3531 
3532 static int kmem_cache_open(struct kmem_cache *s, unsigned long flags)
3533 {
3534     s->flags = kmem_cache_flags(s->size, flags, s->name, s->ctor);
3535     s->reserved = 0;
3536 
3537     if (need_reserve_slab_rcu && (s->flags & SLAB_DESTROY_BY_RCU))
3538         s->reserved = sizeof(struct rcu_head);
3539 
3540     if (!calculate_sizes(s, -1))
3541         goto error;
3542     if (disable_higher_order_debug) {
3543         /*
3544          * Disable debugging flags that store metadata if the min slab
3545          * order increased.
3546          */
3547         if (get_order(s->size) > get_order(s->object_size)) {
3548             s->flags &= ~DEBUG_METADATA_FLAGS;
3549             s->offset = 0;
3550             if (!calculate_sizes(s, -1))
3551                 goto error;
3552         }
3553     }
3554 
3555 #if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
3556     defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
3557     if (system_has_cmpxchg_double() && (s->flags & SLAB_NO_CMPXCHG) == 0)
3558         /* Enable fast mode */
3559         s->flags |= __CMPXCHG_DOUBLE;
3560 #endif
3561 
3562     /*
3563      * The larger the object size is, the more pages we want on the partial
3564      * list to avoid pounding the page allocator excessively.
3565      */
3566     set_min_partial(s, ilog2(s->size) / 2);
3567 
3568     /*
3569      * cpu_partial determined the maximum number of objects kept in the
3570      * per cpu partial lists of a processor.
3571      *
3572      * Per cpu partial lists mainly contain slabs that just have one
3573      * object freed. If they are used for allocation then they can be
3574      * filled up again with minimal effort. The slab will never hit the
3575      * per node partial lists and therefore no locking will be required.
3576      *
3577      * This setting also determines
3578      *
3579      * A) The number of objects from per cpu partial slabs dumped to the
3580      *    per node list when we reach the limit.
3581      * B) The number of objects in cpu partial slabs to extract from the
3582      *    per node list when we run out of per cpu objects. We only fetch
3583      *    50% to keep some capacity around for frees.
3584      */
3585     if (!kmem_cache_has_cpu_partial(s))
3586         s->cpu_partial = 0;
3587     else if (s->size >= PAGE_SIZE)
3588         s->cpu_partial = 2;
3589     else if (s->size >= 1024)
3590         s->cpu_partial = 6;
3591     else if (s->size >= 256)
3592         s->cpu_partial = 13;
3593     else
3594         s->cpu_partial = 30;
3595 
3596 #ifdef CONFIG_NUMA
3597     s->remote_node_defrag_ratio = 1000;
3598 #endif
3599 
3600     /* Initialize the pre-computed randomized freelist if slab is up */
3601     if (slab_state >= UP) {
3602         if (init_cache_random_seq(s))
3603             goto error;
3604     }
3605 
3606     if (!init_kmem_cache_nodes(s))
3607         goto error;
3608 
3609     if (alloc_kmem_cache_cpus(s))
3610         return 0;
3611 
3612     free_kmem_cache_nodes(s);
3613 error:
3614     if (flags & SLAB_PANIC)
3615         panic("Cannot create slab %s size=%lu realsize=%u order=%u offset=%u flags=%lx\n",
3616               s->name, (unsigned long)s->size, s->size,
3617               oo_order(s->oo), s->offset, flags);
3618     return -EINVAL;
3619 }
3620 
3621 static void list_slab_objects(struct kmem_cache *s, struct page *page,
3622                             const char *text)
3623 {
3624 #ifdef CONFIG_SLUB_DEBUG
3625     void *addr = page_address(page);
3626     void *p;
3627     unsigned long *map = kzalloc(BITS_TO_LONGS(page->objects) *
3628                      sizeof(long), GFP_ATOMIC);
3629     if (!map)
3630         return;
3631     slab_err(s, page, text, s->name);
3632     slab_lock(page);
3633 
3634     get_map(s, page, map);
3635     for_each_object(p, s, addr, page->objects) {
3636 
3637         if (!test_bit(slab_index(p, s, addr), map)) {
3638             pr_err("INFO: Object 0x%p @offset=%tu\n", p, p - addr);
3639             print_tracking(s, p);
3640         }
3641     }
3642     slab_unlock(page);
3643     kfree(map);
3644 #endif
3645 }
3646 
3647 /*
3648  * Attempt to free all partial slabs on a node.
3649  * This is called from __kmem_cache_shutdown(). We must take list_lock
3650  * because sysfs file might still access partial list after the shutdowning.
3651  */
3652 static void free_partial(struct kmem_cache *s, struct kmem_cache_node *n)
3653 {
3654     LIST_HEAD(discard);
3655     struct page *page, *h;
3656 
3657     BUG_ON(irqs_disabled());
3658     spin_lock_irq(&n->list_lock);
3659     list_for_each_entry_safe(page, h, &n->partial, lru) {
3660         if (!page->inuse) {
3661             remove_partial(n, page);
3662             list_add(&page->lru, &discard);
3663         } else {
3664             list_slab_objects(s, page,
3665             "Objects remaining in %s on __kmem_cache_shutdown()");
3666         }
3667     }
3668     spin_unlock_irq(&n->list_lock);
3669 
3670     list_for_each_entry_safe(page, h, &discard, lru)
3671         discard_slab(s, page);
3672 }
3673 
3674 /*
3675  * Release all resources used by a slab cache.
3676  */
3677 int __kmem_cache_shutdown(struct kmem_cache *s)
3678 {
3679     int node;
3680     struct kmem_cache_node *n;
3681 
3682     flush_all(s);
3683     /* Attempt to free all objects */
3684     for_each_kmem_cache_node(s, node, n) {
3685         free_partial(s, n);
3686         if (n->nr_partial || slabs_node(s, node))
3687             return 1;
3688     }
3689     return 0;
3690 }
3691 
3692 /********************************************************************
3693  *      Kmalloc subsystem
3694  *******************************************************************/
3695 
3696 static int __init setup_slub_min_order(char *str)
3697 {
3698     get_option(&str, &slub_min_order);
3699 
3700     return 1;
3701 }
3702 
3703 __setup("slub_min_order=", setup_slub_min_order);
3704 
3705 static int __init setup_slub_max_order(char *str)
3706 {
3707     get_option(&str, &slub_max_order);
3708     slub_max_order = min(slub_max_order, MAX_ORDER - 1);
3709 
3710     return 1;
3711 }
3712 
3713 __setup("slub_max_order=", setup_slub_max_order);
3714 
3715 static int __init setup_slub_min_objects(char *str)
3716 {
3717     get_option(&str, &slub_min_objects);
3718 
3719     return 1;
3720 }
3721 
3722 __setup("slub_min_objects=", setup_slub_min_objects);
3723 
3724 void *__kmalloc(size_t size, gfp_t flags)
3725 {
3726     struct kmem_cache *s;
3727     void *ret;
3728 
3729     if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
3730         return kmalloc_large(size, flags);
3731 
3732     s = kmalloc_slab(size, flags);
3733 
3734     if (unlikely(ZERO_OR_NULL_PTR(s)))
3735         return s;
3736 
3737     ret = slab_alloc(s, flags, _RET_IP_);
3738 
3739     trace_kmalloc(_RET_IP_, ret, size, s->size, flags);
3740 
3741     kasan_kmalloc(s, ret, size, flags);
3742 
3743     return ret;
3744 }
3745 EXPORT_SYMBOL(__kmalloc);
3746 
3747 #ifdef CONFIG_NUMA
3748 static void *kmalloc_large_node(size_t size, gfp_t flags, int node)
3749 {
3750     struct page *page;
3751     void *ptr = NULL;
3752 
3753     flags |= __GFP_COMP | __GFP_NOTRACK;
3754     page = alloc_pages_node(node, flags, get_order(size));
3755     if (page)
3756         ptr = page_address(page);
3757 
3758     kmalloc_large_node_hook(ptr, size, flags);
3759     return ptr;
3760 }
3761 
3762 void *__kmalloc_node(size_t size, gfp_t flags, int node)
3763 {
3764     struct kmem_cache *s;
3765     void *ret;
3766 
3767     if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
3768         ret = kmalloc_large_node(size, flags, node);
3769 
3770         trace_kmalloc_node(_RET_IP_, ret,
3771                    size, PAGE_SIZE << get_order(size),
3772                    flags, node);
3773 
3774         return ret;
3775     }
3776 
3777     s = kmalloc_slab(size, flags);
3778 
3779     if (unlikely(ZERO_OR_NULL_PTR(s)))
3780         return s;
3781 
3782     ret = slab_alloc_node(s, flags, node, _RET_IP_);
3783 
3784     trace_kmalloc_node(_RET_IP_, ret, size, s->size, flags, node);
3785 
3786     kasan_kmalloc(s, ret, size, flags);
3787 
3788     return ret;
3789 }
3790 EXPORT_SYMBOL(__kmalloc_node);
3791 #endif
3792 
3793 #ifdef CONFIG_HARDENED_USERCOPY
3794 /*
3795  * Rejects objects that are incorrectly sized.
3796  *
3797  * Returns NULL if check passes, otherwise const char * to name of cache
3798  * to indicate an error.
3799  */
3800 const char *__check_heap_object(const void *ptr, unsigned long n,
3801                 struct page *page)
3802 {
3803     struct kmem_cache *s;
3804     unsigned long offset;
3805     size_t object_size;
3806 
3807     /* Find object and usable object size. */
3808     s = page->slab_cache;
3809     object_size = slab_ksize(s);
3810 
3811     /* Reject impossible pointers. */
3812     if (ptr < page_address(page))
3813         return s->name;
3814 
3815     /* Find offset within object. */
3816     offset = (ptr - page_address(page)) % s->size;
3817 
3818     /* Adjust for redzone and reject if within the redzone. */
3819     if (kmem_cache_debug(s) && s->flags & SLAB_RED_ZONE) {
3820         if (offset < s->red_left_pad)
3821             return s->name;
3822         offset -= s->red_left_pad;
3823     }
3824 
3825     /* Allow address range falling entirely within object size. */
3826     if (offset <= object_size && n <= object_size - offset)
3827         return NULL;
3828 
3829     return s->name;
3830 }
3831 #endif /* CONFIG_HARDENED_USERCOPY */
3832 
3833 static size_t __ksize(const void *object)
3834 {
3835     struct page *page;
3836 
3837     if (unlikely(object == ZERO_SIZE_PTR))
3838         return 0;
3839 
3840     page = virt_to_head_page(object);
3841 
3842     if (unlikely(!PageSlab(page))) {
3843         WARN_ON(!PageCompound(page));
3844         return PAGE_SIZE << compound_order(page);
3845     }
3846 
3847     return slab_ksize(page->slab_cache);
3848 }
3849 
3850 size_t ksize(const void *object)
3851 {
3852     size_t size = __ksize(object);
3853     /* We assume that ksize callers could use whole allocated area,
3854      * so we need to unpoison this area.
3855      */
3856     kasan_unpoison_shadow(object, size);
3857     return size;
3858 }
3859 EXPORT_SYMBOL(ksize);
3860 
3861 void kfree(const void *x)
3862 {
3863     struct page *page;
3864     void *object = (void *)x;
3865 
3866     trace_kfree(_RET_IP_, x);
3867 
3868     if (unlikely(ZERO_OR_NULL_PTR(x)))
3869         return;
3870 
3871     page = virt_to_head_page(x);
3872     if (unlikely(!PageSlab(page))) {
3873         BUG_ON(!PageCompound(page));
3874         kfree_hook(x);
3875         __free_pages(page, compound_order(page));
3876         return;
3877     }
3878     slab_free(page->slab_cache, page, object, NULL, 1, _RET_IP_);
3879 }
3880 EXPORT_SYMBOL(kfree);
3881 
3882 #define SHRINK_PROMOTE_MAX 32
3883 
3884 /*
3885  * kmem_cache_shrink discards empty slabs and promotes the slabs filled
3886  * up most to the head of the partial lists. New allocations will then
3887  * fill those up and thus they can be removed from the partial lists.
3888  *
3889  * The slabs with the least items are placed last. This results in them
3890  * being allocated from last increasing the chance that the last objects
3891  * are freed in them.
3892  */
3893 int __kmem_cache_shrink(struct kmem_cache *s)
3894 {
3895     int node;
3896     int i;
3897     struct kmem_cache_node *n;
3898     struct page *page;
3899     struct page *t;
3900     struct list_head discard;
3901     struct list_head promote[SHRINK_PROMOTE_MAX];
3902     unsigned long flags;
3903     int ret = 0;
3904 
3905     flush_all(s);
3906     for_each_kmem_cache_node(s, node, n) {
3907         INIT_LIST_HEAD(&discard);
3908         for (i = 0; i < SHRINK_PROMOTE_MAX; i++)
3909             INIT_LIST_HEAD(promote + i);
3910 
3911         spin_lock_irqsave(&n->list_lock, flags);
3912 
3913         /*
3914          * Build lists of slabs to discard or promote.
3915          *
3916          * Note that concurrent frees may occur while we hold the
3917          * list_lock. page->inuse here is the upper limit.
3918          */
3919         list_for_each_entry_safe(page, t, &n->partial, lru) {
3920             int free = page->objects - page->inuse;
3921 
3922             /* Do not reread page->inuse */
3923             barrier();
3924 
3925             /* We do not keep full slabs on the list */
3926             BUG_ON(free <= 0);
3927 
3928             if (free == page->objects) {
3929                 list_move(&page->lru, &discard);
3930                 n->nr_partial--;
3931             } else if (free <= SHRINK_PROMOTE_MAX)
3932                 list_move(&page->lru, promote + free - 1);
3933         }
3934 
3935         /*
3936          * Promote the slabs filled up most to the head of the
3937          * partial list.
3938          */
3939         for (i = SHRINK_PROMOTE_MAX - 1; i >= 0; i--)
3940             list_splice(promote + i, &n->partial);
3941 
3942         spin_unlock_irqrestore(&n->list_lock, flags);
3943 
3944         /* Release empty slabs */
3945         list_for_each_entry_safe(page, t, &discard, lru)
3946             discard_slab(s, page);
3947 
3948         if (slabs_node(s, node))
3949             ret = 1;
3950     }
3951 
3952     return ret;
3953 }
3954 
3955 static int slab_mem_going_offline_callback(void *arg)
3956 {
3957     struct kmem_cache *s;
3958 
3959     mutex_lock(&slab_mutex);
3960     list_for_each_entry(s, &slab_caches, list)
3961         __kmem_cache_shrink(s);
3962     mutex_unlock(&slab_mutex);
3963 
3964     return 0;
3965 }
3966 
3967 static void slab_mem_offline_callback(void *arg)
3968 {
3969     struct kmem_cache_node *n;
3970     struct kmem_cache *s;
3971     struct memory_notify *marg = arg;
3972     int offline_node;
3973 
3974     offline_node = marg->status_change_nid_normal;
3975 
3976     /*
3977      * If the node still has available memory. we need kmem_cache_node
3978      * for it yet.
3979      */
3980     if (offline_node < 0)
3981         return;
3982 
3983     mutex_lock(&slab_mutex);
3984     list_for_each_entry(s, &slab_caches, list) {
3985         n = get_node(s, offline_node);
3986         if (n) {
3987             /*
3988              * if n->nr_slabs > 0, slabs still exist on the node
3989              * that is going down. We were unable to free them,
3990              * and offline_pages() function shouldn't call this
3991              * callback. So, we must fail.
3992              */
3993             BUG_ON(slabs_node(s, offline_node));
3994 
3995             s->node[offline_node] = NULL;
3996             kmem_cache_free(kmem_cache_node, n);
3997         }
3998     }
3999     mutex_unlock(&slab_mutex);
4000 }
4001 
4002 static int slab_mem_going_online_callback(void *arg)
4003 {
4004     struct kmem_cache_node *n;
4005     struct kmem_cache *s;
4006     struct memory_notify *marg = arg;
4007     int nid = marg->status_change_nid_normal;
4008     int ret = 0;
4009 
4010     /*
4011      * If the node's memory is already available, then kmem_cache_node is
4012      * already created. Nothing to do.
4013      */
4014     if (nid < 0)
4015         return 0;
4016 
4017     /*
4018      * We are bringing a node online. No memory is available yet. We must
4019      * allocate a kmem_cache_node structure in order to bring the node
4020      * online.
4021      */
4022     mutex_lock(&slab_mutex);
4023     list_for_each_entry(s, &slab_caches, list) {
4024         /*
4025          * XXX: kmem_cache_alloc_node will fallback to other nodes
4026          *      since memory is not yet available from the node that
4027          *      is brought up.
4028          */
4029         n = kmem_cache_alloc(kmem_cache_node, GFP_KERNEL);
4030         if (!n) {
4031             ret = -ENOMEM;
4032             goto out;
4033         }
4034         init_kmem_cache_node(n);
4035         s->node[nid] = n;
4036     }
4037 out:
4038     mutex_unlock(&slab_mutex);
4039     return ret;
4040 }
4041 
4042 static int slab_memory_callback(struct notifier_block *self,
4043                 unsigned long action, void *arg)
4044 {
4045     int ret = 0;
4046 
4047     switch (action) {
4048     case MEM_GOING_ONLINE:
4049         ret = slab_mem_going_online_callback(arg);
4050         break;
4051     case MEM_GOING_OFFLINE:
4052         ret = slab_mem_going_offline_callback(arg);
4053         break;
4054     case MEM_OFFLINE:
4055     case MEM_CANCEL_ONLINE:
4056         slab_mem_offline_callback(arg);
4057         break;
4058     case MEM_ONLINE:
4059     case MEM_CANCEL_OFFLINE:
4060         break;
4061     }
4062     if (ret)
4063         ret = notifier_from_errno(ret);
4064     else
4065         ret = NOTIFY_OK;
4066     return ret;
4067 }
4068 
4069 static struct notifier_block slab_memory_callback_nb = {
4070     .notifier_call = slab_memory_callback,
4071     .priority = SLAB_CALLBACK_PRI,
4072 };
4073 
4074 /********************************************************************
4075  *          Basic setup of slabs
4076  *******************************************************************/
4077 
4078 /*
4079  * Used for early kmem_cache structures that were allocated using
4080  * the page allocator. Allocate them properly then fix up the pointers
4081  * that may be pointing to the wrong kmem_cache structure.
4082  */
4083 
4084 static struct kmem_cache * __init bootstrap(struct kmem_cache *static_cache)
4085 {
4086     int node;
4087     struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
4088     struct kmem_cache_node *n;
4089 
4090     memcpy(s, static_cache, kmem_cache->object_size);
4091 
4092     /*
4093      * This runs very early, and only the boot processor is supposed to be
4094      * up.  Even if it weren't true, IRQs are not up so we couldn't fire
4095      * IPIs around.
4096      */
4097     __flush_cpu_slab(s, smp_processor_id());
4098     for_each_kmem_cache_node(s, node, n) {
4099         struct page *p;
4100 
4101         list_for_each_entry(p, &n->partial, lru)
4102             p->slab_cache = s;
4103 
4104 #ifdef CONFIG_SLUB_DEBUG
4105         list_for_each_entry(p, &n->full, lru)
4106             p->slab_cache = s;
4107 #endif
4108     }
4109     slab_init_memcg_params(s);
4110     list_add(&s->list, &slab_caches);
4111     return s;
4112 }
4113 
4114 void __init kmem_cache_init(void)
4115 {
4116     static __initdata struct kmem_cache boot_kmem_cache,
4117         boot_kmem_cache_node;
4118 
4119     if (debug_guardpage_minorder())
4120         slub_max_order = 0;
4121 
4122     kmem_cache_node = &boot_kmem_cache_node;
4123     kmem_cache = &boot_kmem_cache;
4124 
4125     create_boot_cache(kmem_cache_node, "kmem_cache_node",
4126         sizeof(struct kmem_cache_node), SLAB_HWCACHE_ALIGN);
4127 
4128     register_hotmemory_notifier(&slab_memory_callback_nb);
4129 
4130     /* Able to allocate the per node structures */
4131     slab_state = PARTIAL;
4132 
4133     create_boot_cache(kmem_cache, "kmem_cache",
4134             offsetof(struct kmem_cache, node) +
4135                 nr_node_ids * sizeof(struct kmem_cache_node *),
4136                SLAB_HWCACHE_ALIGN);
4137 
4138     kmem_cache = bootstrap(&boot_kmem_cache);
4139 
4140     /*
4141      * Allocate kmem_cache_node properly from the kmem_cache slab.
4142      * kmem_cache_node is separately allocated so no need to
4143      * update any list pointers.
4144      */
4145     kmem_cache_node = bootstrap(&boot_kmem_cache_node);
4146 
4147     /* Now we can use the kmem_cache to allocate kmalloc slabs */
4148     setup_kmalloc_cache_index_table();
4149     create_kmalloc_caches(0);
4150 
4151     /* Setup random freelists for each cache */
4152     init_freelist_randomization();
4153 
4154     cpuhp_setup_state_nocalls(CPUHP_SLUB_DEAD, "slub:dead", NULL,
4155                   slub_cpu_dead);
4156 
4157     pr_info("SLUB: HWalign=%d, Order=%d-%d, MinObjects=%d, CPUs=%d, Nodes=%d\n",
4158         cache_line_size(),
4159         slub_min_order, slub_max_order, slub_min_objects,
4160         nr_cpu_ids, nr_node_ids);
4161 }
4162 
4163 void __init kmem_cache_init_late(void)
4164 {
4165 }
4166 
4167 struct kmem_cache *
4168 __kmem_cache_alias(const char *name, size_t size, size_t align,
4169            unsigned long flags, void (*ctor)(void *))
4170 {
4171     struct kmem_cache *s, *c;
4172 
4173     s = find_mergeable(size, align, flags, name, ctor);
4174     if (s) {
4175         s->refcount++;
4176 
4177         /*
4178          * Adjust the object sizes so that we clear
4179          * the complete object on kzalloc.
4180          */
4181         s->object_size = max(s->object_size, (int)size);
4182         s->inuse = max_t(int, s->inuse, ALIGN(size, sizeof(void *)));
4183 
4184         for_each_memcg_cache(c, s) {
4185             c->object_size = s->object_size;
4186             c->inuse = max_t(int, c->inuse,
4187                      ALIGN(size, sizeof(void *)));
4188         }
4189 
4190         if (sysfs_slab_alias(s, name)) {
4191             s->refcount--;
4192             s = NULL;
4193         }
4194     }
4195 
4196     return s;
4197 }
4198 
4199 int __kmem_cache_create(struct kmem_cache *s, unsigned long flags)
4200 {
4201     int err;
4202 
4203     err = kmem_cache_open(s, flags);
4204     if (err)
4205         return err;
4206 
4207     /* Mutex is not taken during early boot */
4208     if (slab_state <= UP)
4209         return 0;
4210 
4211     memcg_propagate_slab_attrs(s);
4212     err = sysfs_slab_add(s);
4213     if (err)
4214         __kmem_cache_release(s);
4215 
4216     return err;
4217 }
4218 
4219 void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller)
4220 {
4221     struct kmem_cache *s;
4222     void *ret;
4223 
4224     if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
4225         return kmalloc_large(size, gfpflags);
4226 
4227     s = kmalloc_slab(size, gfpflags);
4228 
4229     if (unlikely(ZERO_OR_NULL_PTR(s)))
4230         return s;
4231 
4232     ret = slab_alloc(s, gfpflags, caller);
4233 
4234     /* Honor the call site pointer we received. */
4235     trace_kmalloc(caller, ret, size, s->size, gfpflags);
4236 
4237     return ret;
4238 }
4239 
4240 #ifdef CONFIG_NUMA
4241 void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
4242                     int node, unsigned long caller)
4243 {
4244     struct kmem_cache *s;
4245     void *ret;
4246 
4247     if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
4248         ret = kmalloc_large_node(size, gfpflags, node);
4249 
4250         trace_kmalloc_node(caller, ret,
4251                    size, PAGE_SIZE << get_order(size),
4252                    gfpflags, node);
4253 
4254         return ret;
4255     }
4256 
4257     s = kmalloc_slab(size, gfpflags);
4258 
4259     if (unlikely(ZERO_OR_NULL_PTR(s)))
4260         return s;
4261 
4262     ret = slab_alloc_node(s, gfpflags, node, caller);
4263 
4264     /* Honor the call site pointer we received. */
4265     trace_kmalloc_node(caller, ret, size, s->size, gfpflags, node);
4266 
4267     return ret;
4268 }
4269 #endif
4270 
4271 #ifdef CONFIG_SYSFS
4272 static int count_inuse(struct page *page)
4273 {
4274     return page->inuse;
4275 }
4276 
4277 static int count_total(struct page *page)
4278 {
4279     return page->objects;
4280 }
4281 #endif
4282 
4283 #ifdef CONFIG_SLUB_DEBUG
4284 static int validate_slab(struct kmem_cache *s, struct page *page,
4285                         unsigned long *map)
4286 {
4287     void *p;
4288     void *addr = page_address(page);
4289 
4290     if (!check_slab(s, page) ||
4291             !on_freelist(s, page, NULL))
4292         return 0;
4293 
4294     /* Now we know that a valid freelist exists */
4295     bitmap_zero(map, page->objects);
4296 
4297     get_map(s, page, map);
4298     for_each_object(p, s, addr, page->objects) {
4299         if (test_bit(slab_index(p, s, addr), map))
4300             if (!check_object(s, page, p, SLUB_RED_INACTIVE))
4301                 return 0;
4302     }
4303 
4304     for_each_object(p, s, addr, page->objects)
4305         if (!test_bit(slab_index(p, s, addr), map))
4306             if (!check_object(s, page, p, SLUB_RED_ACTIVE))
4307                 return 0;
4308     return 1;
4309 }
4310 
4311 static void validate_slab_slab(struct kmem_cache *s, struct page *page,
4312                         unsigned long *map)
4313 {
4314     slab_lock(page);
4315     validate_slab(s, page, map);
4316     slab_unlock(page);
4317 }
4318 
4319 static int validate_slab_node(struct kmem_cache *s,
4320         struct kmem_cache_node *n, unsigned long *map)
4321 {
4322     unsigned long count = 0;
4323     struct page *page;
4324     unsigned long flags;
4325 
4326     spin_lock_irqsave(&n->list_lock, flags);
4327 
4328     list_for_each_entry(page, &n->partial, lru) {
4329         validate_slab_slab(s, page, map);
4330         count++;
4331     }
4332     if (count != n->nr_partial)
4333         pr_err("SLUB %s: %ld partial slabs counted but counter=%ld\n",
4334                s->name, count, n->nr_partial);
4335 
4336     if (!(s->flags & SLAB_STORE_USER))
4337         goto out;
4338 
4339     list_for_each_entry(page, &n->full, lru) {
4340         validate_slab_slab(s, page, map);
4341         count++;
4342     }
4343     if (count != atomic_long_read(&n->nr_slabs))
4344         pr_err("SLUB: %s %ld slabs counted but counter=%ld\n",
4345                s->name, count, atomic_long_read(&n->nr_slabs));
4346 
4347 out:
4348     spin_unlock_irqrestore(&n->list_lock, flags);
4349     return count;
4350 }
4351 
4352 static long validate_slab_cache(struct kmem_cache *s)
4353 {
4354     int node;
4355     unsigned long count = 0;
4356     unsigned long *map = kmalloc(BITS_TO_LONGS(oo_objects(s->max)) *
4357                 sizeof(unsigned long), GFP_KERNEL);
4358     struct kmem_cache_node *n;
4359 
4360     if (!map)
4361         return -ENOMEM;
4362 
4363     flush_all(s);
4364     for_each_kmem_cache_node(s, node, n)
4365         count += validate_slab_node(s, n, map);
4366     kfree(map);
4367     return count;
4368 }
4369 /*
4370  * Generate lists of code addresses where slabcache objects are allocated
4371  * and freed.
4372  */
4373 
4374 struct location {
4375     unsigned long count;
4376     unsigned long addr;
4377     long long sum_time;
4378     long min_time;
4379     long max_time;
4380     long min_pid;
4381     long max_pid;
4382     DECLARE_BITMAP(cpus, NR_CPUS);
4383     nodemask_t nodes;
4384 };
4385 
4386 struct loc_track {
4387     unsigned long max;
4388     unsigned long count;
4389     struct location *loc;
4390 };
4391 
4392 static void free_loc_track(struct loc_track *t)
4393 {
4394     if (t->max)
4395         free_pages((unsigned long)t->loc,
4396             get_order(sizeof(struct location) * t->max));
4397 }
4398 
4399 static int alloc_loc_track(struct loc_track *t, unsigned long max, gfp_t flags)
4400 {
4401     struct location *l;
4402     int order;
4403 
4404     order = get_order(sizeof(struct location) * max);
4405 
4406     l = (void *)__get_free_pages(flags, order);
4407     if (!l)
4408         return 0;
4409 
4410     if (t->count) {
4411         memcpy(l, t->loc, sizeof(struct location) * t->count);
4412         free_loc_track(t);
4413     }
4414     t->max = max;
4415     t->loc = l;
4416     return 1;
4417 }
4418 
4419 static int add_location(struct loc_track *t, struct kmem_cache *s,
4420                 const struct track *track)
4421 {
4422     long start, end, pos;
4423     struct location *l;
4424     unsigned long caddr;
4425     unsigned long age = jiffies - track->when;
4426 
4427     start = -1;
4428     end = t->count;
4429 
4430     for ( ; ; ) {
4431         pos = start + (end - start + 1) / 2;
4432 
4433         /*
4434          * There is nothing at "end". If we end up there
4435          * we need to add something to before end.
4436          */
4437         if (pos == end)
4438             break;
4439 
4440         caddr = t->loc[pos].addr;
4441         if (track->addr == caddr) {
4442 
4443             l = &t->loc[pos];
4444             l->count++;
4445             if (track->when) {
4446                 l->sum_time += age;
4447                 if (age < l->min_time)
4448                     l->min_time = age;
4449                 if (age > l->max_time)
4450                     l->max_time = age;
4451 
4452                 if (track->pid < l->min_pid)
4453                     l->min_pid = track->pid;
4454                 if (track->pid > l->max_pid)
4455                     l->max_pid = track->pid;
4456 
4457                 cpumask_set_cpu(track->cpu,
4458                         to_cpumask(l->cpus));
4459             }
4460             node_set(page_to_nid(virt_to_page(track)), l->nodes);
4461             return 1;
4462         }
4463 
4464         if (track->addr < caddr)
4465             end = pos;
4466         else
4467             start = pos;
4468     }
4469 
4470     /*
4471      * Not found. Insert new tracking element.
4472      */
4473     if (t->count >= t->max && !alloc_loc_track(t, 2 * t->max, GFP_ATOMIC))
4474         return 0;
4475 
4476     l = t->loc + pos;
4477     if (pos < t->count)
4478         memmove(l + 1, l,
4479             (t->count - pos) * sizeof(struct location));
4480     t->count++;
4481     l->count = 1;
4482     l->addr = track->addr;
4483     l->sum_time = age;
4484     l->min_time = age;
4485     l->max_time = age;
4486     l->min_pid = track->pid;
4487     l->max_pid = track->pid;
4488     cpumask_clear(to_cpumask(l->cpus));
4489     cpumask_set_cpu(track->cpu, to_cpumask(l->cpus));
4490     nodes_clear(l->nodes);
4491     node_set(page_to_nid(virt_to_page(track)), l->nodes);
4492     return 1;
4493 }
4494 
4495 static void process_slab(struct loc_track *t, struct kmem_cache *s,
4496         struct page *page, enum track_item alloc,
4497         unsigned long *map)
4498 {
4499     void *addr = page_address(page);
4500     void *p;
4501 
4502     bitmap_zero(map, page->objects);
4503     get_map(s, page, map);
4504 
4505     for_each_object(p, s, addr, page->objects)
4506         if (!test_bit(slab_index(p, s, addr), map))
4507             add_location(t, s, get_track(s, p, alloc));
4508 }
4509 
4510 static int list_locations(struct kmem_cache *s, char *buf,
4511                     enum track_item alloc)
4512 {
4513     int len = 0;
4514     unsigned long i;
4515     struct loc_track t = { 0, 0, NULL };
4516     int node;
4517     unsigned long *map = kmalloc(BITS_TO_LONGS(oo_objects(s->max)) *
4518                      sizeof(unsigned long), GFP_KERNEL);
4519     struct kmem_cache_node *n;
4520 
4521     if (!map || !alloc_loc_track(&t, PAGE_SIZE / sizeof(struct location),
4522                      GFP_TEMPORARY)) {
4523         kfree(map);
4524         return sprintf(buf, "Out of memory\n");
4525     }
4526     /* Push back cpu slabs */
4527     flush_all(s);
4528 
4529     for_each_kmem_cache_node(s, node, n) {
4530         unsigned long flags;
4531         struct page *page;
4532 
4533         if (!atomic_long_read(&n->nr_slabs))
4534             continue;
4535 
4536         spin_lock_irqsave(&n->list_lock, flags);
4537         list_for_each_entry(page, &n->partial, lru)
4538             process_slab(&t, s, page, alloc, map);
4539         list_for_each_entry(page, &n->full, lru)
4540             process_slab(&t, s, page, alloc, map);
4541         spin_unlock_irqrestore(&n->list_lock, flags);
4542     }
4543 
4544     for (i = 0; i < t.count; i++) {
4545         struct location *l = &t.loc[i];
4546 
4547         if (len > PAGE_SIZE - KSYM_SYMBOL_LEN - 100)
4548             break;
4549         len += sprintf(buf + len, "%7ld ", l->count);
4550 
4551         if (l->addr)
4552             len += sprintf(buf + len, "%pS", (void *)l->addr);
4553         else
4554             len += sprintf(buf + len, "<not-available>");
4555 
4556         if (l->sum_time != l->min_time) {
4557             len += sprintf(buf + len, " age=%ld/%ld/%ld",
4558                 l->min_time,
4559                 (long)div_u64(l->sum_time, l->count),
4560                 l->max_time);
4561         } else
4562             len += sprintf(buf + len, " age=%ld",
4563                 l->min_time);
4564 
4565         if (l->min_pid != l->max_pid)
4566             len += sprintf(buf + len, " pid=%ld-%ld",
4567                 l->min_pid, l->max_pid);
4568         else
4569             len += sprintf(buf + len, " pid=%ld",
4570                 l->min_pid);
4571 
4572         if (num_online_cpus() > 1 &&
4573                 !cpumask_empty(to_cpumask(l->cpus)) &&
4574                 len < PAGE_SIZE - 60)
4575             len += scnprintf(buf + len, PAGE_SIZE - len - 50,
4576                      " cpus=%*pbl",
4577                      cpumask_pr_args(to_cpumask(l->cpus)));
4578 
4579         if (nr_online_nodes > 1 && !nodes_empty(l->nodes) &&
4580                 len < PAGE_SIZE - 60)
4581             len += scnprintf(buf + len, PAGE_SIZE - len - 50,
4582                      " nodes=%*pbl",
4583                      nodemask_pr_args(&l->nodes));
4584 
4585         len += sprintf(buf + len, "\n");
4586     }
4587 
4588     free_loc_track(&t);
4589     kfree(map);
4590     if (!t.count)
4591         len += sprintf(buf, "No data\n");
4592     return len;
4593 }
4594 #endif
4595 
4596 #ifdef SLUB_RESILIENCY_TEST
4597 static void __init resiliency_test(void)
4598 {
4599     u8 *p;
4600 
4601     BUILD_BUG_ON(KMALLOC_MIN_SIZE > 16 || KMALLOC_SHIFT_HIGH < 10);
4602 
4603     pr_err("SLUB resiliency testing\n");
4604     pr_err("-----------------------\n");
4605     pr_err("A. Corruption after allocation\n");
4606 
4607     p = kzalloc(16, GFP_KERNEL);
4608     p[16] = 0x12;
4609     pr_err("\n1. kmalloc-16: Clobber Redzone/next pointer 0x12->0x%p\n\n",
4610            p + 16);
4611 
4612     validate_slab_cache(kmalloc_caches[4]);
4613 
4614     /* Hmmm... The next two are dangerous */
4615     p = kzalloc(32, GFP_KERNEL);
4616     p[32 + sizeof(void *)] = 0x34;
4617     pr_err("\n2. kmalloc-32: Clobber next pointer/next slab 0x34 -> -0x%p\n",
4618            p);
4619     pr_err("If allocated object is overwritten then not detectable\n\n");
4620 
4621     validate_slab_cache(kmalloc_caches[5]);
4622     p = kzalloc(64, GFP_KERNEL);
4623     p += 64 + (get_cycles() & 0xff) * sizeof(void *);
4624     *p = 0x56;
4625     pr_err("\n3. kmalloc-64: corrupting random byte 0x56->0x%p\n",
4626            p);
4627     pr_err("If allocated object is overwritten then not detectable\n\n");
4628     validate_slab_cache(kmalloc_caches[6]);
4629 
4630     pr_err("\nB. Corruption after free\n");
4631     p = kzalloc(128, GFP_KERNEL);
4632     kfree(p);
4633     *p = 0x78;
4634     pr_err("1. kmalloc-128: Clobber first word 0x78->0x%p\n\n", p);
4635     validate_slab_cache(kmalloc_caches[7]);
4636 
4637     p = kzalloc(256, GFP_KERNEL);
4638     kfree(p);
4639     p[50] = 0x9a;
4640     pr_err("\n2. kmalloc-256: Clobber 50th byte 0x9a->0x%p\n\n", p);
4641     validate_slab_cache(kmalloc_caches[8]);
4642 
4643     p = kzalloc(512, GFP_KERNEL);
4644     kfree(p);
4645     p[512] = 0xab;
4646     pr_err("\n3. kmalloc-512: Clobber redzone 0xab->0x%p\n\n", p);
4647     validate_slab_cache(kmalloc_caches[9]);
4648 }
4649 #else
4650 #ifdef CONFIG_SYSFS
4651 static void resiliency_test(void) {};
4652 #endif
4653 #endif
4654 
4655 #ifdef CONFIG_SYSFS
4656 enum slab_stat_type {
4657     SL_ALL,         /* All slabs */
4658     SL_PARTIAL,     /* Only partially allocated slabs */
4659     SL_CPU,         /* Only slabs used for cpu caches */
4660     SL_OBJECTS,     /* Determine allocated objects not slabs */
4661     SL_TOTAL        /* Determine object capacity not slabs */
4662 };
4663 
4664 #define SO_ALL      (1 << SL_ALL)
4665 #define SO_PARTIAL  (1 << SL_PARTIAL)
4666 #define SO_CPU      (1 << SL_CPU)
4667 #define SO_OBJECTS  (1 << SL_OBJECTS)
4668 #define SO_TOTAL    (1 << SL_TOTAL)
4669 
4670 static ssize_t show_slab_objects(struct kmem_cache *s,
4671                 char *buf, unsigned long flags)
4672 {
4673     unsigned long total = 0;
4674     int node;
4675     int x;
4676     unsigned long *nodes;
4677 
4678     nodes = kzalloc(sizeof(unsigned long) * nr_node_ids, GFP_KERNEL);
4679     if (!nodes)
4680         return -ENOMEM;
4681 
4682     if (flags & SO_CPU) {
4683         int cpu;
4684 
4685         for_each_possible_cpu(cpu) {
4686             struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab,
4687                                    cpu);
4688             int node;
4689             struct page *page;
4690 
4691             page = READ_ONCE(c->page);
4692             if (!page)
4693                 continue;
4694 
4695             node = page_to_nid(page);
4696             if (flags & SO_TOTAL)
4697                 x = page->objects;
4698             else if (flags & SO_OBJECTS)
4699                 x = page->inuse;
4700             else
4701                 x = 1;
4702 
4703             total += x;
4704             nodes[node] += x;
4705 
4706             page = READ_ONCE(c->partial);
4707             if (page) {
4708                 node = page_to_nid(page);
4709                 if (flags & SO_TOTAL)
4710                     WARN_ON_ONCE(1);
4711                 else if (flags & SO_OBJECTS)
4712                     WARN_ON_ONCE(1);
4713                 else
4714                     x = page->pages;
4715                 total += x;
4716                 nodes[node] += x;
4717             }
4718         }
4719     }
4720 
4721     get_online_mems();
4722 #ifdef CONFIG_SLUB_DEBUG
4723     if (flags & SO_ALL) {
4724         struct kmem_cache_node *n;
4725 
4726         for_each_kmem_cache_node(s, node, n) {
4727 
4728             if (flags & SO_TOTAL)
4729                 x = atomic_long_read(&n->total_objects);
4730             else if (flags & SO_OBJECTS)
4731                 x = atomic_long_read(&n->total_objects) -
4732                     count_partial(n, count_free);
4733             else
4734                 x = atomic_long_read(&n->nr_slabs);
4735             total += x;
4736             nodes[node] += x;
4737         }
4738 
4739     } else
4740 #endif
4741     if (flags & SO_PARTIAL) {
4742         struct kmem_cache_node *n;
4743 
4744         for_each_kmem_cache_node(s, node, n) {
4745             if (flags & SO_TOTAL)
4746                 x = count_partial(n, count_total);
4747             else if (flags & SO_OBJECTS)
4748                 x = count_partial(n, count_inuse);
4749             else
4750                 x = n->nr_partial;
4751             total += x;
4752             nodes[node] += x;
4753         }
4754     }
4755     x = sprintf(buf, "%lu", total);
4756 #ifdef CONFIG_NUMA
4757     for (node = 0; node < nr_node_ids; node++)
4758         if (nodes[node])
4759             x += sprintf(buf + x, " N%d=%lu",
4760                     node, nodes[node]);
4761 #endif
4762     put_online_mems();
4763     kfree(nodes);
4764     return x + sprintf(buf + x, "\n");
4765 }
4766 
4767 #ifdef CONFIG_SLUB_DEBUG
4768 static int any_slab_objects(struct kmem_cache *s)
4769 {
4770     int node;
4771     struct kmem_cache_node *n;
4772 
4773     for_each_kmem_cache_node(s, node, n)
4774         if (atomic_long_read(&n->total_objects))
4775             return 1;
4776 
4777     return 0;
4778 }
4779 #endif
4780 
4781 #define to_slab_attr(n) container_of(n, struct slab_attribute, attr)
4782 #define to_slab(n) container_of(n, struct kmem_cache, kobj)
4783 
4784 struct slab_attribute {
4785     struct attribute attr;
4786     ssize_t (*show)(struct kmem_cache *s, char *buf);
4787     ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
4788 };
4789 
4790 #define SLAB_ATTR_RO(_name) \
4791     static struct slab_attribute _name##_attr = \
4792     __ATTR(_name, 0400, _name##_show, NULL)
4793 
4794 #define SLAB_ATTR(_name) \
4795     static struct slab_attribute _name##_attr =  \
4796     __ATTR(_name, 0600, _name##_show, _name##_store)
4797 
4798 static ssize_t slab_size_show(struct kmem_cache *s, char *buf)
4799 {
4800     return sprintf(buf, "%d\n", s->size);
4801 }
4802 SLAB_ATTR_RO(slab_size);
4803 
4804 static ssize_t align_show(struct kmem_cache *s, char *buf)
4805 {
4806     return sprintf(buf, "%d\n", s->align);
4807 }
4808 SLAB_ATTR_RO(align);
4809 
4810 static ssize_t object_size_show(struct kmem_cache *s, char *buf)
4811 {
4812     return sprintf(buf, "%d\n", s->object_size);
4813 }
4814 SLAB_ATTR_RO(object_size);
4815 
4816 static ssize_t objs_per_slab_show(struct kmem_cache *s, char *buf)
4817 {
4818     return sprintf(buf, "%d\n", oo_objects(s->oo));
4819 }
4820 SLAB_ATTR_RO(objs_per_slab);
4821 
4822 static ssize_t order_store(struct kmem_cache *s,
4823                 const char *buf, size_t length)
4824 {
4825     unsigned long order;
4826     int err;
4827 
4828     err = kstrtoul(buf, 10, &order);
4829     if (err)
4830         return err;
4831 
4832     if (order > slub_max_order || order < slub_min_order)
4833         return -EINVAL;
4834 
4835     calculate_sizes(s, order);
4836     return length;
4837 }
4838 
4839 static ssize_t order_show(struct kmem_cache *s, char *buf)
4840 {
4841     return sprintf(buf, "%d\n", oo_order(s->oo));
4842 }
4843 SLAB_ATTR(order);
4844 
4845 static ssize_t min_partial_show(struct kmem_cache *s, char *buf)
4846 {
4847     return sprintf(buf, "%lu\n", s->min_partial);
4848 }
4849 
4850 static ssize_t min_partial_store(struct kmem_cache *s, const char *buf,
4851                  size_t length)
4852 {
4853     unsigned long min;
4854     int err;
4855 
4856     err = kstrtoul(buf, 10, &min);
4857     if (err)
4858         return err;
4859 
4860     set_min_partial(s, min);
4861     return length;
4862 }
4863 SLAB_ATTR(min_partial);
4864 
4865 static ssize_t cpu_partial_show(struct kmem_cache *s, char *buf)
4866 {
4867     return sprintf(buf, "%u\n", s->cpu_partial);
4868 }
4869 
4870 static ssize_t cpu_partial_store(struct kmem_cache *s, const char *buf,
4871                  size_t length)
4872 {
4873     unsigned long objects;
4874     int err;
4875 
4876     err = kstrtoul(buf, 10, &objects);
4877     if (err)
4878         return err;
4879     if (objects && !kmem_cache_has_cpu_partial(s))
4880         return -EINVAL;
4881 
4882     s->cpu_partial = objects;
4883     flush_all(s);
4884     return length;
4885 }
4886 SLAB_ATTR(cpu_partial);
4887 
4888 static ssize_t ctor_show(struct kmem_cache *s, char *buf)
4889 {
4890     if (!s->ctor)
4891         return 0;
4892     return sprintf(buf, "%pS\n", s->ctor);
4893 }
4894 SLAB_ATTR_RO(ctor);
4895 
4896 static ssize_t aliases_show(struct kmem_cache *s, char *buf)
4897 {
4898     return sprintf(buf, "%d\n", s->refcount < 0 ? 0 : s->refcount - 1);
4899 }
4900 SLAB_ATTR_RO(aliases);
4901 
4902 static ssize_t partial_show(struct kmem_cache *s, char *buf)
4903 {
4904     return show_slab_objects(s, buf, SO_PARTIAL);
4905 }
4906 SLAB_ATTR_RO(partial);
4907 
4908 static ssize_t cpu_slabs_show(struct kmem_cache *s, char *buf)
4909 {
4910     return show_slab_objects(s, buf, SO_CPU);
4911 }
4912 SLAB_ATTR_RO(cpu_slabs);
4913 
4914 static ssize_t objects_show(struct kmem_cache *s, char *buf)
4915 {
4916     return show_slab_objects(s, buf, SO_ALL|SO_OBJECTS);
4917 }
4918 SLAB_ATTR_RO(objects);
4919 
4920 static ssize_t objects_partial_show(struct kmem_cache *s, char *buf)
4921 {
4922     return show_slab_objects(s, buf, SO_PARTIAL|SO_OBJECTS);
4923 }
4924 SLAB_ATTR_RO(objects_partial);
4925 
4926 static ssize_t slabs_cpu_partial_show(struct kmem_cache *s, char *buf)
4927 {
4928     int objects = 0;
4929     int pages = 0;
4930     int cpu;
4931     int len;
4932 
4933     for_each_online_cpu(cpu) {
4934         struct page *page = per_cpu_ptr(s->cpu_slab, cpu)->partial;
4935 
4936         if (page) {
4937             pages += page->pages;
4938             objects += page->pobjects;
4939         }
4940     }
4941 
4942     len = sprintf(buf, "%d(%d)", objects, pages);
4943 
4944 #ifdef CONFIG_SMP
4945     for_each_online_cpu(cpu) {
4946         struct page *page = per_cpu_ptr(s->cpu_slab, cpu) ->partial;
4947 
4948         if (page && len < PAGE_SIZE - 20)
4949             len += sprintf(buf + len, " C%d=%d(%d)", cpu,
4950                 page->pobjects, page->pages);
4951     }
4952 #endif
4953     return len + sprintf(buf + len, "\n");
4954 }
4955 SLAB_ATTR_RO(slabs_cpu_partial);
4956 
4957 static ssize_t reclaim_account_show(struct kmem_cache *s, char *buf)
4958 {
4959     return sprintf(buf, "%d\n", !!(s->flags & SLAB_RECLAIM_ACCOUNT));
4960 }
4961 
4962 static ssize_t reclaim_account_store(struct kmem_cache *s,
4963                 const char *buf, size_t length)
4964 {
4965     s->flags &= ~SLAB_RECLAIM_ACCOUNT;
4966     if (buf[0] == '1')
4967         s->flags |= SLAB_RECLAIM_ACCOUNT;
4968     return length;
4969 }
4970 SLAB_ATTR(reclaim_account);
4971 
4972 static ssize_t hwcache_align_show(struct kmem_cache *s, char *buf)
4973 {
4974     return sprintf(buf, "%d\n", !!(s->flags & SLAB_HWCACHE_ALIGN));
4975 }
4976 SLAB_ATTR_RO(hwcache_align);
4977 
4978 #ifdef CONFIG_ZONE_DMA
4979 static ssize_t cache_dma_show(struct kmem_cache *s, char *buf)
4980 {
4981     return sprintf(buf, "%d\n", !!(s->flags & SLAB_CACHE_DMA));
4982 }
4983 SLAB_ATTR_RO(cache_dma);
4984 #endif
4985 
4986 static ssize_t destroy_by_rcu_show(struct kmem_cache *s, char *buf)
4987 {
4988     return sprintf(buf, "%d\n", !!(s->flags & SLAB_DESTROY_BY_RCU));
4989 }
4990 SLAB_ATTR_RO(destroy_by_rcu);
4991 
4992 static ssize_t reserved_show(struct kmem_cache *s, char *buf)
4993 {
4994     return sprintf(buf, "%d\n", s->reserved);
4995 }
4996 SLAB_ATTR_RO(reserved);
4997 
4998 #ifdef CONFIG_SLUB_DEBUG
4999 static ssize_t slabs_show(struct kmem_cache *s, char *buf)
5000 {
5001     return show_slab_objects(s, buf, SO_ALL);
5002 }
5003 SLAB_ATTR_RO(slabs);
5004 
5005 static ssize_t total_objects_show(struct kmem_cache *s, char *buf)
5006 {
5007     return show_slab_objects(s, buf, SO_ALL|SO_TOTAL);
5008 }
5009 SLAB_ATTR_RO(total_objects);
5010 
5011 static ssize_t sanity_checks_show(struct kmem_cache *s, char *buf)
5012 {
5013     return sprintf(buf, "%d\n", !!(s->flags & SLAB_CONSISTENCY_CHECKS));
5014 }
5015 
5016 static ssize_t sanity_checks_store(struct kmem_cache *s,
5017                 const char *buf, size_t length)
5018 {
5019     s->flags &= ~SLAB_CONSISTENCY_CHECKS;
5020     if (buf[0] == '1') {
5021         s->flags &= ~__CMPXCHG_DOUBLE;
5022         s->flags |= SLAB_CONSISTENCY_CHECKS;
5023     }
5024     return length;
5025 }
5026 SLAB_ATTR(sanity_checks);
5027 
5028 static ssize_t trace_show(struct kmem_cache *s, char *buf)
5029 {
5030     return sprintf(buf, "%d\n", !!(s->flags & SLAB_TRACE));
5031 }
5032 
5033 static ssize_t trace_store(struct kmem_cache *s, const char *buf,
5034                             size_t length)
5035 {
5036     /*
5037      * Tracing a merged cache is going to give confusing results
5038      * as well as cause other issues like converting a mergeable
5039      * cache into an umergeable one.
5040      */
5041     if (s->refcount > 1)
5042         return -EINVAL;
5043 
5044     s->flags &= ~SLAB_TRACE;
5045     if (buf[0] == '1') {
5046         s->flags &= ~__CMPXCHG_DOUBLE;
5047         s->flags |= SLAB_TRACE;
5048     }
5049     return length;
5050 }
5051 SLAB_ATTR(trace);
5052 
5053 static ssize_t red_zone_show(struct kmem_cache *s, char *buf)
5054 {
5055     return sprintf(buf, "%d\n", !!(s->flags & SLAB_RED_ZONE));
5056 }
5057 
5058 static ssize_t red_zone_store(struct kmem_cache *s,
5059                 const char *buf, size_t length)
5060 {
5061     if (any_slab_objects(s))
5062         return -EBUSY;
5063 
5064     s->flags &= ~SLAB_RED_ZONE;
5065     if (buf[0] == '1') {
5066         s->flags |= SLAB_RED_ZONE;
5067     }
5068     calculate_sizes(s, -1);
5069     return length;
5070 }
5071 SLAB_ATTR(red_zone);
5072 
5073 static ssize_t poison_show(struct kmem_cache *s, char *buf)
5074 {
5075     return sprintf(buf, "%d\n", !!(s->flags & SLAB_POISON));
5076 }
5077 
5078 static ssize_t poison_store(struct kmem_cache *s,
5079                 const char *buf, size_t length)
5080 {
5081     if (any_slab_objects(s))
5082         return -EBUSY;
5083 
5084     s->flags &= ~SLAB_POISON;
5085     if (buf[0] == '1') {
5086         s->flags |= SLAB_POISON;
5087     }
5088     calculate_sizes(s, -1);
5089     return length;
5090 }
5091 SLAB_ATTR(poison);
5092 
5093 static ssize_t store_user_show(struct kmem_cache *s, char *buf)
5094 {
5095     return sprintf(buf, "%d\n", !!(s->flags & SLAB_STORE_USER));
5096 }
5097 
5098 static ssize_t store_user_store(struct kmem_cache *s,
5099                 const char *buf, size_t length)
5100 {
5101     if (any_slab_objects(s))
5102         return -EBUSY;
5103 
5104     s->flags &= ~SLAB_STORE_USER;
5105     if (buf[0] == '1') {
5106         s->flags &= ~__CMPXCHG_DOUBLE;
5107         s->flags |= SLAB_STORE_USER;
5108     }
5109     calculate_sizes(s, -1);
5110     return length;
5111 }
5112 SLAB_ATTR(store_user);
5113 
5114 static ssize_t validate_show(struct kmem_cache *s, char *buf)
5115 {
5116     return 0;
5117 }
5118 
5119 static ssize_t validate_store(struct kmem_cache *s,
5120             const char *buf, size_t length)
5121 {
5122     int ret = -EINVAL;
5123 
5124     if (buf[0] == '1') {
5125         ret = validate_slab_cache(s);
5126         if (ret >= 0)
5127             ret = length;
5128     }
5129     return ret;
5130 }
5131 SLAB_ATTR(validate);
5132 
5133 static ssize_t alloc_calls_show(struct kmem_cache *s, char *buf)
5134 {
5135     if (!(s->flags & SLAB_STORE_USER))
5136         return -ENOSYS;
5137     return list_locations(s, buf, TRACK_ALLOC);
5138 }
5139 SLAB_ATTR_RO(alloc_calls);
5140 
5141 static ssize_t free_calls_show(struct kmem_cache *s, char *buf)
5142 {
5143     if (!(s->flags & SLAB_STORE_USER))
5144         return -ENOSYS;
5145     return list_locations(s, buf, TRACK_FREE);
5146 }
5147 SLAB_ATTR_RO(free_calls);
5148 #endif /* CONFIG_SLUB_DEBUG */
5149 
5150 #ifdef CONFIG_FAILSLAB
5151 static ssize_t failslab_show(struct kmem_cache *s, char *buf)
5152 {
5153     return sprintf(buf, "%d\n", !!(s->flags & SLAB_FAILSLAB));
5154 }
5155 
5156 static ssize_t failslab_store(struct kmem_cache *s, const char *buf,
5157                             size_t length)
5158 {
5159     if (s->refcount > 1)
5160         return -EINVAL;
5161 
5162     s->flags &= ~SLAB_FAILSLAB;
5163     if (buf[0] == '1')
5164         s->flags |= SLAB_FAILSLAB;
5165     return length;
5166 }
5167 SLAB_ATTR(failslab);
5168 #endif
5169 
5170 static ssize_t shrink_show(struct kmem_cache *s, char *buf)
5171 {
5172     return 0;
5173 }
5174 
5175 static ssize_t shrink_store(struct kmem_cache *s,
5176             const char *buf, size_t length)
5177 {
5178     if (buf[0] == '1')
5179         kmem_cache_shrink(s);
5180     else
5181         return -EINVAL;
5182     return length;
5183 }
5184 SLAB_ATTR(shrink);
5185 
5186 #ifdef CONFIG_NUMA
5187 static ssize_t remote_node_defrag_ratio_show(struct kmem_cache *s, char *buf)
5188 {
5189     return sprintf(buf, "%d\n", s->remote_node_defrag_ratio / 10);
5190 }
5191 
5192 static ssize_t remote_node_defrag_ratio_store(struct kmem_cache *s,
5193                 const char *buf, size_t length)
5194 {
5195     unsigned long ratio;
5196     int err;
5197 
5198     err = kstrtoul(buf, 10, &ratio);
5199     if (err)
5200         return err;
5201 
5202     if (ratio <= 100)
5203         s->remote_node_defrag_ratio = ratio * 10;
5204 
5205     return length;
5206 }
5207 SLAB_ATTR(remote_node_defrag_ratio);
5208 #endif
5209 
5210 #ifdef CONFIG_SLUB_STATS
5211 static int show_stat(struct kmem_cache *s, char *buf, enum stat_item si)
5212 {
5213     unsigned long sum  = 0;
5214     int cpu;
5215     int len;
5216     int *data = kmalloc(nr_cpu_ids * sizeof(int), GFP_KERNEL);
5217 
5218     if (!data)
5219         return -ENOMEM;
5220 
5221     for_each_online_cpu(cpu) {
5222         unsigned x = per_cpu_ptr(s->cpu_slab, cpu)->stat[si];
5223 
5224         data[cpu] = x;
5225         sum += x;
5226     }
5227 
5228     len = sprintf(buf, "%lu", sum);
5229 
5230 #ifdef CONFIG_SMP
5231     for_each_online_cpu(cpu) {
5232         if (data[cpu] && len < PAGE_SIZE - 20)
5233             len += sprintf(buf + len, " C%d=%u", cpu, data[cpu]);
5234     }
5235 #endif
5236     kfree(data);
5237     return len + sprintf(buf + len, "\n");
5238 }
5239 
5240 static void clear_stat(struct kmem_cache *s, enum stat_item si)
5241 {
5242     int cpu;
5243 
5244     for_each_online_cpu(cpu)
5245         per_cpu_ptr(s->cpu_slab, cpu)->stat[si] = 0;
5246 }
5247 
5248 #define STAT_ATTR(si, text)                     \
5249 static ssize_t text##_show(struct kmem_cache *s, char *buf) \
5250 {                               \
5251     return show_stat(s, buf, si);               \
5252 }                               \
5253 static ssize_t text##_store(struct kmem_cache *s,       \
5254                 const char *buf, size_t length) \
5255 {                               \
5256     if (buf[0] != '0')                  \
5257         return -EINVAL;                 \
5258     clear_stat(s, si);                  \
5259     return length;                      \
5260 }                               \
5261 SLAB_ATTR(text);                        \
5262 
5263 STAT_ATTR(ALLOC_FASTPATH, alloc_fastpath);
5264 STAT_ATTR(ALLOC_SLOWPATH, alloc_slowpath);
5265 STAT_ATTR(FREE_FASTPATH, free_fastpath);
5266 STAT_ATTR(FREE_SLOWPATH, free_slowpath);
5267 STAT_ATTR(FREE_FROZEN, free_frozen);
5268 STAT_ATTR(FREE_ADD_PARTIAL, free_add_partial);
5269 STAT_ATTR(FREE_REMOVE_PARTIAL, free_remove_partial);
5270 STAT_ATTR(ALLOC_FROM_PARTIAL, alloc_from_partial);
5271 STAT_ATTR(ALLOC_SLAB, alloc_slab);
5272 STAT_ATTR(ALLOC_REFILL, alloc_refill);
5273 STAT_ATTR(ALLOC_NODE_MISMATCH, alloc_node_mismatch);
5274 STAT_ATTR(FREE_SLAB, free_slab);
5275 STAT_ATTR(CPUSLAB_FLUSH, cpuslab_flush);
5276 STAT_ATTR(DEACTIVATE_FULL, deactivate_full);
5277 STAT_ATTR(DEACTIVATE_EMPTY, deactivate_empty);
5278 STAT_ATTR(DEACTIVATE_TO_HEAD, deactivate_to_head);
5279 STAT_ATTR(DEACTIVATE_TO_TAIL, deactivate_to_tail);
5280 STAT_ATTR(DEACTIVATE_REMOTE_FREES, deactivate_remote_frees);
5281 STAT_ATTR(DEACTIVATE_BYPASS, deactivate_bypass);
5282 STAT_ATTR(ORDER_FALLBACK, order_fallback);
5283 STAT_ATTR(CMPXCHG_DOUBLE_CPU_FAIL, cmpxchg_double_cpu_fail);
5284 STAT_ATTR(CMPXCHG_DOUBLE_FAIL, cmpxchg_double_fail);
5285 STAT_ATTR(CPU_PARTIAL_ALLOC, cpu_partial_alloc);
5286 STAT_ATTR(CPU_PARTIAL_FREE, cpu_partial_free);
5287 STAT_ATTR(CPU_PARTIAL_NODE, cpu_partial_node);
5288 STAT_ATTR(CPU_PARTIAL_DRAIN, cpu_partial_drain);
5289 #endif
5290 
5291 static struct attribute *slab_attrs[] = {
5292     &slab_size_attr.attr,
5293     &object_size_attr.attr,
5294     &objs_per_slab_attr.attr,
5295     &order_attr.attr,
5296     &min_partial_attr.attr,
5297     &cpu_partial_attr.attr,
5298     &objects_attr.attr,
5299     &objects_partial_attr.attr,
5300     &partial_attr.attr,
5301     &cpu_slabs_attr.attr,
5302     &ctor_attr.attr,
5303     &aliases_attr.attr,
5304     &align_attr.attr,
5305     &hwcache_align_attr.attr,
5306     &reclaim_account_attr.attr,
5307     &destroy_by_rcu_attr.attr,
5308     &shrink_attr.attr,
5309     &reserved_attr.attr,
5310     &slabs_cpu_partial_attr.attr,
5311 #ifdef CONFIG_SLUB_DEBUG
5312     &total_objects_attr.attr,
5313     &slabs_attr.attr,
5314     &sanity_checks_attr.attr,
5315     &trace_attr.attr,
5316     &red_zone_attr.attr,
5317     &poison_attr.attr,
5318     &store_user_attr.attr,
5319     &validate_attr.attr,
5320     &alloc_calls_attr.attr,
5321     &free_calls_attr.attr,
5322 #endif
5323 #ifdef CONFIG_ZONE_DMA
5324     &cache_dma_attr.attr,
5325 #endif
5326 #ifdef CONFIG_NUMA
5327     &remote_node_defrag_ratio_attr.attr,
5328 #endif
5329 #ifdef CONFIG_SLUB_STATS
5330     &alloc_fastpath_attr.attr,
5331     &alloc_slowpath_attr.attr,
5332     &free_fastpath_attr.attr,
5333     &free_slowpath_attr.attr,
5334     &free_frozen_attr.attr,
5335     &free_add_partial_attr.attr,
5336     &free_remove_partial_attr.attr,
5337     &alloc_from_partial_attr.attr,
5338     &alloc_slab_attr.attr,
5339     &alloc_refill_attr.attr,
5340     &alloc_node_mismatch_attr.attr,
5341     &free_slab_attr.attr,
5342     &cpuslab_flush_attr.attr,
5343     &deactivate_full_attr.attr,
5344     &deactivate_empty_attr.attr,
5345     &deactivate_to_head_attr.attr,
5346     &deactivate_to_tail_attr.attr,
5347     &deactivate_remote_frees_attr.attr,
5348     &deactivate_bypass_attr.attr,
5349     &order_fallback_attr.attr,
5350     &cmpxchg_double_fail_attr.attr,
5351     &cmpxchg_double_cpu_fail_attr.attr,
5352     &cpu_partial_alloc_attr.attr,
5353     &cpu_partial_free_attr.attr,
5354     &cpu_partial_node_attr.attr,
5355     &cpu_partial_drain_attr.attr,
5356 #endif
5357 #ifdef CONFIG_FAILSLAB
5358     &failslab_attr.attr,
5359 #endif
5360 
5361     NULL
5362 };
5363 
5364 static struct attribute_group slab_attr_group = {
5365     .attrs = slab_attrs,
5366 };
5367 
5368 static ssize_t slab_attr_show(struct kobject *kobj,
5369                 struct attribute *attr,
5370                 char *buf)
5371 {
5372     struct slab_attribute *attribute;
5373     struct kmem_cache *s;
5374     int err;
5375 
5376     attribute = to_slab_attr(attr);
5377     s = to_slab(kobj);
5378 
5379     if (!attribute->show)
5380         return -EIO;
5381 
5382     err = attribute->show(s, buf);
5383 
5384     return err;
5385 }
5386 
5387 static ssize_t slab_attr_store(struct kobject *kobj,
5388                 struct attribute *attr,
5389                 const char *buf, size_t len)
5390 {
5391     struct slab_attribute *attribute;
5392     struct kmem_cache *s;
5393     int err;
5394 
5395     attribute = to_slab_attr(attr);
5396     s = to_slab(kobj);
5397 
5398     if (!attribute->store)
5399         return -EIO;
5400 
5401     err = attribute->store(s, buf, len);
5402 #ifdef CONFIG_MEMCG
5403     if (slab_state >= FULL && err >= 0 && is_root_cache(s)) {
5404         struct kmem_cache *c;
5405 
5406         mutex_lock(&slab_mutex);
5407         if (s->max_attr_size < len)
5408             s->max_attr_size = len;
5409 
5410         /*
5411          * This is a best effort propagation, so this function's return
5412          * value will be determined by the parent cache only. This is
5413          * basically because not all attributes will have a well
5414          * defined semantics for rollbacks - most of the actions will
5415          * have permanent effects.
5416          *
5417          * Returning the error value of any of the children that fail
5418          * is not 100 % defined, in the sense that users seeing the
5419          * error code won't be able to know anything about the state of
5420          * the cache.
5421          *
5422          * Only returning the error code for the parent cache at least
5423          * has well defined semantics. The cache being written to
5424          * directly either failed or succeeded, in which case we loop
5425          * through the descendants with best-effort propagation.
5426          */
5427         for_each_memcg_cache(c, s)
5428             attribute->store(c, buf, len);
5429         mutex_unlock(&slab_mutex);
5430     }
5431 #endif
5432     return err;
5433 }
5434 
5435 static void memcg_propagate_slab_attrs(struct kmem_cache *s)
5436 {
5437 #ifdef CONFIG_MEMCG
5438     int i;
5439     char *buffer = NULL;
5440     struct kmem_cache *root_cache;
5441 
5442     if (is_root_cache(s))
5443         return;
5444 
5445     root_cache = s->memcg_params.root_cache;
5446 
5447     /*
5448      * This mean this cache had no attribute written. Therefore, no point
5449      * in copying default values around
5450      */
5451     if (!root_cache->max_attr_size)
5452         return;
5453 
5454     for (i = 0; i < ARRAY_SIZE(slab_attrs); i++) {
5455         char mbuf[64];
5456         char *buf;
5457         struct slab_attribute *attr = to_slab_attr(slab_attrs[i]);
5458 
5459         if (!attr || !attr->store || !attr->show)
5460             continue;
5461 
5462         /*
5463          * It is really bad that we have to allocate here, so we will
5464          * do it only as a fallback. If we actually allocate, though,
5465          * we can just use the allocated buffer until the end.
5466          *
5467          * Most of the slub attributes will tend to be very small in
5468          * size, but sysfs allows buffers up to a page, so they can
5469          * theoretically happen.
5470          */
5471         if (buffer)
5472             buf = buffer;
5473         else if (root_cache->max_attr_size < ARRAY_SIZE(mbuf))
5474             buf = mbuf;
5475         else {
5476             buffer = (char *) get_zeroed_page(GFP_KERNEL);
5477             if (WARN_ON(!buffer))
5478                 continue;
5479             buf = buffer;
5480         }
5481 
5482         attr->show(root_cache, buf);
5483         attr->store(s, buf, strlen(buf));
5484     }
5485 
5486     if (buffer)
5487         free_page((unsigned long)buffer);
5488 #endif
5489 }
5490 
5491 static void kmem_cache_release(struct kobject *k)
5492 {
5493     slab_kmem_cache_release(to_slab(k));
5494 }
5495 
5496 static const struct sysfs_ops slab_sysfs_ops = {
5497     .show = slab_attr_show,
5498     .store = slab_attr_store,
5499 };
5500 
5501 static struct kobj_type slab_ktype = {
5502     .sysfs_ops = &slab_sysfs_ops,
5503     .release = kmem_cache_release,
5504 };
5505 
5506 static int uevent_filter(struct kset *kset, struct kobject *kobj)
5507 {
5508     struct kobj_type *ktype = get_ktype(kobj);
5509 
5510     if (ktype == &slab_ktype)
5511         return 1;
5512     return 0;
5513 }
5514 
5515 static const struct kset_uevent_ops slab_uevent_ops = {
5516     .filter = uevent_filter,
5517 };
5518 
5519 static struct kset *slab_kset;
5520 
5521 static inline struct kset *cache_kset(struct kmem_cache *s)
5522 {
5523 #ifdef CONFIG_MEMCG
5524     if (!is_root_cache(s))
5525         return s->memcg_params.root_cache->memcg_kset;
5526 #endif
5527     return slab_kset;
5528 }
5529 
5530 #define ID_STR_LENGTH 64
5531 
5532 /* Create a unique string id for a slab cache:
5533  *
5534  * Format   :[flags-]size
5535  */
5536 static char *create_unique_id(struct kmem_cache *s)
5537 {
5538     char *name = kmalloc(ID_STR_LENGTH, GFP_KERNEL);
5539     char *p = name;
5540 
5541     BUG_ON(!name);
5542 
5543     *p++ = ':';
5544     /*
5545      * First flags affecting slabcache operations. We will only
5546      * get here for aliasable slabs so we do not need to support
5547      * too many flags. The flags here must cover all flags that
5548      * are matched during merging to guarantee that the id is
5549      * unique.
5550      */
5551     if (s->flags & SLAB_CACHE_DMA)
5552         *p++ = 'd';
5553     if (s->flags & SLAB_RECLAIM_ACCOUNT)
5554         *p++ = 'a';
5555     if (s->flags & SLAB_CONSISTENCY_CHECKS)
5556         *p++ = 'F';
5557     if (!(s->flags & SLAB_NOTRACK))
5558         *p++ = 't';
5559     if (s->flags & SLAB_ACCOUNT)
5560         *p++ = 'A';
5561     if (p != name + 1)
5562         *p++ = '-';
5563     p += sprintf(p, "%07d", s->size);
5564 
5565     BUG_ON(p > name + ID_STR_LENGTH - 1);
5566     return name;
5567 }
5568 
5569 static int sysfs_slab_add(struct kmem_cache *s)
5570 {
5571     int err;
5572     const char *name;
5573     int unmergeable = slab_unmergeable(s);
5574 
5575     if (unmergeable) {
5576         /*
5577          * Slabcache can never be merged so we can use the name proper.
5578          * This is typically the case for debug situations. In that
5579          * case we can catch duplicate names easily.
5580          */
5581         sysfs_remove_link(&slab_kset->kobj, s->name);
5582         name = s->name;
5583     } else {
5584         /*
5585          * Create a unique name for the slab as a target
5586          * for the symlinks.
5587          */
5588         name = create_unique_id(s);
5589     }
5590 
5591     s->kobj.kset = cache_kset(s);
5592     err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, "%s", name);
5593     if (err)
5594         goto out;
5595 
5596     err = sysfs_create_group(&s->kobj, &slab_attr_group);
5597     if (err)
5598         goto out_del_kobj;
5599 
5600 #ifdef CONFIG_MEMCG
5601     if (is_root_cache(s)) {
5602         s->memcg_kset = kset_create_and_add("cgroup", NULL, &s->kobj);
5603         if (!s->memcg_kset) {
5604             err = -ENOMEM;
5605             goto out_del_kobj;
5606         }
5607     }
5608 #endif
5609 
5610     kobject_uevent(&s->kobj, KOBJ_ADD);
5611     if (!unmergeable) {
5612         /* Setup first alias */
5613         sysfs_slab_alias(s, s->name);
5614     }
5615 out:
5616     if (!unmergeable)
5617         kfree(name);
5618     return err;
5619 out_del_kobj:
5620     kobject_del(&s->kobj);
5621     goto out;
5622 }
5623 
5624 void sysfs_slab_remove(struct kmem_cache *s)
5625 {
5626     if (slab_state < FULL)
5627         /*
5628          * Sysfs has not been setup yet so no need to remove the
5629          * cache from sysfs.
5630          */
5631         return;
5632 
5633 #ifdef CONFIG_MEMCG
5634     kset_unregister(s->memcg_kset);
5635 #endif
5636     kobject_uevent(&s->kobj, KOBJ_REMOVE);
5637     kobject_del(&s->kobj);
5638     kobject_put(&s->kobj);
5639 }
5640 
5641 /*
5642  * Need to buffer aliases during bootup until sysfs becomes
5643  * available lest we lose that information.
5644  */
5645 struct saved_alias {
5646     struct kmem_cache *s;
5647     const char *name;
5648     struct saved_alias *next;
5649 };
5650 
5651 static struct saved_alias *alias_list;
5652 
5653 static int sysfs_slab_alias(struct kmem_cache *s, const char *name)
5654 {
5655     struct saved_alias *al;
5656 
5657     if (slab_state == FULL) {
5658         /*
5659          * If we have a leftover link then remove it.
5660          */
5661         sysfs_remove_link(&slab_kset->kobj, name);
5662         return sysfs_create_link(&slab_kset->kobj, &s->kobj, name);
5663     }
5664 
5665     al = kmalloc(sizeof(struct saved_alias), GFP_KERNEL);
5666     if (!al)
5667         return -ENOMEM;
5668 
5669     al->s = s;
5670     al->name = name;
5671     al->next = alias_list;
5672     alias_list = al;
5673     return 0;
5674 }
5675 
5676 static int __init slab_sysfs_init(void)
5677 {
5678     struct kmem_cache *s;
5679     int err;
5680 
5681     mutex_lock(&slab_mutex);
5682 
5683     slab_kset = kset_create_and_add("slab", &slab_uevent_ops, kernel_kobj);
5684     if (!slab_kset) {
5685         mutex_unlock(&slab_mutex);
5686         pr_err("Cannot register slab subsystem.\n");
5687         return -ENOSYS;
5688     }
5689 
5690     slab_state = FULL;
5691 
5692     list_for_each_entry(s, &slab_caches, list) {
5693         err = sysfs_slab_add(s);
5694         if (err)
5695             pr_err("SLUB: Unable to add boot slab %s to sysfs\n",
5696                    s->name);
5697     }
5698 
5699     while (alias_list) {
5700         struct saved_alias *al = alias_list;
5701 
5702         alias_list = alias_list->next;
5703         err = sysfs_slab_alias(al->s, al->name);
5704         if (err)
5705             pr_err("SLUB: Unable to add boot slab alias %s to sysfs\n",
5706                    al->name);
5707         kfree(al);
5708     }
5709 
5710     mutex_unlock(&slab_mutex);
5711     resiliency_test();
5712     return 0;
5713 }
5714 
5715 __initcall(slab_sysfs_init);
5716 #endif /* CONFIG_SYSFS */
5717 
5718 /*
5719  * The /proc/slabinfo ABI
5720  */
5721 #ifdef CONFIG_SLABINFO
5722 void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo)
5723 {
5724     unsigned long nr_slabs = 0;
5725     unsigned long nr_objs = 0;
5726     unsigned long nr_free = 0;
5727     int node;
5728     struct kmem_cache_node *n;
5729 
5730     for_each_kmem_cache_node(s, node, n) {
5731         nr_slabs += node_nr_slabs(n);
5732         nr_objs += node_nr_objs(n);
5733         nr_free += count_partial(n, count_free);
5734     }
5735 
5736     sinfo->active_objs = nr_objs - nr_free;
5737     sinfo->num_objs = nr_objs;
5738     sinfo->active_slabs = nr_slabs;
5739     sinfo->num_slabs = nr_slabs;
5740     sinfo->objects_per_slab = oo_objects(s->oo);
5741     sinfo->cache_order = oo_order(s->oo);
5742 }
5743 
5744 void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *s)
5745 {
5746 }
5747 
5748 ssize_t slabinfo_write(struct file *file, const char __user *buffer,
5749                size_t count, loff_t *ppos)
5750 {
5751     return -EIO;
5752 }
5753 #endif /* CONFIG_SLABINFO */