Back to home page

LXR

 
 

    


0001 /*
0002  * mm/kmemleak.c
0003  *
0004  * Copyright (C) 2008 ARM Limited
0005  * Written by Catalin Marinas <catalin.marinas@arm.com>
0006  *
0007  * This program is free software; you can redistribute it and/or modify
0008  * it under the terms of the GNU General Public License version 2 as
0009  * published by the Free Software Foundation.
0010  *
0011  * This program is distributed in the hope that it will be useful,
0012  * but WITHOUT ANY WARRANTY; without even the implied warranty of
0013  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
0014  * GNU General Public License for more details.
0015  *
0016  * You should have received a copy of the GNU General Public License
0017  * along with this program; if not, write to the Free Software
0018  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
0019  *
0020  *
0021  * For more information on the algorithm and kmemleak usage, please see
0022  * Documentation/dev-tools/kmemleak.rst.
0023  *
0024  * Notes on locking
0025  * ----------------
0026  *
0027  * The following locks and mutexes are used by kmemleak:
0028  *
0029  * - kmemleak_lock (rwlock): protects the object_list modifications and
0030  *   accesses to the object_tree_root. The object_list is the main list
0031  *   holding the metadata (struct kmemleak_object) for the allocated memory
0032  *   blocks. The object_tree_root is a red black tree used to look-up
0033  *   metadata based on a pointer to the corresponding memory block.  The
0034  *   kmemleak_object structures are added to the object_list and
0035  *   object_tree_root in the create_object() function called from the
0036  *   kmemleak_alloc() callback and removed in delete_object() called from the
0037  *   kmemleak_free() callback
0038  * - kmemleak_object.lock (spinlock): protects a kmemleak_object. Accesses to
0039  *   the metadata (e.g. count) are protected by this lock. Note that some
0040  *   members of this structure may be protected by other means (atomic or
0041  *   kmemleak_lock). This lock is also held when scanning the corresponding
0042  *   memory block to avoid the kernel freeing it via the kmemleak_free()
0043  *   callback. This is less heavyweight than holding a global lock like
0044  *   kmemleak_lock during scanning
0045  * - scan_mutex (mutex): ensures that only one thread may scan the memory for
0046  *   unreferenced objects at a time. The gray_list contains the objects which
0047  *   are already referenced or marked as false positives and need to be
0048  *   scanned. This list is only modified during a scanning episode when the
0049  *   scan_mutex is held. At the end of a scan, the gray_list is always empty.
0050  *   Note that the kmemleak_object.use_count is incremented when an object is
0051  *   added to the gray_list and therefore cannot be freed. This mutex also
0052  *   prevents multiple users of the "kmemleak" debugfs file together with
0053  *   modifications to the memory scanning parameters including the scan_thread
0054  *   pointer
0055  *
0056  * Locks and mutexes are acquired/nested in the following order:
0057  *
0058  *   scan_mutex [-> object->lock] -> kmemleak_lock -> other_object->lock (SINGLE_DEPTH_NESTING)
0059  *
0060  * No kmemleak_lock and object->lock nesting is allowed outside scan_mutex
0061  * regions.
0062  *
0063  * The kmemleak_object structures have a use_count incremented or decremented
0064  * using the get_object()/put_object() functions. When the use_count becomes
0065  * 0, this count can no longer be incremented and put_object() schedules the
0066  * kmemleak_object freeing via an RCU callback. All calls to the get_object()
0067  * function must be protected by rcu_read_lock() to avoid accessing a freed
0068  * structure.
0069  */
0070 
0071 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
0072 
0073 #include <linux/init.h>
0074 #include <linux/kernel.h>
0075 #include <linux/list.h>
0076 #include <linux/sched.h>
0077 #include <linux/jiffies.h>
0078 #include <linux/delay.h>
0079 #include <linux/export.h>
0080 #include <linux/kthread.h>
0081 #include <linux/rbtree.h>
0082 #include <linux/fs.h>
0083 #include <linux/debugfs.h>
0084 #include <linux/seq_file.h>
0085 #include <linux/cpumask.h>
0086 #include <linux/spinlock.h>
0087 #include <linux/mutex.h>
0088 #include <linux/rcupdate.h>
0089 #include <linux/stacktrace.h>
0090 #include <linux/cache.h>
0091 #include <linux/percpu.h>
0092 #include <linux/hardirq.h>
0093 #include <linux/bootmem.h>
0094 #include <linux/pfn.h>
0095 #include <linux/mmzone.h>
0096 #include <linux/slab.h>
0097 #include <linux/thread_info.h>
0098 #include <linux/err.h>
0099 #include <linux/uaccess.h>
0100 #include <linux/string.h>
0101 #include <linux/nodemask.h>
0102 #include <linux/mm.h>
0103 #include <linux/workqueue.h>
0104 #include <linux/crc32.h>
0105 
0106 #include <asm/sections.h>
0107 #include <asm/processor.h>
0108 #include <linux/atomic.h>
0109 
0110 #include <linux/kasan.h>
0111 #include <linux/kmemcheck.h>
0112 #include <linux/kmemleak.h>
0113 #include <linux/memory_hotplug.h>
0114 
0115 /*
0116  * Kmemleak configuration and common defines.
0117  */
0118 #define MAX_TRACE       16  /* stack trace length */
0119 #define MSECS_MIN_AGE       5000    /* minimum object age for reporting */
0120 #define SECS_FIRST_SCAN     60  /* delay before the first scan */
0121 #define SECS_SCAN_WAIT      600 /* subsequent auto scanning delay */
0122 #define MAX_SCAN_SIZE       4096    /* maximum size of a scanned block */
0123 
0124 #define BYTES_PER_POINTER   sizeof(void *)
0125 
0126 /* GFP bitmask for kmemleak internal allocations */
0127 #define gfp_kmemleak_mask(gfp)  (((gfp) & (GFP_KERNEL | GFP_ATOMIC)) | \
0128                  __GFP_NORETRY | __GFP_NOMEMALLOC | \
0129                  __GFP_NOWARN)
0130 
0131 /* scanning area inside a memory block */
0132 struct kmemleak_scan_area {
0133     struct hlist_node node;
0134     unsigned long start;
0135     size_t size;
0136 };
0137 
0138 #define KMEMLEAK_GREY   0
0139 #define KMEMLEAK_BLACK  -1
0140 
0141 /*
0142  * Structure holding the metadata for each allocated memory block.
0143  * Modifications to such objects should be made while holding the
0144  * object->lock. Insertions or deletions from object_list, gray_list or
0145  * rb_node are already protected by the corresponding locks or mutex (see
0146  * the notes on locking above). These objects are reference-counted
0147  * (use_count) and freed using the RCU mechanism.
0148  */
0149 struct kmemleak_object {
0150     spinlock_t lock;
0151     unsigned long flags;        /* object status flags */
0152     struct list_head object_list;
0153     struct list_head gray_list;
0154     struct rb_node rb_node;
0155     struct rcu_head rcu;        /* object_list lockless traversal */
0156     /* object usage count; object freed when use_count == 0 */
0157     atomic_t use_count;
0158     unsigned long pointer;
0159     size_t size;
0160     /* minimum number of a pointers found before it is considered leak */
0161     int min_count;
0162     /* the total number of pointers found pointing to this object */
0163     int count;
0164     /* checksum for detecting modified objects */
0165     u32 checksum;
0166     /* memory ranges to be scanned inside an object (empty for all) */
0167     struct hlist_head area_list;
0168     unsigned long trace[MAX_TRACE];
0169     unsigned int trace_len;
0170     unsigned long jiffies;      /* creation timestamp */
0171     pid_t pid;          /* pid of the current task */
0172     char comm[TASK_COMM_LEN];   /* executable name */
0173 };
0174 
0175 /* flag representing the memory block allocation status */
0176 #define OBJECT_ALLOCATED    (1 << 0)
0177 /* flag set after the first reporting of an unreference object */
0178 #define OBJECT_REPORTED     (1 << 1)
0179 /* flag set to not scan the object */
0180 #define OBJECT_NO_SCAN      (1 << 2)
0181 
0182 /* number of bytes to print per line; must be 16 or 32 */
0183 #define HEX_ROW_SIZE        16
0184 /* number of bytes to print at a time (1, 2, 4, 8) */
0185 #define HEX_GROUP_SIZE      1
0186 /* include ASCII after the hex output */
0187 #define HEX_ASCII       1
0188 /* max number of lines to be printed */
0189 #define HEX_MAX_LINES       2
0190 
0191 /* the list of all allocated objects */
0192 static LIST_HEAD(object_list);
0193 /* the list of gray-colored objects (see color_gray comment below) */
0194 static LIST_HEAD(gray_list);
0195 /* search tree for object boundaries */
0196 static struct rb_root object_tree_root = RB_ROOT;
0197 /* rw_lock protecting the access to object_list and object_tree_root */
0198 static DEFINE_RWLOCK(kmemleak_lock);
0199 
0200 /* allocation caches for kmemleak internal data */
0201 static struct kmem_cache *object_cache;
0202 static struct kmem_cache *scan_area_cache;
0203 
0204 /* set if tracing memory operations is enabled */
0205 static int kmemleak_enabled;
0206 /* same as above but only for the kmemleak_free() callback */
0207 static int kmemleak_free_enabled;
0208 /* set in the late_initcall if there were no errors */
0209 static int kmemleak_initialized;
0210 /* enables or disables early logging of the memory operations */
0211 static int kmemleak_early_log = 1;
0212 /* set if a kmemleak warning was issued */
0213 static int kmemleak_warning;
0214 /* set if a fatal kmemleak error has occurred */
0215 static int kmemleak_error;
0216 
0217 /* minimum and maximum address that may be valid pointers */
0218 static unsigned long min_addr = ULONG_MAX;
0219 static unsigned long max_addr;
0220 
0221 static struct task_struct *scan_thread;
0222 /* used to avoid reporting of recently allocated objects */
0223 static unsigned long jiffies_min_age;
0224 static unsigned long jiffies_last_scan;
0225 /* delay between automatic memory scannings */
0226 static signed long jiffies_scan_wait;
0227 /* enables or disables the task stacks scanning */
0228 static int kmemleak_stack_scan = 1;
0229 /* protects the memory scanning, parameters and debug/kmemleak file access */
0230 static DEFINE_MUTEX(scan_mutex);
0231 /* setting kmemleak=on, will set this var, skipping the disable */
0232 static int kmemleak_skip_disable;
0233 /* If there are leaks that can be reported */
0234 static bool kmemleak_found_leaks;
0235 
0236 /*
0237  * Early object allocation/freeing logging. Kmemleak is initialized after the
0238  * kernel allocator. However, both the kernel allocator and kmemleak may
0239  * allocate memory blocks which need to be tracked. Kmemleak defines an
0240  * arbitrary buffer to hold the allocation/freeing information before it is
0241  * fully initialized.
0242  */
0243 
0244 /* kmemleak operation type for early logging */
0245 enum {
0246     KMEMLEAK_ALLOC,
0247     KMEMLEAK_ALLOC_PERCPU,
0248     KMEMLEAK_FREE,
0249     KMEMLEAK_FREE_PART,
0250     KMEMLEAK_FREE_PERCPU,
0251     KMEMLEAK_NOT_LEAK,
0252     KMEMLEAK_IGNORE,
0253     KMEMLEAK_SCAN_AREA,
0254     KMEMLEAK_NO_SCAN
0255 };
0256 
0257 /*
0258  * Structure holding the information passed to kmemleak callbacks during the
0259  * early logging.
0260  */
0261 struct early_log {
0262     int op_type;            /* kmemleak operation type */
0263     const void *ptr;        /* allocated/freed memory block */
0264     size_t size;            /* memory block size */
0265     int min_count;          /* minimum reference count */
0266     unsigned long trace[MAX_TRACE]; /* stack trace */
0267     unsigned int trace_len;     /* stack trace length */
0268 };
0269 
0270 /* early logging buffer and current position */
0271 static struct early_log
0272     early_log[CONFIG_DEBUG_KMEMLEAK_EARLY_LOG_SIZE] __initdata;
0273 static int crt_early_log __initdata;
0274 
0275 static void kmemleak_disable(void);
0276 
0277 /*
0278  * Print a warning and dump the stack trace.
0279  */
0280 #define kmemleak_warn(x...) do {        \
0281     pr_warn(x);             \
0282     dump_stack();               \
0283     kmemleak_warning = 1;           \
0284 } while (0)
0285 
0286 /*
0287  * Macro invoked when a serious kmemleak condition occurred and cannot be
0288  * recovered from. Kmemleak will be disabled and further allocation/freeing
0289  * tracing no longer available.
0290  */
0291 #define kmemleak_stop(x...) do {    \
0292     kmemleak_warn(x);       \
0293     kmemleak_disable();     \
0294 } while (0)
0295 
0296 /*
0297  * Printing of the objects hex dump to the seq file. The number of lines to be
0298  * printed is limited to HEX_MAX_LINES to prevent seq file spamming. The
0299  * actual number of printed bytes depends on HEX_ROW_SIZE. It must be called
0300  * with the object->lock held.
0301  */
0302 static void hex_dump_object(struct seq_file *seq,
0303                 struct kmemleak_object *object)
0304 {
0305     const u8 *ptr = (const u8 *)object->pointer;
0306     size_t len;
0307 
0308     /* limit the number of lines to HEX_MAX_LINES */
0309     len = min_t(size_t, object->size, HEX_MAX_LINES * HEX_ROW_SIZE);
0310 
0311     seq_printf(seq, "  hex dump (first %zu bytes):\n", len);
0312     kasan_disable_current();
0313     seq_hex_dump(seq, "    ", DUMP_PREFIX_NONE, HEX_ROW_SIZE,
0314              HEX_GROUP_SIZE, ptr, len, HEX_ASCII);
0315     kasan_enable_current();
0316 }
0317 
0318 /*
0319  * Object colors, encoded with count and min_count:
0320  * - white - orphan object, not enough references to it (count < min_count)
0321  * - gray  - not orphan, not marked as false positive (min_count == 0) or
0322  *      sufficient references to it (count >= min_count)
0323  * - black - ignore, it doesn't contain references (e.g. text section)
0324  *      (min_count == -1). No function defined for this color.
0325  * Newly created objects don't have any color assigned (object->count == -1)
0326  * before the next memory scan when they become white.
0327  */
0328 static bool color_white(const struct kmemleak_object *object)
0329 {
0330     return object->count != KMEMLEAK_BLACK &&
0331         object->count < object->min_count;
0332 }
0333 
0334 static bool color_gray(const struct kmemleak_object *object)
0335 {
0336     return object->min_count != KMEMLEAK_BLACK &&
0337         object->count >= object->min_count;
0338 }
0339 
0340 /*
0341  * Objects are considered unreferenced only if their color is white, they have
0342  * not be deleted and have a minimum age to avoid false positives caused by
0343  * pointers temporarily stored in CPU registers.
0344  */
0345 static bool unreferenced_object(struct kmemleak_object *object)
0346 {
0347     return (color_white(object) && object->flags & OBJECT_ALLOCATED) &&
0348         time_before_eq(object->jiffies + jiffies_min_age,
0349                    jiffies_last_scan);
0350 }
0351 
0352 /*
0353  * Printing of the unreferenced objects information to the seq file. The
0354  * print_unreferenced function must be called with the object->lock held.
0355  */
0356 static void print_unreferenced(struct seq_file *seq,
0357                    struct kmemleak_object *object)
0358 {
0359     int i;
0360     unsigned int msecs_age = jiffies_to_msecs(jiffies - object->jiffies);
0361 
0362     seq_printf(seq, "unreferenced object 0x%08lx (size %zu):\n",
0363            object->pointer, object->size);
0364     seq_printf(seq, "  comm \"%s\", pid %d, jiffies %lu (age %d.%03ds)\n",
0365            object->comm, object->pid, object->jiffies,
0366            msecs_age / 1000, msecs_age % 1000);
0367     hex_dump_object(seq, object);
0368     seq_printf(seq, "  backtrace:\n");
0369 
0370     for (i = 0; i < object->trace_len; i++) {
0371         void *ptr = (void *)object->trace[i];
0372         seq_printf(seq, "    [<%p>] %pS\n", ptr, ptr);
0373     }
0374 }
0375 
0376 /*
0377  * Print the kmemleak_object information. This function is used mainly for
0378  * debugging special cases when kmemleak operations. It must be called with
0379  * the object->lock held.
0380  */
0381 static void dump_object_info(struct kmemleak_object *object)
0382 {
0383     struct stack_trace trace;
0384 
0385     trace.nr_entries = object->trace_len;
0386     trace.entries = object->trace;
0387 
0388     pr_notice("Object 0x%08lx (size %zu):\n",
0389           object->pointer, object->size);
0390     pr_notice("  comm \"%s\", pid %d, jiffies %lu\n",
0391           object->comm, object->pid, object->jiffies);
0392     pr_notice("  min_count = %d\n", object->min_count);
0393     pr_notice("  count = %d\n", object->count);
0394     pr_notice("  flags = 0x%lx\n", object->flags);
0395     pr_notice("  checksum = %u\n", object->checksum);
0396     pr_notice("  backtrace:\n");
0397     print_stack_trace(&trace, 4);
0398 }
0399 
0400 /*
0401  * Look-up a memory block metadata (kmemleak_object) in the object search
0402  * tree based on a pointer value. If alias is 0, only values pointing to the
0403  * beginning of the memory block are allowed. The kmemleak_lock must be held
0404  * when calling this function.
0405  */
0406 static struct kmemleak_object *lookup_object(unsigned long ptr, int alias)
0407 {
0408     struct rb_node *rb = object_tree_root.rb_node;
0409 
0410     while (rb) {
0411         struct kmemleak_object *object =
0412             rb_entry(rb, struct kmemleak_object, rb_node);
0413         if (ptr < object->pointer)
0414             rb = object->rb_node.rb_left;
0415         else if (object->pointer + object->size <= ptr)
0416             rb = object->rb_node.rb_right;
0417         else if (object->pointer == ptr || alias)
0418             return object;
0419         else {
0420             kmemleak_warn("Found object by alias at 0x%08lx\n",
0421                       ptr);
0422             dump_object_info(object);
0423             break;
0424         }
0425     }
0426     return NULL;
0427 }
0428 
0429 /*
0430  * Increment the object use_count. Return 1 if successful or 0 otherwise. Note
0431  * that once an object's use_count reached 0, the RCU freeing was already
0432  * registered and the object should no longer be used. This function must be
0433  * called under the protection of rcu_read_lock().
0434  */
0435 static int get_object(struct kmemleak_object *object)
0436 {
0437     return atomic_inc_not_zero(&object->use_count);
0438 }
0439 
0440 /*
0441  * RCU callback to free a kmemleak_object.
0442  */
0443 static void free_object_rcu(struct rcu_head *rcu)
0444 {
0445     struct hlist_node *tmp;
0446     struct kmemleak_scan_area *area;
0447     struct kmemleak_object *object =
0448         container_of(rcu, struct kmemleak_object, rcu);
0449 
0450     /*
0451      * Once use_count is 0 (guaranteed by put_object), there is no other
0452      * code accessing this object, hence no need for locking.
0453      */
0454     hlist_for_each_entry_safe(area, tmp, &object->area_list, node) {
0455         hlist_del(&area->node);
0456         kmem_cache_free(scan_area_cache, area);
0457     }
0458     kmem_cache_free(object_cache, object);
0459 }
0460 
0461 /*
0462  * Decrement the object use_count. Once the count is 0, free the object using
0463  * an RCU callback. Since put_object() may be called via the kmemleak_free() ->
0464  * delete_object() path, the delayed RCU freeing ensures that there is no
0465  * recursive call to the kernel allocator. Lock-less RCU object_list traversal
0466  * is also possible.
0467  */
0468 static void put_object(struct kmemleak_object *object)
0469 {
0470     if (!atomic_dec_and_test(&object->use_count))
0471         return;
0472 
0473     /* should only get here after delete_object was called */
0474     WARN_ON(object->flags & OBJECT_ALLOCATED);
0475 
0476     call_rcu(&object->rcu, free_object_rcu);
0477 }
0478 
0479 /*
0480  * Look up an object in the object search tree and increase its use_count.
0481  */
0482 static struct kmemleak_object *find_and_get_object(unsigned long ptr, int alias)
0483 {
0484     unsigned long flags;
0485     struct kmemleak_object *object;
0486 
0487     rcu_read_lock();
0488     read_lock_irqsave(&kmemleak_lock, flags);
0489     object = lookup_object(ptr, alias);
0490     read_unlock_irqrestore(&kmemleak_lock, flags);
0491 
0492     /* check whether the object is still available */
0493     if (object && !get_object(object))
0494         object = NULL;
0495     rcu_read_unlock();
0496 
0497     return object;
0498 }
0499 
0500 /*
0501  * Look up an object in the object search tree and remove it from both
0502  * object_tree_root and object_list. The returned object's use_count should be
0503  * at least 1, as initially set by create_object().
0504  */
0505 static struct kmemleak_object *find_and_remove_object(unsigned long ptr, int alias)
0506 {
0507     unsigned long flags;
0508     struct kmemleak_object *object;
0509 
0510     write_lock_irqsave(&kmemleak_lock, flags);
0511     object = lookup_object(ptr, alias);
0512     if (object) {
0513         rb_erase(&object->rb_node, &object_tree_root);
0514         list_del_rcu(&object->object_list);
0515     }
0516     write_unlock_irqrestore(&kmemleak_lock, flags);
0517 
0518     return object;
0519 }
0520 
0521 /*
0522  * Save stack trace to the given array of MAX_TRACE size.
0523  */
0524 static int __save_stack_trace(unsigned long *trace)
0525 {
0526     struct stack_trace stack_trace;
0527 
0528     stack_trace.max_entries = MAX_TRACE;
0529     stack_trace.nr_entries = 0;
0530     stack_trace.entries = trace;
0531     stack_trace.skip = 2;
0532     save_stack_trace(&stack_trace);
0533 
0534     return stack_trace.nr_entries;
0535 }
0536 
0537 /*
0538  * Create the metadata (struct kmemleak_object) corresponding to an allocated
0539  * memory block and add it to the object_list and object_tree_root.
0540  */
0541 static struct kmemleak_object *create_object(unsigned long ptr, size_t size,
0542                          int min_count, gfp_t gfp)
0543 {
0544     unsigned long flags;
0545     struct kmemleak_object *object, *parent;
0546     struct rb_node **link, *rb_parent;
0547 
0548     object = kmem_cache_alloc(object_cache, gfp_kmemleak_mask(gfp));
0549     if (!object) {
0550         pr_warn("Cannot allocate a kmemleak_object structure\n");
0551         kmemleak_disable();
0552         return NULL;
0553     }
0554 
0555     INIT_LIST_HEAD(&object->object_list);
0556     INIT_LIST_HEAD(&object->gray_list);
0557     INIT_HLIST_HEAD(&object->area_list);
0558     spin_lock_init(&object->lock);
0559     atomic_set(&object->use_count, 1);
0560     object->flags = OBJECT_ALLOCATED;
0561     object->pointer = ptr;
0562     object->size = size;
0563     object->min_count = min_count;
0564     object->count = 0;          /* white color initially */
0565     object->jiffies = jiffies;
0566     object->checksum = 0;
0567 
0568     /* task information */
0569     if (in_irq()) {
0570         object->pid = 0;
0571         strncpy(object->comm, "hardirq", sizeof(object->comm));
0572     } else if (in_softirq()) {
0573         object->pid = 0;
0574         strncpy(object->comm, "softirq", sizeof(object->comm));
0575     } else {
0576         object->pid = current->pid;
0577         /*
0578          * There is a small chance of a race with set_task_comm(),
0579          * however using get_task_comm() here may cause locking
0580          * dependency issues with current->alloc_lock. In the worst
0581          * case, the command line is not correct.
0582          */
0583         strncpy(object->comm, current->comm, sizeof(object->comm));
0584     }
0585 
0586     /* kernel backtrace */
0587     object->trace_len = __save_stack_trace(object->trace);
0588 
0589     write_lock_irqsave(&kmemleak_lock, flags);
0590 
0591     min_addr = min(min_addr, ptr);
0592     max_addr = max(max_addr, ptr + size);
0593     link = &object_tree_root.rb_node;
0594     rb_parent = NULL;
0595     while (*link) {
0596         rb_parent = *link;
0597         parent = rb_entry(rb_parent, struct kmemleak_object, rb_node);
0598         if (ptr + size <= parent->pointer)
0599             link = &parent->rb_node.rb_left;
0600         else if (parent->pointer + parent->size <= ptr)
0601             link = &parent->rb_node.rb_right;
0602         else {
0603             kmemleak_stop("Cannot insert 0x%lx into the object search tree (overlaps existing)\n",
0604                       ptr);
0605             /*
0606              * No need for parent->lock here since "parent" cannot
0607              * be freed while the kmemleak_lock is held.
0608              */
0609             dump_object_info(parent);
0610             kmem_cache_free(object_cache, object);
0611             object = NULL;
0612             goto out;
0613         }
0614     }
0615     rb_link_node(&object->rb_node, rb_parent, link);
0616     rb_insert_color(&object->rb_node, &object_tree_root);
0617 
0618     list_add_tail_rcu(&object->object_list, &object_list);
0619 out:
0620     write_unlock_irqrestore(&kmemleak_lock, flags);
0621     return object;
0622 }
0623 
0624 /*
0625  * Mark the object as not allocated and schedule RCU freeing via put_object().
0626  */
0627 static void __delete_object(struct kmemleak_object *object)
0628 {
0629     unsigned long flags;
0630 
0631     WARN_ON(!(object->flags & OBJECT_ALLOCATED));
0632     WARN_ON(atomic_read(&object->use_count) < 1);
0633 
0634     /*
0635      * Locking here also ensures that the corresponding memory block
0636      * cannot be freed when it is being scanned.
0637      */
0638     spin_lock_irqsave(&object->lock, flags);
0639     object->flags &= ~OBJECT_ALLOCATED;
0640     spin_unlock_irqrestore(&object->lock, flags);
0641     put_object(object);
0642 }
0643 
0644 /*
0645  * Look up the metadata (struct kmemleak_object) corresponding to ptr and
0646  * delete it.
0647  */
0648 static void delete_object_full(unsigned long ptr)
0649 {
0650     struct kmemleak_object *object;
0651 
0652     object = find_and_remove_object(ptr, 0);
0653     if (!object) {
0654 #ifdef DEBUG
0655         kmemleak_warn("Freeing unknown object at 0x%08lx\n",
0656                   ptr);
0657 #endif
0658         return;
0659     }
0660     __delete_object(object);
0661 }
0662 
0663 /*
0664  * Look up the metadata (struct kmemleak_object) corresponding to ptr and
0665  * delete it. If the memory block is partially freed, the function may create
0666  * additional metadata for the remaining parts of the block.
0667  */
0668 static void delete_object_part(unsigned long ptr, size_t size)
0669 {
0670     struct kmemleak_object *object;
0671     unsigned long start, end;
0672 
0673     object = find_and_remove_object(ptr, 1);
0674     if (!object) {
0675 #ifdef DEBUG
0676         kmemleak_warn("Partially freeing unknown object at 0x%08lx (size %zu)\n",
0677                   ptr, size);
0678 #endif
0679         return;
0680     }
0681 
0682     /*
0683      * Create one or two objects that may result from the memory block
0684      * split. Note that partial freeing is only done by free_bootmem() and
0685      * this happens before kmemleak_init() is called. The path below is
0686      * only executed during early log recording in kmemleak_init(), so
0687      * GFP_KERNEL is enough.
0688      */
0689     start = object->pointer;
0690     end = object->pointer + object->size;
0691     if (ptr > start)
0692         create_object(start, ptr - start, object->min_count,
0693                   GFP_KERNEL);
0694     if (ptr + size < end)
0695         create_object(ptr + size, end - ptr - size, object->min_count,
0696                   GFP_KERNEL);
0697 
0698     __delete_object(object);
0699 }
0700 
0701 static void __paint_it(struct kmemleak_object *object, int color)
0702 {
0703     object->min_count = color;
0704     if (color == KMEMLEAK_BLACK)
0705         object->flags |= OBJECT_NO_SCAN;
0706 }
0707 
0708 static void paint_it(struct kmemleak_object *object, int color)
0709 {
0710     unsigned long flags;
0711 
0712     spin_lock_irqsave(&object->lock, flags);
0713     __paint_it(object, color);
0714     spin_unlock_irqrestore(&object->lock, flags);
0715 }
0716 
0717 static void paint_ptr(unsigned long ptr, int color)
0718 {
0719     struct kmemleak_object *object;
0720 
0721     object = find_and_get_object(ptr, 0);
0722     if (!object) {
0723         kmemleak_warn("Trying to color unknown object at 0x%08lx as %s\n",
0724                   ptr,
0725                   (color == KMEMLEAK_GREY) ? "Grey" :
0726                   (color == KMEMLEAK_BLACK) ? "Black" : "Unknown");
0727         return;
0728     }
0729     paint_it(object, color);
0730     put_object(object);
0731 }
0732 
0733 /*
0734  * Mark an object permanently as gray-colored so that it can no longer be
0735  * reported as a leak. This is used in general to mark a false positive.
0736  */
0737 static void make_gray_object(unsigned long ptr)
0738 {
0739     paint_ptr(ptr, KMEMLEAK_GREY);
0740 }
0741 
0742 /*
0743  * Mark the object as black-colored so that it is ignored from scans and
0744  * reporting.
0745  */
0746 static void make_black_object(unsigned long ptr)
0747 {
0748     paint_ptr(ptr, KMEMLEAK_BLACK);
0749 }
0750 
0751 /*
0752  * Add a scanning area to the object. If at least one such area is added,
0753  * kmemleak will only scan these ranges rather than the whole memory block.
0754  */
0755 static void add_scan_area(unsigned long ptr, size_t size, gfp_t gfp)
0756 {
0757     unsigned long flags;
0758     struct kmemleak_object *object;
0759     struct kmemleak_scan_area *area;
0760 
0761     object = find_and_get_object(ptr, 1);
0762     if (!object) {
0763         kmemleak_warn("Adding scan area to unknown object at 0x%08lx\n",
0764                   ptr);
0765         return;
0766     }
0767 
0768     area = kmem_cache_alloc(scan_area_cache, gfp_kmemleak_mask(gfp));
0769     if (!area) {
0770         pr_warn("Cannot allocate a scan area\n");
0771         goto out;
0772     }
0773 
0774     spin_lock_irqsave(&object->lock, flags);
0775     if (size == SIZE_MAX) {
0776         size = object->pointer + object->size - ptr;
0777     } else if (ptr + size > object->pointer + object->size) {
0778         kmemleak_warn("Scan area larger than object 0x%08lx\n", ptr);
0779         dump_object_info(object);
0780         kmem_cache_free(scan_area_cache, area);
0781         goto out_unlock;
0782     }
0783 
0784     INIT_HLIST_NODE(&area->node);
0785     area->start = ptr;
0786     area->size = size;
0787 
0788     hlist_add_head(&area->node, &object->area_list);
0789 out_unlock:
0790     spin_unlock_irqrestore(&object->lock, flags);
0791 out:
0792     put_object(object);
0793 }
0794 
0795 /*
0796  * Set the OBJECT_NO_SCAN flag for the object corresponding to the give
0797  * pointer. Such object will not be scanned by kmemleak but references to it
0798  * are searched.
0799  */
0800 static void object_no_scan(unsigned long ptr)
0801 {
0802     unsigned long flags;
0803     struct kmemleak_object *object;
0804 
0805     object = find_and_get_object(ptr, 0);
0806     if (!object) {
0807         kmemleak_warn("Not scanning unknown object at 0x%08lx\n", ptr);
0808         return;
0809     }
0810 
0811     spin_lock_irqsave(&object->lock, flags);
0812     object->flags |= OBJECT_NO_SCAN;
0813     spin_unlock_irqrestore(&object->lock, flags);
0814     put_object(object);
0815 }
0816 
0817 /*
0818  * Log an early kmemleak_* call to the early_log buffer. These calls will be
0819  * processed later once kmemleak is fully initialized.
0820  */
0821 static void __init log_early(int op_type, const void *ptr, size_t size,
0822                  int min_count)
0823 {
0824     unsigned long flags;
0825     struct early_log *log;
0826 
0827     if (kmemleak_error) {
0828         /* kmemleak stopped recording, just count the requests */
0829         crt_early_log++;
0830         return;
0831     }
0832 
0833     if (crt_early_log >= ARRAY_SIZE(early_log)) {
0834         crt_early_log++;
0835         kmemleak_disable();
0836         return;
0837     }
0838 
0839     /*
0840      * There is no need for locking since the kernel is still in UP mode
0841      * at this stage. Disabling the IRQs is enough.
0842      */
0843     local_irq_save(flags);
0844     log = &early_log[crt_early_log];
0845     log->op_type = op_type;
0846     log->ptr = ptr;
0847     log->size = size;
0848     log->min_count = min_count;
0849     log->trace_len = __save_stack_trace(log->trace);
0850     crt_early_log++;
0851     local_irq_restore(flags);
0852 }
0853 
0854 /*
0855  * Log an early allocated block and populate the stack trace.
0856  */
0857 static void early_alloc(struct early_log *log)
0858 {
0859     struct kmemleak_object *object;
0860     unsigned long flags;
0861     int i;
0862 
0863     if (!kmemleak_enabled || !log->ptr || IS_ERR(log->ptr))
0864         return;
0865 
0866     /*
0867      * RCU locking needed to ensure object is not freed via put_object().
0868      */
0869     rcu_read_lock();
0870     object = create_object((unsigned long)log->ptr, log->size,
0871                    log->min_count, GFP_ATOMIC);
0872     if (!object)
0873         goto out;
0874     spin_lock_irqsave(&object->lock, flags);
0875     for (i = 0; i < log->trace_len; i++)
0876         object->trace[i] = log->trace[i];
0877     object->trace_len = log->trace_len;
0878     spin_unlock_irqrestore(&object->lock, flags);
0879 out:
0880     rcu_read_unlock();
0881 }
0882 
0883 /*
0884  * Log an early allocated block and populate the stack trace.
0885  */
0886 static void early_alloc_percpu(struct early_log *log)
0887 {
0888     unsigned int cpu;
0889     const void __percpu *ptr = log->ptr;
0890 
0891     for_each_possible_cpu(cpu) {
0892         log->ptr = per_cpu_ptr(ptr, cpu);
0893         early_alloc(log);
0894     }
0895 }
0896 
0897 /**
0898  * kmemleak_alloc - register a newly allocated object
0899  * @ptr:    pointer to beginning of the object
0900  * @size:   size of the object
0901  * @min_count:  minimum number of references to this object. If during memory
0902  *      scanning a number of references less than @min_count is found,
0903  *      the object is reported as a memory leak. If @min_count is 0,
0904  *      the object is never reported as a leak. If @min_count is -1,
0905  *      the object is ignored (not scanned and not reported as a leak)
0906  * @gfp:    kmalloc() flags used for kmemleak internal memory allocations
0907  *
0908  * This function is called from the kernel allocators when a new object
0909  * (memory block) is allocated (kmem_cache_alloc, kmalloc, vmalloc etc.).
0910  */
0911 void __ref kmemleak_alloc(const void *ptr, size_t size, int min_count,
0912               gfp_t gfp)
0913 {
0914     pr_debug("%s(0x%p, %zu, %d)\n", __func__, ptr, size, min_count);
0915 
0916     if (kmemleak_enabled && ptr && !IS_ERR(ptr))
0917         create_object((unsigned long)ptr, size, min_count, gfp);
0918     else if (kmemleak_early_log)
0919         log_early(KMEMLEAK_ALLOC, ptr, size, min_count);
0920 }
0921 EXPORT_SYMBOL_GPL(kmemleak_alloc);
0922 
0923 /**
0924  * kmemleak_alloc_percpu - register a newly allocated __percpu object
0925  * @ptr:    __percpu pointer to beginning of the object
0926  * @size:   size of the object
0927  * @gfp:    flags used for kmemleak internal memory allocations
0928  *
0929  * This function is called from the kernel percpu allocator when a new object
0930  * (memory block) is allocated (alloc_percpu).
0931  */
0932 void __ref kmemleak_alloc_percpu(const void __percpu *ptr, size_t size,
0933                  gfp_t gfp)
0934 {
0935     unsigned int cpu;
0936 
0937     pr_debug("%s(0x%p, %zu)\n", __func__, ptr, size);
0938 
0939     /*
0940      * Percpu allocations are only scanned and not reported as leaks
0941      * (min_count is set to 0).
0942      */
0943     if (kmemleak_enabled && ptr && !IS_ERR(ptr))
0944         for_each_possible_cpu(cpu)
0945             create_object((unsigned long)per_cpu_ptr(ptr, cpu),
0946                       size, 0, gfp);
0947     else if (kmemleak_early_log)
0948         log_early(KMEMLEAK_ALLOC_PERCPU, ptr, size, 0);
0949 }
0950 EXPORT_SYMBOL_GPL(kmemleak_alloc_percpu);
0951 
0952 /**
0953  * kmemleak_free - unregister a previously registered object
0954  * @ptr:    pointer to beginning of the object
0955  *
0956  * This function is called from the kernel allocators when an object (memory
0957  * block) is freed (kmem_cache_free, kfree, vfree etc.).
0958  */
0959 void __ref kmemleak_free(const void *ptr)
0960 {
0961     pr_debug("%s(0x%p)\n", __func__, ptr);
0962 
0963     if (kmemleak_free_enabled && ptr && !IS_ERR(ptr))
0964         delete_object_full((unsigned long)ptr);
0965     else if (kmemleak_early_log)
0966         log_early(KMEMLEAK_FREE, ptr, 0, 0);
0967 }
0968 EXPORT_SYMBOL_GPL(kmemleak_free);
0969 
0970 /**
0971  * kmemleak_free_part - partially unregister a previously registered object
0972  * @ptr:    pointer to the beginning or inside the object. This also
0973  *      represents the start of the range to be freed
0974  * @size:   size to be unregistered
0975  *
0976  * This function is called when only a part of a memory block is freed
0977  * (usually from the bootmem allocator).
0978  */
0979 void __ref kmemleak_free_part(const void *ptr, size_t size)
0980 {
0981     pr_debug("%s(0x%p)\n", __func__, ptr);
0982 
0983     if (kmemleak_enabled && ptr && !IS_ERR(ptr))
0984         delete_object_part((unsigned long)ptr, size);
0985     else if (kmemleak_early_log)
0986         log_early(KMEMLEAK_FREE_PART, ptr, size, 0);
0987 }
0988 EXPORT_SYMBOL_GPL(kmemleak_free_part);
0989 
0990 /**
0991  * kmemleak_free_percpu - unregister a previously registered __percpu object
0992  * @ptr:    __percpu pointer to beginning of the object
0993  *
0994  * This function is called from the kernel percpu allocator when an object
0995  * (memory block) is freed (free_percpu).
0996  */
0997 void __ref kmemleak_free_percpu(const void __percpu *ptr)
0998 {
0999     unsigned int cpu;
1000 
1001     pr_debug("%s(0x%p)\n", __func__, ptr);
1002 
1003     if (kmemleak_free_enabled && ptr && !IS_ERR(ptr))
1004         for_each_possible_cpu(cpu)
1005             delete_object_full((unsigned long)per_cpu_ptr(ptr,
1006                                       cpu));
1007     else if (kmemleak_early_log)
1008         log_early(KMEMLEAK_FREE_PERCPU, ptr, 0, 0);
1009 }
1010 EXPORT_SYMBOL_GPL(kmemleak_free_percpu);
1011 
1012 /**
1013  * kmemleak_update_trace - update object allocation stack trace
1014  * @ptr:    pointer to beginning of the object
1015  *
1016  * Override the object allocation stack trace for cases where the actual
1017  * allocation place is not always useful.
1018  */
1019 void __ref kmemleak_update_trace(const void *ptr)
1020 {
1021     struct kmemleak_object *object;
1022     unsigned long flags;
1023 
1024     pr_debug("%s(0x%p)\n", __func__, ptr);
1025 
1026     if (!kmemleak_enabled || IS_ERR_OR_NULL(ptr))
1027         return;
1028 
1029     object = find_and_get_object((unsigned long)ptr, 1);
1030     if (!object) {
1031 #ifdef DEBUG
1032         kmemleak_warn("Updating stack trace for unknown object at %p\n",
1033                   ptr);
1034 #endif
1035         return;
1036     }
1037 
1038     spin_lock_irqsave(&object->lock, flags);
1039     object->trace_len = __save_stack_trace(object->trace);
1040     spin_unlock_irqrestore(&object->lock, flags);
1041 
1042     put_object(object);
1043 }
1044 EXPORT_SYMBOL(kmemleak_update_trace);
1045 
1046 /**
1047  * kmemleak_not_leak - mark an allocated object as false positive
1048  * @ptr:    pointer to beginning of the object
1049  *
1050  * Calling this function on an object will cause the memory block to no longer
1051  * be reported as leak and always be scanned.
1052  */
1053 void __ref kmemleak_not_leak(const void *ptr)
1054 {
1055     pr_debug("%s(0x%p)\n", __func__, ptr);
1056 
1057     if (kmemleak_enabled && ptr && !IS_ERR(ptr))
1058         make_gray_object((unsigned long)ptr);
1059     else if (kmemleak_early_log)
1060         log_early(KMEMLEAK_NOT_LEAK, ptr, 0, 0);
1061 }
1062 EXPORT_SYMBOL(kmemleak_not_leak);
1063 
1064 /**
1065  * kmemleak_ignore - ignore an allocated object
1066  * @ptr:    pointer to beginning of the object
1067  *
1068  * Calling this function on an object will cause the memory block to be
1069  * ignored (not scanned and not reported as a leak). This is usually done when
1070  * it is known that the corresponding block is not a leak and does not contain
1071  * any references to other allocated memory blocks.
1072  */
1073 void __ref kmemleak_ignore(const void *ptr)
1074 {
1075     pr_debug("%s(0x%p)\n", __func__, ptr);
1076 
1077     if (kmemleak_enabled && ptr && !IS_ERR(ptr))
1078         make_black_object((unsigned long)ptr);
1079     else if (kmemleak_early_log)
1080         log_early(KMEMLEAK_IGNORE, ptr, 0, 0);
1081 }
1082 EXPORT_SYMBOL(kmemleak_ignore);
1083 
1084 /**
1085  * kmemleak_scan_area - limit the range to be scanned in an allocated object
1086  * @ptr:    pointer to beginning or inside the object. This also
1087  *      represents the start of the scan area
1088  * @size:   size of the scan area
1089  * @gfp:    kmalloc() flags used for kmemleak internal memory allocations
1090  *
1091  * This function is used when it is known that only certain parts of an object
1092  * contain references to other objects. Kmemleak will only scan these areas
1093  * reducing the number false negatives.
1094  */
1095 void __ref kmemleak_scan_area(const void *ptr, size_t size, gfp_t gfp)
1096 {
1097     pr_debug("%s(0x%p)\n", __func__, ptr);
1098 
1099     if (kmemleak_enabled && ptr && size && !IS_ERR(ptr))
1100         add_scan_area((unsigned long)ptr, size, gfp);
1101     else if (kmemleak_early_log)
1102         log_early(KMEMLEAK_SCAN_AREA, ptr, size, 0);
1103 }
1104 EXPORT_SYMBOL(kmemleak_scan_area);
1105 
1106 /**
1107  * kmemleak_no_scan - do not scan an allocated object
1108  * @ptr:    pointer to beginning of the object
1109  *
1110  * This function notifies kmemleak not to scan the given memory block. Useful
1111  * in situations where it is known that the given object does not contain any
1112  * references to other objects. Kmemleak will not scan such objects reducing
1113  * the number of false negatives.
1114  */
1115 void __ref kmemleak_no_scan(const void *ptr)
1116 {
1117     pr_debug("%s(0x%p)\n", __func__, ptr);
1118 
1119     if (kmemleak_enabled && ptr && !IS_ERR(ptr))
1120         object_no_scan((unsigned long)ptr);
1121     else if (kmemleak_early_log)
1122         log_early(KMEMLEAK_NO_SCAN, ptr, 0, 0);
1123 }
1124 EXPORT_SYMBOL(kmemleak_no_scan);
1125 
1126 /**
1127  * kmemleak_alloc_phys - similar to kmemleak_alloc but taking a physical
1128  *           address argument
1129  */
1130 void __ref kmemleak_alloc_phys(phys_addr_t phys, size_t size, int min_count,
1131                    gfp_t gfp)
1132 {
1133     if (!IS_ENABLED(CONFIG_HIGHMEM) || PHYS_PFN(phys) < max_low_pfn)
1134         kmemleak_alloc(__va(phys), size, min_count, gfp);
1135 }
1136 EXPORT_SYMBOL(kmemleak_alloc_phys);
1137 
1138 /**
1139  * kmemleak_free_part_phys - similar to kmemleak_free_part but taking a
1140  *               physical address argument
1141  */
1142 void __ref kmemleak_free_part_phys(phys_addr_t phys, size_t size)
1143 {
1144     if (!IS_ENABLED(CONFIG_HIGHMEM) || PHYS_PFN(phys) < max_low_pfn)
1145         kmemleak_free_part(__va(phys), size);
1146 }
1147 EXPORT_SYMBOL(kmemleak_free_part_phys);
1148 
1149 /**
1150  * kmemleak_not_leak_phys - similar to kmemleak_not_leak but taking a physical
1151  *              address argument
1152  */
1153 void __ref kmemleak_not_leak_phys(phys_addr_t phys)
1154 {
1155     if (!IS_ENABLED(CONFIG_HIGHMEM) || PHYS_PFN(phys) < max_low_pfn)
1156         kmemleak_not_leak(__va(phys));
1157 }
1158 EXPORT_SYMBOL(kmemleak_not_leak_phys);
1159 
1160 /**
1161  * kmemleak_ignore_phys - similar to kmemleak_ignore but taking a physical
1162  *            address argument
1163  */
1164 void __ref kmemleak_ignore_phys(phys_addr_t phys)
1165 {
1166     if (!IS_ENABLED(CONFIG_HIGHMEM) || PHYS_PFN(phys) < max_low_pfn)
1167         kmemleak_ignore(__va(phys));
1168 }
1169 EXPORT_SYMBOL(kmemleak_ignore_phys);
1170 
1171 /*
1172  * Update an object's checksum and return true if it was modified.
1173  */
1174 static bool update_checksum(struct kmemleak_object *object)
1175 {
1176     u32 old_csum = object->checksum;
1177 
1178     if (!kmemcheck_is_obj_initialized(object->pointer, object->size))
1179         return false;
1180 
1181     kasan_disable_current();
1182     object->checksum = crc32(0, (void *)object->pointer, object->size);
1183     kasan_enable_current();
1184 
1185     return object->checksum != old_csum;
1186 }
1187 
1188 /*
1189  * Memory scanning is a long process and it needs to be interruptable. This
1190  * function checks whether such interrupt condition occurred.
1191  */
1192 static int scan_should_stop(void)
1193 {
1194     if (!kmemleak_enabled)
1195         return 1;
1196 
1197     /*
1198      * This function may be called from either process or kthread context,
1199      * hence the need to check for both stop conditions.
1200      */
1201     if (current->mm)
1202         return signal_pending(current);
1203     else
1204         return kthread_should_stop();
1205 
1206     return 0;
1207 }
1208 
1209 /*
1210  * Scan a memory block (exclusive range) for valid pointers and add those
1211  * found to the gray list.
1212  */
1213 static void scan_block(void *_start, void *_end,
1214                struct kmemleak_object *scanned)
1215 {
1216     unsigned long *ptr;
1217     unsigned long *start = PTR_ALIGN(_start, BYTES_PER_POINTER);
1218     unsigned long *end = _end - (BYTES_PER_POINTER - 1);
1219     unsigned long flags;
1220 
1221     read_lock_irqsave(&kmemleak_lock, flags);
1222     for (ptr = start; ptr < end; ptr++) {
1223         struct kmemleak_object *object;
1224         unsigned long pointer;
1225 
1226         if (scan_should_stop())
1227             break;
1228 
1229         /* don't scan uninitialized memory */
1230         if (!kmemcheck_is_obj_initialized((unsigned long)ptr,
1231                           BYTES_PER_POINTER))
1232             continue;
1233 
1234         kasan_disable_current();
1235         pointer = *ptr;
1236         kasan_enable_current();
1237 
1238         if (pointer < min_addr || pointer >= max_addr)
1239             continue;
1240 
1241         /*
1242          * No need for get_object() here since we hold kmemleak_lock.
1243          * object->use_count cannot be dropped to 0 while the object
1244          * is still present in object_tree_root and object_list
1245          * (with updates protected by kmemleak_lock).
1246          */
1247         object = lookup_object(pointer, 1);
1248         if (!object)
1249             continue;
1250         if (object == scanned)
1251             /* self referenced, ignore */
1252             continue;
1253 
1254         /*
1255          * Avoid the lockdep recursive warning on object->lock being
1256          * previously acquired in scan_object(). These locks are
1257          * enclosed by scan_mutex.
1258          */
1259         spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING);
1260         if (!color_white(object)) {
1261             /* non-orphan, ignored or new */
1262             spin_unlock(&object->lock);
1263             continue;
1264         }
1265 
1266         /*
1267          * Increase the object's reference count (number of pointers
1268          * to the memory block). If this count reaches the required
1269          * minimum, the object's color will become gray and it will be
1270          * added to the gray_list.
1271          */
1272         object->count++;
1273         if (color_gray(object)) {
1274             /* put_object() called when removing from gray_list */
1275             WARN_ON(!get_object(object));
1276             list_add_tail(&object->gray_list, &gray_list);
1277         }
1278         spin_unlock(&object->lock);
1279     }
1280     read_unlock_irqrestore(&kmemleak_lock, flags);
1281 }
1282 
1283 /*
1284  * Scan a large memory block in MAX_SCAN_SIZE chunks to reduce the latency.
1285  */
1286 static void scan_large_block(void *start, void *end)
1287 {
1288     void *next;
1289 
1290     while (start < end) {
1291         next = min(start + MAX_SCAN_SIZE, end);
1292         scan_block(start, next, NULL);
1293         start = next;
1294         cond_resched();
1295     }
1296 }
1297 
1298 /*
1299  * Scan a memory block corresponding to a kmemleak_object. A condition is
1300  * that object->use_count >= 1.
1301  */
1302 static void scan_object(struct kmemleak_object *object)
1303 {
1304     struct kmemleak_scan_area *area;
1305     unsigned long flags;
1306 
1307     /*
1308      * Once the object->lock is acquired, the corresponding memory block
1309      * cannot be freed (the same lock is acquired in delete_object).
1310      */
1311     spin_lock_irqsave(&object->lock, flags);
1312     if (object->flags & OBJECT_NO_SCAN)
1313         goto out;
1314     if (!(object->flags & OBJECT_ALLOCATED))
1315         /* already freed object */
1316         goto out;
1317     if (hlist_empty(&object->area_list)) {
1318         void *start = (void *)object->pointer;
1319         void *end = (void *)(object->pointer + object->size);
1320         void *next;
1321 
1322         do {
1323             next = min(start + MAX_SCAN_SIZE, end);
1324             scan_block(start, next, object);
1325 
1326             start = next;
1327             if (start >= end)
1328                 break;
1329 
1330             spin_unlock_irqrestore(&object->lock, flags);
1331             cond_resched();
1332             spin_lock_irqsave(&object->lock, flags);
1333         } while (object->flags & OBJECT_ALLOCATED);
1334     } else
1335         hlist_for_each_entry(area, &object->area_list, node)
1336             scan_block((void *)area->start,
1337                    (void *)(area->start + area->size),
1338                    object);
1339 out:
1340     spin_unlock_irqrestore(&object->lock, flags);
1341 }
1342 
1343 /*
1344  * Scan the objects already referenced (gray objects). More objects will be
1345  * referenced and, if there are no memory leaks, all the objects are scanned.
1346  */
1347 static void scan_gray_list(void)
1348 {
1349     struct kmemleak_object *object, *tmp;
1350 
1351     /*
1352      * The list traversal is safe for both tail additions and removals
1353      * from inside the loop. The kmemleak objects cannot be freed from
1354      * outside the loop because their use_count was incremented.
1355      */
1356     object = list_entry(gray_list.next, typeof(*object), gray_list);
1357     while (&object->gray_list != &gray_list) {
1358         cond_resched();
1359 
1360         /* may add new objects to the list */
1361         if (!scan_should_stop())
1362             scan_object(object);
1363 
1364         tmp = list_entry(object->gray_list.next, typeof(*object),
1365                  gray_list);
1366 
1367         /* remove the object from the list and release it */
1368         list_del(&object->gray_list);
1369         put_object(object);
1370 
1371         object = tmp;
1372     }
1373     WARN_ON(!list_empty(&gray_list));
1374 }
1375 
1376 /*
1377  * Scan data sections and all the referenced memory blocks allocated via the
1378  * kernel's standard allocators. This function must be called with the
1379  * scan_mutex held.
1380  */
1381 static void kmemleak_scan(void)
1382 {
1383     unsigned long flags;
1384     struct kmemleak_object *object;
1385     int i;
1386     int new_leaks = 0;
1387 
1388     jiffies_last_scan = jiffies;
1389 
1390     /* prepare the kmemleak_object's */
1391     rcu_read_lock();
1392     list_for_each_entry_rcu(object, &object_list, object_list) {
1393         spin_lock_irqsave(&object->lock, flags);
1394 #ifdef DEBUG
1395         /*
1396          * With a few exceptions there should be a maximum of
1397          * 1 reference to any object at this point.
1398          */
1399         if (atomic_read(&object->use_count) > 1) {
1400             pr_debug("object->use_count = %d\n",
1401                  atomic_read(&object->use_count));
1402             dump_object_info(object);
1403         }
1404 #endif
1405         /* reset the reference count (whiten the object) */
1406         object->count = 0;
1407         if (color_gray(object) && get_object(object))
1408             list_add_tail(&object->gray_list, &gray_list);
1409 
1410         spin_unlock_irqrestore(&object->lock, flags);
1411     }
1412     rcu_read_unlock();
1413 
1414     /* data/bss scanning */
1415     scan_large_block(_sdata, _edata);
1416     scan_large_block(__bss_start, __bss_stop);
1417     scan_large_block(__start_data_ro_after_init, __end_data_ro_after_init);
1418 
1419 #ifdef CONFIG_SMP
1420     /* per-cpu sections scanning */
1421     for_each_possible_cpu(i)
1422         scan_large_block(__per_cpu_start + per_cpu_offset(i),
1423                  __per_cpu_end + per_cpu_offset(i));
1424 #endif
1425 
1426     /*
1427      * Struct page scanning for each node.
1428      */
1429     get_online_mems();
1430     for_each_online_node(i) {
1431         unsigned long start_pfn = node_start_pfn(i);
1432         unsigned long end_pfn = node_end_pfn(i);
1433         unsigned long pfn;
1434 
1435         for (pfn = start_pfn; pfn < end_pfn; pfn++) {
1436             struct page *page;
1437 
1438             if (!pfn_valid(pfn))
1439                 continue;
1440             page = pfn_to_page(pfn);
1441             /* only scan if page is in use */
1442             if (page_count(page) == 0)
1443                 continue;
1444             scan_block(page, page + 1, NULL);
1445         }
1446     }
1447     put_online_mems();
1448 
1449     /*
1450      * Scanning the task stacks (may introduce false negatives).
1451      */
1452     if (kmemleak_stack_scan) {
1453         struct task_struct *p, *g;
1454 
1455         read_lock(&tasklist_lock);
1456         do_each_thread(g, p) {
1457             void *stack = try_get_task_stack(p);
1458             if (stack) {
1459                 scan_block(stack, stack + THREAD_SIZE, NULL);
1460                 put_task_stack(p);
1461             }
1462         } while_each_thread(g, p);
1463         read_unlock(&tasklist_lock);
1464     }
1465 
1466     /*
1467      * Scan the objects already referenced from the sections scanned
1468      * above.
1469      */
1470     scan_gray_list();
1471 
1472     /*
1473      * Check for new or unreferenced objects modified since the previous
1474      * scan and color them gray until the next scan.
1475      */
1476     rcu_read_lock();
1477     list_for_each_entry_rcu(object, &object_list, object_list) {
1478         spin_lock_irqsave(&object->lock, flags);
1479         if (color_white(object) && (object->flags & OBJECT_ALLOCATED)
1480             && update_checksum(object) && get_object(object)) {
1481             /* color it gray temporarily */
1482             object->count = object->min_count;
1483             list_add_tail(&object->gray_list, &gray_list);
1484         }
1485         spin_unlock_irqrestore(&object->lock, flags);
1486     }
1487     rcu_read_unlock();
1488 
1489     /*
1490      * Re-scan the gray list for modified unreferenced objects.
1491      */
1492     scan_gray_list();
1493 
1494     /*
1495      * If scanning was stopped do not report any new unreferenced objects.
1496      */
1497     if (scan_should_stop())
1498         return;
1499 
1500     /*
1501      * Scanning result reporting.
1502      */
1503     rcu_read_lock();
1504     list_for_each_entry_rcu(object, &object_list, object_list) {
1505         spin_lock_irqsave(&object->lock, flags);
1506         if (unreferenced_object(object) &&
1507             !(object->flags & OBJECT_REPORTED)) {
1508             object->flags |= OBJECT_REPORTED;
1509             new_leaks++;
1510         }
1511         spin_unlock_irqrestore(&object->lock, flags);
1512     }
1513     rcu_read_unlock();
1514 
1515     if (new_leaks) {
1516         kmemleak_found_leaks = true;
1517 
1518         pr_info("%d new suspected memory leaks (see /sys/kernel/debug/kmemleak)\n",
1519             new_leaks);
1520     }
1521 
1522 }
1523 
1524 /*
1525  * Thread function performing automatic memory scanning. Unreferenced objects
1526  * at the end of a memory scan are reported but only the first time.
1527  */
1528 static int kmemleak_scan_thread(void *arg)
1529 {
1530     static int first_run = 1;
1531 
1532     pr_info("Automatic memory scanning thread started\n");
1533     set_user_nice(current, 10);
1534 
1535     /*
1536      * Wait before the first scan to allow the system to fully initialize.
1537      */
1538     if (first_run) {
1539         signed long timeout = msecs_to_jiffies(SECS_FIRST_SCAN * 1000);
1540         first_run = 0;
1541         while (timeout && !kthread_should_stop())
1542             timeout = schedule_timeout_interruptible(timeout);
1543     }
1544 
1545     while (!kthread_should_stop()) {
1546         signed long timeout = jiffies_scan_wait;
1547 
1548         mutex_lock(&scan_mutex);
1549         kmemleak_scan();
1550         mutex_unlock(&scan_mutex);
1551 
1552         /* wait before the next scan */
1553         while (timeout && !kthread_should_stop())
1554             timeout = schedule_timeout_interruptible(timeout);
1555     }
1556 
1557     pr_info("Automatic memory scanning thread ended\n");
1558 
1559     return 0;
1560 }
1561 
1562 /*
1563  * Start the automatic memory scanning thread. This function must be called
1564  * with the scan_mutex held.
1565  */
1566 static void start_scan_thread(void)
1567 {
1568     if (scan_thread)
1569         return;
1570     scan_thread = kthread_run(kmemleak_scan_thread, NULL, "kmemleak");
1571     if (IS_ERR(scan_thread)) {
1572         pr_warn("Failed to create the scan thread\n");
1573         scan_thread = NULL;
1574     }
1575 }
1576 
1577 /*
1578  * Stop the automatic memory scanning thread. This function must be called
1579  * with the scan_mutex held.
1580  */
1581 static void stop_scan_thread(void)
1582 {
1583     if (scan_thread) {
1584         kthread_stop(scan_thread);
1585         scan_thread = NULL;
1586     }
1587 }
1588 
1589 /*
1590  * Iterate over the object_list and return the first valid object at or after
1591  * the required position with its use_count incremented. The function triggers
1592  * a memory scanning when the pos argument points to the first position.
1593  */
1594 static void *kmemleak_seq_start(struct seq_file *seq, loff_t *pos)
1595 {
1596     struct kmemleak_object *object;
1597     loff_t n = *pos;
1598     int err;
1599 
1600     err = mutex_lock_interruptible(&scan_mutex);
1601     if (err < 0)
1602         return ERR_PTR(err);
1603 
1604     rcu_read_lock();
1605     list_for_each_entry_rcu(object, &object_list, object_list) {
1606         if (n-- > 0)
1607             continue;
1608         if (get_object(object))
1609             goto out;
1610     }
1611     object = NULL;
1612 out:
1613     return object;
1614 }
1615 
1616 /*
1617  * Return the next object in the object_list. The function decrements the
1618  * use_count of the previous object and increases that of the next one.
1619  */
1620 static void *kmemleak_seq_next(struct seq_file *seq, void *v, loff_t *pos)
1621 {
1622     struct kmemleak_object *prev_obj = v;
1623     struct kmemleak_object *next_obj = NULL;
1624     struct kmemleak_object *obj = prev_obj;
1625 
1626     ++(*pos);
1627 
1628     list_for_each_entry_continue_rcu(obj, &object_list, object_list) {
1629         if (get_object(obj)) {
1630             next_obj = obj;
1631             break;
1632         }
1633     }
1634 
1635     put_object(prev_obj);
1636     return next_obj;
1637 }
1638 
1639 /*
1640  * Decrement the use_count of the last object required, if any.
1641  */
1642 static void kmemleak_seq_stop(struct seq_file *seq, void *v)
1643 {
1644     if (!IS_ERR(v)) {
1645         /*
1646          * kmemleak_seq_start may return ERR_PTR if the scan_mutex
1647          * waiting was interrupted, so only release it if !IS_ERR.
1648          */
1649         rcu_read_unlock();
1650         mutex_unlock(&scan_mutex);
1651         if (v)
1652             put_object(v);
1653     }
1654 }
1655 
1656 /*
1657  * Print the information for an unreferenced object to the seq file.
1658  */
1659 static int kmemleak_seq_show(struct seq_file *seq, void *v)
1660 {
1661     struct kmemleak_object *object = v;
1662     unsigned long flags;
1663 
1664     spin_lock_irqsave(&object->lock, flags);
1665     if ((object->flags & OBJECT_REPORTED) && unreferenced_object(object))
1666         print_unreferenced(seq, object);
1667     spin_unlock_irqrestore(&object->lock, flags);
1668     return 0;
1669 }
1670 
1671 static const struct seq_operations kmemleak_seq_ops = {
1672     .start = kmemleak_seq_start,
1673     .next  = kmemleak_seq_next,
1674     .stop  = kmemleak_seq_stop,
1675     .show  = kmemleak_seq_show,
1676 };
1677 
1678 static int kmemleak_open(struct inode *inode, struct file *file)
1679 {
1680     return seq_open(file, &kmemleak_seq_ops);
1681 }
1682 
1683 static int dump_str_object_info(const char *str)
1684 {
1685     unsigned long flags;
1686     struct kmemleak_object *object;
1687     unsigned long addr;
1688 
1689     if (kstrtoul(str, 0, &addr))
1690         return -EINVAL;
1691     object = find_and_get_object(addr, 0);
1692     if (!object) {
1693         pr_info("Unknown object at 0x%08lx\n", addr);
1694         return -EINVAL;
1695     }
1696 
1697     spin_lock_irqsave(&object->lock, flags);
1698     dump_object_info(object);
1699     spin_unlock_irqrestore(&object->lock, flags);
1700 
1701     put_object(object);
1702     return 0;
1703 }
1704 
1705 /*
1706  * We use grey instead of black to ensure we can do future scans on the same
1707  * objects. If we did not do future scans these black objects could
1708  * potentially contain references to newly allocated objects in the future and
1709  * we'd end up with false positives.
1710  */
1711 static void kmemleak_clear(void)
1712 {
1713     struct kmemleak_object *object;
1714     unsigned long flags;
1715 
1716     rcu_read_lock();
1717     list_for_each_entry_rcu(object, &object_list, object_list) {
1718         spin_lock_irqsave(&object->lock, flags);
1719         if ((object->flags & OBJECT_REPORTED) &&
1720             unreferenced_object(object))
1721             __paint_it(object, KMEMLEAK_GREY);
1722         spin_unlock_irqrestore(&object->lock, flags);
1723     }
1724     rcu_read_unlock();
1725 
1726     kmemleak_found_leaks = false;
1727 }
1728 
1729 static void __kmemleak_do_cleanup(void);
1730 
1731 /*
1732  * File write operation to configure kmemleak at run-time. The following
1733  * commands can be written to the /sys/kernel/debug/kmemleak file:
1734  *   off    - disable kmemleak (irreversible)
1735  *   stack=on   - enable the task stacks scanning
1736  *   stack=off  - disable the tasks stacks scanning
1737  *   scan=on    - start the automatic memory scanning thread
1738  *   scan=off   - stop the automatic memory scanning thread
1739  *   scan=...   - set the automatic memory scanning period in seconds (0 to
1740  *        disable it)
1741  *   scan   - trigger a memory scan
1742  *   clear  - mark all current reported unreferenced kmemleak objects as
1743  *        grey to ignore printing them, or free all kmemleak objects
1744  *        if kmemleak has been disabled.
1745  *   dump=...   - dump information about the object found at the given address
1746  */
1747 static ssize_t kmemleak_write(struct file *file, const char __user *user_buf,
1748                   size_t size, loff_t *ppos)
1749 {
1750     char buf[64];
1751     int buf_size;
1752     int ret;
1753 
1754     buf_size = min(size, (sizeof(buf) - 1));
1755     if (strncpy_from_user(buf, user_buf, buf_size) < 0)
1756         return -EFAULT;
1757     buf[buf_size] = 0;
1758 
1759     ret = mutex_lock_interruptible(&scan_mutex);
1760     if (ret < 0)
1761         return ret;
1762 
1763     if (strncmp(buf, "clear", 5) == 0) {
1764         if (kmemleak_enabled)
1765             kmemleak_clear();
1766         else
1767             __kmemleak_do_cleanup();
1768         goto out;
1769     }
1770 
1771     if (!kmemleak_enabled) {
1772         ret = -EBUSY;
1773         goto out;
1774     }
1775 
1776     if (strncmp(buf, "off", 3) == 0)
1777         kmemleak_disable();
1778     else if (strncmp(buf, "stack=on", 8) == 0)
1779         kmemleak_stack_scan = 1;
1780     else if (strncmp(buf, "stack=off", 9) == 0)
1781         kmemleak_stack_scan = 0;
1782     else if (strncmp(buf, "scan=on", 7) == 0)
1783         start_scan_thread();
1784     else if (strncmp(buf, "scan=off", 8) == 0)
1785         stop_scan_thread();
1786     else if (strncmp(buf, "scan=", 5) == 0) {
1787         unsigned long secs;
1788 
1789         ret = kstrtoul(buf + 5, 0, &secs);
1790         if (ret < 0)
1791             goto out;
1792         stop_scan_thread();
1793         if (secs) {
1794             jiffies_scan_wait = msecs_to_jiffies(secs * 1000);
1795             start_scan_thread();
1796         }
1797     } else if (strncmp(buf, "scan", 4) == 0)
1798         kmemleak_scan();
1799     else if (strncmp(buf, "dump=", 5) == 0)
1800         ret = dump_str_object_info(buf + 5);
1801     else
1802         ret = -EINVAL;
1803 
1804 out:
1805     mutex_unlock(&scan_mutex);
1806     if (ret < 0)
1807         return ret;
1808 
1809     /* ignore the rest of the buffer, only one command at a time */
1810     *ppos += size;
1811     return size;
1812 }
1813 
1814 static const struct file_operations kmemleak_fops = {
1815     .owner      = THIS_MODULE,
1816     .open       = kmemleak_open,
1817     .read       = seq_read,
1818     .write      = kmemleak_write,
1819     .llseek     = seq_lseek,
1820     .release    = seq_release,
1821 };
1822 
1823 static void __kmemleak_do_cleanup(void)
1824 {
1825     struct kmemleak_object *object;
1826 
1827     rcu_read_lock();
1828     list_for_each_entry_rcu(object, &object_list, object_list)
1829         delete_object_full(object->pointer);
1830     rcu_read_unlock();
1831 }
1832 
1833 /*
1834  * Stop the memory scanning thread and free the kmemleak internal objects if
1835  * no previous scan thread (otherwise, kmemleak may still have some useful
1836  * information on memory leaks).
1837  */
1838 static void kmemleak_do_cleanup(struct work_struct *work)
1839 {
1840     stop_scan_thread();
1841 
1842     /*
1843      * Once the scan thread has stopped, it is safe to no longer track
1844      * object freeing. Ordering of the scan thread stopping and the memory
1845      * accesses below is guaranteed by the kthread_stop() function.
1846      */
1847     kmemleak_free_enabled = 0;
1848 
1849     if (!kmemleak_found_leaks)
1850         __kmemleak_do_cleanup();
1851     else
1852         pr_info("Kmemleak disabled without freeing internal data. Reclaim the memory with \"echo clear > /sys/kernel/debug/kmemleak\".\n");
1853 }
1854 
1855 static DECLARE_WORK(cleanup_work, kmemleak_do_cleanup);
1856 
1857 /*
1858  * Disable kmemleak. No memory allocation/freeing will be traced once this
1859  * function is called. Disabling kmemleak is an irreversible operation.
1860  */
1861 static void kmemleak_disable(void)
1862 {
1863     /* atomically check whether it was already invoked */
1864     if (cmpxchg(&kmemleak_error, 0, 1))
1865         return;
1866 
1867     /* stop any memory operation tracing */
1868     kmemleak_enabled = 0;
1869 
1870     /* check whether it is too early for a kernel thread */
1871     if (kmemleak_initialized)
1872         schedule_work(&cleanup_work);
1873     else
1874         kmemleak_free_enabled = 0;
1875 
1876     pr_info("Kernel memory leak detector disabled\n");
1877 }
1878 
1879 /*
1880  * Allow boot-time kmemleak disabling (enabled by default).
1881  */
1882 static int kmemleak_boot_config(char *str)
1883 {
1884     if (!str)
1885         return -EINVAL;
1886     if (strcmp(str, "off") == 0)
1887         kmemleak_disable();
1888     else if (strcmp(str, "on") == 0)
1889         kmemleak_skip_disable = 1;
1890     else
1891         return -EINVAL;
1892     return 0;
1893 }
1894 early_param("kmemleak", kmemleak_boot_config);
1895 
1896 static void __init print_log_trace(struct early_log *log)
1897 {
1898     struct stack_trace trace;
1899 
1900     trace.nr_entries = log->trace_len;
1901     trace.entries = log->trace;
1902 
1903     pr_notice("Early log backtrace:\n");
1904     print_stack_trace(&trace, 2);
1905 }
1906 
1907 /*
1908  * Kmemleak initialization.
1909  */
1910 void __init kmemleak_init(void)
1911 {
1912     int i;
1913     unsigned long flags;
1914 
1915 #ifdef CONFIG_DEBUG_KMEMLEAK_DEFAULT_OFF
1916     if (!kmemleak_skip_disable) {
1917         kmemleak_early_log = 0;
1918         kmemleak_disable();
1919         return;
1920     }
1921 #endif
1922 
1923     jiffies_min_age = msecs_to_jiffies(MSECS_MIN_AGE);
1924     jiffies_scan_wait = msecs_to_jiffies(SECS_SCAN_WAIT * 1000);
1925 
1926     object_cache = KMEM_CACHE(kmemleak_object, SLAB_NOLEAKTRACE);
1927     scan_area_cache = KMEM_CACHE(kmemleak_scan_area, SLAB_NOLEAKTRACE);
1928 
1929     if (crt_early_log > ARRAY_SIZE(early_log))
1930         pr_warn("Early log buffer exceeded (%d), please increase DEBUG_KMEMLEAK_EARLY_LOG_SIZE\n",
1931             crt_early_log);
1932 
1933     /* the kernel is still in UP mode, so disabling the IRQs is enough */
1934     local_irq_save(flags);
1935     kmemleak_early_log = 0;
1936     if (kmemleak_error) {
1937         local_irq_restore(flags);
1938         return;
1939     } else {
1940         kmemleak_enabled = 1;
1941         kmemleak_free_enabled = 1;
1942     }
1943     local_irq_restore(flags);
1944 
1945     /*
1946      * This is the point where tracking allocations is safe. Automatic
1947      * scanning is started during the late initcall. Add the early logged
1948      * callbacks to the kmemleak infrastructure.
1949      */
1950     for (i = 0; i < crt_early_log; i++) {
1951         struct early_log *log = &early_log[i];
1952 
1953         switch (log->op_type) {
1954         case KMEMLEAK_ALLOC:
1955             early_alloc(log);
1956             break;
1957         case KMEMLEAK_ALLOC_PERCPU:
1958             early_alloc_percpu(log);
1959             break;
1960         case KMEMLEAK_FREE:
1961             kmemleak_free(log->ptr);
1962             break;
1963         case KMEMLEAK_FREE_PART:
1964             kmemleak_free_part(log->ptr, log->size);
1965             break;
1966         case KMEMLEAK_FREE_PERCPU:
1967             kmemleak_free_percpu(log->ptr);
1968             break;
1969         case KMEMLEAK_NOT_LEAK:
1970             kmemleak_not_leak(log->ptr);
1971             break;
1972         case KMEMLEAK_IGNORE:
1973             kmemleak_ignore(log->ptr);
1974             break;
1975         case KMEMLEAK_SCAN_AREA:
1976             kmemleak_scan_area(log->ptr, log->size, GFP_KERNEL);
1977             break;
1978         case KMEMLEAK_NO_SCAN:
1979             kmemleak_no_scan(log->ptr);
1980             break;
1981         default:
1982             kmemleak_warn("Unknown early log operation: %d\n",
1983                       log->op_type);
1984         }
1985 
1986         if (kmemleak_warning) {
1987             print_log_trace(log);
1988             kmemleak_warning = 0;
1989         }
1990     }
1991 }
1992 
1993 /*
1994  * Late initialization function.
1995  */
1996 static int __init kmemleak_late_init(void)
1997 {
1998     struct dentry *dentry;
1999 
2000     kmemleak_initialized = 1;
2001 
2002     if (kmemleak_error) {
2003         /*
2004          * Some error occurred and kmemleak was disabled. There is a
2005          * small chance that kmemleak_disable() was called immediately
2006          * after setting kmemleak_initialized and we may end up with
2007          * two clean-up threads but serialized by scan_mutex.
2008          */
2009         schedule_work(&cleanup_work);
2010         return -ENOMEM;
2011     }
2012 
2013     dentry = debugfs_create_file("kmemleak", S_IRUGO, NULL, NULL,
2014                      &kmemleak_fops);
2015     if (!dentry)
2016         pr_warn("Failed to create the debugfs kmemleak file\n");
2017     mutex_lock(&scan_mutex);
2018     start_scan_thread();
2019     mutex_unlock(&scan_mutex);
2020 
2021     pr_info("Kernel memory leak detector initialized\n");
2022 
2023     return 0;
2024 }
2025 late_initcall(kmemleak_late_init);