mm/kfence/kfence_test.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * Test cases for KFENCE memory safety error detector. Since the interface with
0004  * which KFENCE's reports are obtained is via the console, this is the output we
0005  * should verify. For each test case checks the presence (or absence) of
0006  * generated reports. Relies on 'console' tracepoint to capture reports as they
0007  * appear in the kernel log.
0008  *
0009  * Copyright (C) 2020, Google LLC.
0010  * Author: Alexander Potapenko <glider@google.com>
0011  *         Marco Elver <elver@google.com>
0012  */
0013
0014 #include <kunit/test.h>
0015 #include <linux/jiffies.h>
0016 #include <linux/kernel.h>
0017 #include <linux/kfence.h>
0018 #include <linux/mm.h>
0019 #include <linux/random.h>
0020 #include <linux/slab.h>
0021 #include <linux/spinlock.h>
0022 #include <linux/string.h>
0023 #include <linux/tracepoint.h>
0024 #include <trace/events/printk.h>
0025
0026 #include <asm/kfence.h>
0027
0028 #include "kfence.h"
0029
0030 /* May be overridden by <asm/kfence.h>. */
0031 #ifndef arch_kfence_test_address
0032 #define arch_kfence_test_address(addr) (addr)
0033 #endif
0034
0035 #define KFENCE_TEST_REQUIRES(test, cond) do {           \
0036     if (!(cond))                        \
0037         kunit_skip((test), "Test requires: " #cond);    \
0038 } while (0)
0039
0040 /* Report as observed from console. */
0041 static struct {
0042     spinlock_t lock;
0043     int nlines;
0044     char lines[2][256];
0045 } observed = {
0046     .lock = __SPIN_LOCK_UNLOCKED(observed.lock),
0047 };
0048
0049 /* Probe for console output: obtains observed lines of interest. */
0050 static void probe_console(void *ignore, const char *buf, size_t len)
0051 {
0052     unsigned long flags;
0053     int nlines;
0054
0055     spin_lock_irqsave(&observed.lock, flags);
0056     nlines = observed.nlines;
0057
0058     if (strnstr(buf, "BUG: KFENCE: ", len) && strnstr(buf, "test_", len)) {
0059         /*
0060          * KFENCE report and related to the test.
0061          *
0062          * The provided @buf is not NUL-terminated; copy no more than
0063          * @len bytes and let strscpy() add the missing NUL-terminator.
0064          */
0065         strscpy(observed.lines[0], buf, min(len + 1, sizeof(observed.lines[0])));
0066         nlines = 1;
0067     } else if (nlines == 1 && (strnstr(buf, "at 0x", len) || strnstr(buf, "of 0x", len))) {
0068         strscpy(observed.lines[nlines++], buf, min(len + 1, sizeof(observed.lines[0])));
0069     }
0070
0071     WRITE_ONCE(observed.nlines, nlines); /* Publish new nlines. */
0072     spin_unlock_irqrestore(&observed.lock, flags);
0073 }
0074
0075 /* Check if a report related to the test exists. */
0076 static bool report_available(void)
0077 {
0078     return READ_ONCE(observed.nlines) == ARRAY_SIZE(observed.lines);
0079 }
0080
0081 /* Information we expect in a report. */
0082 struct expect_report {
0083     enum kfence_error_type type; /* The type or error. */
0084     void *fn; /* Function pointer to expected function where access occurred. */
0085     char *addr; /* Address at which the bad access occurred. */
0086     bool is_write; /* Is access a write. */
0087 };
0088
0089 static const char *get_access_type(const struct expect_report *r)
0090 {
0091     return r->is_write ? "write" : "read";
0092 }
0093
0094 /* Check observed report matches information in @r. */
0095 static bool report_matches(const struct expect_report *r)
0096 {
0097     unsigned long addr = (unsigned long)r->addr;
0098     bool ret = false;
0099     unsigned long flags;
0100     typeof(observed.lines) expect;
0101     const char *end;
0102     char *cur;
0103
0104     /* Doubled-checked locking. */
0105     if (!report_available())
0106         return false;
0107
0108     /* Generate expected report contents. */
0109
0110     /* Title */
0111     cur = expect[0];
0112     end = &expect[0][sizeof(expect[0]) - 1];
0113     switch (r->type) {
0114     case KFENCE_ERROR_OOB:
0115         cur += scnprintf(cur, end - cur, "BUG: KFENCE: out-of-bounds %s",
0116                  get_access_type(r));
0117         break;
0118     case KFENCE_ERROR_UAF:
0119         cur += scnprintf(cur, end - cur, "BUG: KFENCE: use-after-free %s",
0120                  get_access_type(r));
0121         break;
0122     case KFENCE_ERROR_CORRUPTION:
0123         cur += scnprintf(cur, end - cur, "BUG: KFENCE: memory corruption");
0124         break;
0125     case KFENCE_ERROR_INVALID:
0126         cur += scnprintf(cur, end - cur, "BUG: KFENCE: invalid %s",
0127                  get_access_type(r));
0128         break;
0129     case KFENCE_ERROR_INVALID_FREE:
0130         cur += scnprintf(cur, end - cur, "BUG: KFENCE: invalid free");
0131         break;
0132     }
0133
0134     scnprintf(cur, end - cur, " in %pS", r->fn);
0135     /* The exact offset won't match, remove it; also strip module name. */
0136     cur = strchr(expect[0], '+');
0137     if (cur)
0138         *cur = '\0';
0139
0140     /* Access information */
0141     cur = expect[1];
0142     end = &expect[1][sizeof(expect[1]) - 1];
0143
0144     switch (r->type) {
0145     case KFENCE_ERROR_OOB:
0146         cur += scnprintf(cur, end - cur, "Out-of-bounds %s at", get_access_type(r));
0147         addr = arch_kfence_test_address(addr);
0148         break;
0149     case KFENCE_ERROR_UAF:
0150         cur += scnprintf(cur, end - cur, "Use-after-free %s at", get_access_type(r));
0151         addr = arch_kfence_test_address(addr);
0152         break;
0153     case KFENCE_ERROR_CORRUPTION:
0154         cur += scnprintf(cur, end - cur, "Corrupted memory at");
0155         break;
0156     case KFENCE_ERROR_INVALID:
0157         cur += scnprintf(cur, end - cur, "Invalid %s at", get_access_type(r));
0158         addr = arch_kfence_test_address(addr);
0159         break;
0160     case KFENCE_ERROR_INVALID_FREE:
0161         cur += scnprintf(cur, end - cur, "Invalid free of");
0162         break;
0163     }
0164
0165     cur += scnprintf(cur, end - cur, " 0x%p", (void *)addr);
0166
0167     spin_lock_irqsave(&observed.lock, flags);
0168     if (!report_available())
0169         goto out; /* A new report is being captured. */
0170
0171     /* Finally match expected output to what we actually observed. */
0172     ret = strstr(observed.lines[0], expect[0]) && strstr(observed.lines[1], expect[1]);
0173 out:
0174     spin_unlock_irqrestore(&observed.lock, flags);
0175     return ret;
0176 }
0177
0178 /* ===== Test cases ===== */
0179
0180 #define TEST_PRIV_WANT_MEMCACHE ((void *)1)
0181
0182 /* Cache used by tests; if NULL, allocate from kmalloc instead. */
0183 static struct kmem_cache *test_cache;
0184
0185 static size_t setup_test_cache(struct kunit *test, size_t size, slab_flags_t flags,
0186                    void (*ctor)(void *))
0187 {
0188     if (test->priv != TEST_PRIV_WANT_MEMCACHE)
0189         return size;
0190
0191     kunit_info(test, "%s: size=%zu, ctor=%ps\n", __func__, size, ctor);
0192
0193     /*
0194      * Use SLAB_NOLEAKTRACE to prevent merging with existing caches. Any
0195      * other flag in SLAB_NEVER_MERGE also works. Use SLAB_ACCOUNT to
0196      * allocate via memcg, if enabled.
0197      */
0198     flags |= SLAB_NOLEAKTRACE | SLAB_ACCOUNT;
0199     test_cache = kmem_cache_create("test", size, 1, flags, ctor);
0200     KUNIT_ASSERT_TRUE_MSG(test, test_cache, "could not create cache");
0201
0202     return size;
0203 }
0204
0205 static void test_cache_destroy(void)
0206 {
0207     if (!test_cache)
0208         return;
0209
0210     kmem_cache_destroy(test_cache);
0211     test_cache = NULL;
0212 }
0213
0214 static inline size_t kmalloc_cache_alignment(size_t size)
0215 {
0216     return kmalloc_caches[kmalloc_type(GFP_KERNEL)][__kmalloc_index(size, false)]->align;
0217 }
0218
0219 /* Must always inline to match stack trace against caller. */
0220 static __always_inline void test_free(void *ptr)
0221 {
0222     if (test_cache)
0223         kmem_cache_free(test_cache, ptr);
0224     else
0225         kfree(ptr);
0226 }
0227
0228 /*
0229  * If this should be a KFENCE allocation, and on which side the allocation and
0230  * the closest guard page should be.
0231  */
0232 enum allocation_policy {
0233     ALLOCATE_ANY, /* KFENCE, any side. */
0234     ALLOCATE_LEFT, /* KFENCE, left side of page. */
0235     ALLOCATE_RIGHT, /* KFENCE, right side of page. */
0236     ALLOCATE_NONE, /* No KFENCE allocation. */
0237 };
0238
0239 /*
0240  * Try to get a guarded allocation from KFENCE. Uses either kmalloc() or the
0241  * current test_cache if set up.
0242  */
0243 static void *test_alloc(struct kunit *test, size_t size, gfp_t gfp, enum allocation_policy policy)
0244 {
0245     void *alloc;
0246     unsigned long timeout, resched_after;
0247     const char *policy_name;
0248
0249     switch (policy) {
0250     case ALLOCATE_ANY:
0251         policy_name = "any";
0252         break;
0253     case ALLOCATE_LEFT:
0254         policy_name = "left";
0255         break;
0256     case ALLOCATE_RIGHT:
0257         policy_name = "right";
0258         break;
0259     case ALLOCATE_NONE:
0260         policy_name = "none";
0261         break;
0262     }
0263
0264     kunit_info(test, "%s: size=%zu, gfp=%x, policy=%s, cache=%i\n", __func__, size, gfp,
0265            policy_name, !!test_cache);
0266
0267     /*
0268      * 100x the sample interval should be more than enough to ensure we get
0269      * a KFENCE allocation eventually.
0270      */
0271     timeout = jiffies + msecs_to_jiffies(100 * kfence_sample_interval);
0272     /*
0273      * Especially for non-preemption kernels, ensure the allocation-gate
0274      * timer can catch up: after @resched_after, every failed allocation
0275      * attempt yields, to ensure the allocation-gate timer is scheduled.
0276      */
0277     resched_after = jiffies + msecs_to_jiffies(kfence_sample_interval);
0278     do {
0279         if (test_cache)
0280             alloc = kmem_cache_alloc(test_cache, gfp);
0281         else
0282             alloc = kmalloc(size, gfp);
0283
0284         if (is_kfence_address(alloc)) {
0285             struct slab *slab = virt_to_slab(alloc);
0286             struct kmem_cache *s = test_cache ?:
0287                     kmalloc_caches[kmalloc_type(GFP_KERNEL)][__kmalloc_index(size, false)];
0288
0289             /*
0290              * Verify that various helpers return the right values
0291              * even for KFENCE objects; these are required so that
0292              * memcg accounting works correctly.
0293              */
0294             KUNIT_EXPECT_EQ(test, obj_to_index(s, slab, alloc), 0U);
0295             KUNIT_EXPECT_EQ(test, objs_per_slab(s, slab), 1);
0296
0297             if (policy == ALLOCATE_ANY)
0298                 return alloc;
0299             if (policy == ALLOCATE_LEFT && PAGE_ALIGNED(alloc))
0300                 return alloc;
0301             if (policy == ALLOCATE_RIGHT && !PAGE_ALIGNED(alloc))
0302                 return alloc;
0303         } else if (policy == ALLOCATE_NONE)
0304             return alloc;
0305
0306         test_free(alloc);
0307
0308         if (time_after(jiffies, resched_after))
0309             cond_resched();
0310     } while (time_before(jiffies, timeout));
0311
0312     KUNIT_ASSERT_TRUE_MSG(test, false, "failed to allocate from KFENCE");
0313     return NULL; /* Unreachable. */
0314 }
0315
0316 static void test_out_of_bounds_read(struct kunit *test)
0317 {
0318     size_t size = 32;
0319     struct expect_report expect = {
0320         .type = KFENCE_ERROR_OOB,
0321         .fn = test_out_of_bounds_read,
0322         .is_write = false,
0323     };
0324     char *buf;
0325
0326     setup_test_cache(test, size, 0, NULL);
0327
0328     /*
0329      * If we don't have our own cache, adjust based on alignment, so that we
0330      * actually access guard pages on either side.
0331      */
0332     if (!test_cache)
0333         size = kmalloc_cache_alignment(size);
0334
0335     /* Test both sides. */
0336
0337     buf = test_alloc(test, size, GFP_KERNEL, ALLOCATE_LEFT);
0338     expect.addr = buf - 1;
0339     READ_ONCE(*expect.addr);
0340     KUNIT_EXPECT_TRUE(test, report_matches(&expect));
0341     test_free(buf);
0342
0343     buf = test_alloc(test, size, GFP_KERNEL, ALLOCATE_RIGHT);
0344     expect.addr = buf + size;
0345     READ_ONCE(*expect.addr);
0346     KUNIT_EXPECT_TRUE(test, report_matches(&expect));
0347     test_free(buf);
0348 }
0349
0350 static void test_out_of_bounds_write(struct kunit *test)
0351 {
0352     size_t size = 32;
0353     struct expect_report expect = {
0354         .type = KFENCE_ERROR_OOB,
0355         .fn = test_out_of_bounds_write,
0356         .is_write = true,
0357     };
0358     char *buf;
0359
0360     setup_test_cache(test, size, 0, NULL);
0361     buf = test_alloc(test, size, GFP_KERNEL, ALLOCATE_LEFT);
0362     expect.addr = buf - 1;
0363     WRITE_ONCE(*expect.addr, 42);
0364     KUNIT_EXPECT_TRUE(test, report_matches(&expect));
0365     test_free(buf);
0366 }
0367
0368 static void test_use_after_free_read(struct kunit *test)
0369 {
0370     const size_t size = 32;
0371     struct expect_report expect = {
0372         .type = KFENCE_ERROR_UAF,
0373         .fn = test_use_after_free_read,
0374         .is_write = false,
0375     };
0376
0377     setup_test_cache(test, size, 0, NULL);
0378     expect.addr = test_alloc(test, size, GFP_KERNEL, ALLOCATE_ANY);
0379     test_free(expect.addr);
0380     READ_ONCE(*expect.addr);
0381     KUNIT_EXPECT_TRUE(test, report_matches(&expect));
0382 }
0383
0384 static void test_double_free(struct kunit *test)
0385 {
0386     const size_t size = 32;
0387     struct expect_report expect = {
0388         .type = KFENCE_ERROR_INVALID_FREE,
0389         .fn = test_double_free,
0390     };
0391
0392     setup_test_cache(test, size, 0, NULL);
0393     expect.addr = test_alloc(test, size, GFP_KERNEL, ALLOCATE_ANY);
0394     test_free(expect.addr);
0395     test_free(expect.addr); /* Double-free. */
0396     KUNIT_EXPECT_TRUE(test, report_matches(&expect));
0397 }
0398
0399 static void test_invalid_addr_free(struct kunit *test)
0400 {
0401     const size_t size = 32;
0402     struct expect_report expect = {
0403         .type = KFENCE_ERROR_INVALID_FREE,
0404         .fn = test_invalid_addr_free,
0405     };
0406     char *buf;
0407
0408     setup_test_cache(test, size, 0, NULL);
0409     buf = test_alloc(test, size, GFP_KERNEL, ALLOCATE_ANY);
0410     expect.addr = buf + 1; /* Free on invalid address. */
0411     test_free(expect.addr); /* Invalid address free. */
0412     test_free(buf); /* No error. */
0413     KUNIT_EXPECT_TRUE(test, report_matches(&expect));
0414 }
0415
0416 static void test_corruption(struct kunit *test)
0417 {
0418     size_t size = 32;
0419     struct expect_report expect = {
0420         .type = KFENCE_ERROR_CORRUPTION,
0421         .fn = test_corruption,
0422     };
0423     char *buf;
0424
0425     setup_test_cache(test, size, 0, NULL);
0426
0427     /* Test both sides. */
0428
0429     buf = test_alloc(test, size, GFP_KERNEL, ALLOCATE_LEFT);
0430     expect.addr = buf + size;
0431     WRITE_ONCE(*expect.addr, 42);
0432     test_free(buf);
0433     KUNIT_EXPECT_TRUE(test, report_matches(&expect));
0434
0435     buf = test_alloc(test, size, GFP_KERNEL, ALLOCATE_RIGHT);
0436     expect.addr = buf - 1;
0437     WRITE_ONCE(*expect.addr, 42);
0438     test_free(buf);
0439     KUNIT_EXPECT_TRUE(test, report_matches(&expect));
0440 }
0441
0442 /*
0443  * KFENCE is unable to detect an OOB if the allocation's alignment requirements
0444  * leave a gap between the object and the guard page. Specifically, an
0445  * allocation of e.g. 73 bytes is aligned on 8 and 128 bytes for SLUB or SLAB
0446  * respectively. Therefore it is impossible for the allocated object to
0447  * contiguously line up with the right guard page.
0448  *
0449  * However, we test that an access to memory beyond the gap results in KFENCE
0450  * detecting an OOB access.
0451  */
0452 static void test_kmalloc_aligned_oob_read(struct kunit *test)
0453 {
0454     const size_t size = 73;
0455     const size_t align = kmalloc_cache_alignment(size);
0456     struct expect_report expect = {
0457         .type = KFENCE_ERROR_OOB,
0458         .fn = test_kmalloc_aligned_oob_read,
0459         .is_write = false,
0460     };
0461     char *buf;
0462
0463     buf = test_alloc(test, size, GFP_KERNEL, ALLOCATE_RIGHT);
0464
0465     /*
0466      * The object is offset to the right, so there won't be an OOB to the
0467      * left of it.
0468      */
0469     READ_ONCE(*(buf - 1));
0470     KUNIT_EXPECT_FALSE(test, report_available());
0471
0472     /*
0473      * @buf must be aligned on @align, therefore buf + size belongs to the
0474      * same page -> no OOB.
0475      */
0476     READ_ONCE(*(buf + size));
0477     KUNIT_EXPECT_FALSE(test, report_available());
0478
0479     /* Overflowing by @align bytes will result in an OOB. */
0480     expect.addr = buf + size + align;
0481     READ_ONCE(*expect.addr);
0482     KUNIT_EXPECT_TRUE(test, report_matches(&expect));
0483
0484     test_free(buf);
0485 }
0486
0487 static void test_kmalloc_aligned_oob_write(struct kunit *test)
0488 {
0489     const size_t size = 73;
0490     struct expect_report expect = {
0491         .type = KFENCE_ERROR_CORRUPTION,
0492         .fn = test_kmalloc_aligned_oob_write,
0493     };
0494     char *buf;
0495
0496     buf = test_alloc(test, size, GFP_KERNEL, ALLOCATE_RIGHT);
0497     /*
0498      * The object is offset to the right, so we won't get a page
0499      * fault immediately after it.
0500      */
0501     expect.addr = buf + size;
0502     WRITE_ONCE(*expect.addr, READ_ONCE(*expect.addr) + 1);
0503     KUNIT_EXPECT_FALSE(test, report_available());
0504     test_free(buf);
0505     KUNIT_EXPECT_TRUE(test, report_matches(&expect));
0506 }
0507
0508 /* Test cache shrinking and destroying with KFENCE. */
0509 static void test_shrink_memcache(struct kunit *test)
0510 {
0511     const size_t size = 32;
0512     void *buf;
0513
0514     setup_test_cache(test, size, 0, NULL);
0515     KUNIT_EXPECT_TRUE(test, test_cache);
0516     buf = test_alloc(test, size, GFP_KERNEL, ALLOCATE_ANY);
0517     kmem_cache_shrink(test_cache);
0518     test_free(buf);
0519
0520     KUNIT_EXPECT_FALSE(test, report_available());
0521 }
0522
0523 static void ctor_set_x(void *obj)
0524 {
0525     /* Every object has at least 8 bytes. */
0526     memset(obj, 'x', 8);
0527 }
0528
0529 /* Ensure that SL*B does not modify KFENCE objects on bulk free. */
0530 static void test_free_bulk(struct kunit *test)
0531 {
0532     int iter;
0533
0534     for (iter = 0; iter < 5; iter++) {
0535         const size_t size = setup_test_cache(test, 8 + prandom_u32_max(300), 0,
0536                              (iter & 1) ? ctor_set_x : NULL);
0537         void *objects[] = {
0538             test_alloc(test, size, GFP_KERNEL, ALLOCATE_RIGHT),
0539             test_alloc(test, size, GFP_KERNEL, ALLOCATE_NONE),
0540             test_alloc(test, size, GFP_KERNEL, ALLOCATE_LEFT),
0541             test_alloc(test, size, GFP_KERNEL, ALLOCATE_NONE),
0542             test_alloc(test, size, GFP_KERNEL, ALLOCATE_NONE),
0543         };
0544
0545         kmem_cache_free_bulk(test_cache, ARRAY_SIZE(objects), objects);
0546         KUNIT_ASSERT_FALSE(test, report_available());
0547         test_cache_destroy();
0548     }
0549 }
0550
0551 /* Test init-on-free works. */
0552 static void test_init_on_free(struct kunit *test)
0553 {
0554     const size_t size = 32;
0555     struct expect_report expect = {
0556         .type = KFENCE_ERROR_UAF,
0557         .fn = test_init_on_free,
0558         .is_write = false,
0559     };
0560     int i;
0561
0562     KFENCE_TEST_REQUIRES(test, IS_ENABLED(CONFIG_INIT_ON_FREE_DEFAULT_ON));
0563     /* Assume it hasn't been disabled on command line. */
0564
0565     setup_test_cache(test, size, 0, NULL);
0566     expect.addr = test_alloc(test, size, GFP_KERNEL, ALLOCATE_ANY);
0567     for (i = 0; i < size; i++)
0568         expect.addr[i] = i + 1;
0569     test_free(expect.addr);
0570
0571     for (i = 0; i < size; i++) {
0572         /*
0573          * This may fail if the page was recycled by KFENCE and then
0574          * written to again -- this however, is near impossible with a
0575          * default config.
0576          */
0577         KUNIT_EXPECT_EQ(test, expect.addr[i], (char)0);
0578
0579         if (!i) /* Only check first access to not fail test if page is ever re-protected. */
0580             KUNIT_EXPECT_TRUE(test, report_matches(&expect));
0581     }
0582 }
0583
0584 /* Ensure that constructors work properly. */
0585 static void test_memcache_ctor(struct kunit *test)
0586 {
0587     const size_t size = 32;
0588     char *buf;
0589     int i;
0590
0591     setup_test_cache(test, size, 0, ctor_set_x);
0592     buf = test_alloc(test, size, GFP_KERNEL, ALLOCATE_ANY);
0593
0594     for (i = 0; i < 8; i++)
0595         KUNIT_EXPECT_EQ(test, buf[i], (char)'x');
0596
0597     test_free(buf);
0598
0599     KUNIT_EXPECT_FALSE(test, report_available());
0600 }
0601
0602 /* Test that memory is zeroed if requested. */
0603 static void test_gfpzero(struct kunit *test)
0604 {
0605     const size_t size = PAGE_SIZE; /* PAGE_SIZE so we can use ALLOCATE_ANY. */
0606     char *buf1, *buf2;
0607     int i;
0608
0609     /* Skip if we think it'd take too long. */
0610     KFENCE_TEST_REQUIRES(test, kfence_sample_interval <= 100);
0611
0612     setup_test_cache(test, size, 0, NULL);
0613     buf1 = test_alloc(test, size, GFP_KERNEL, ALLOCATE_ANY);
0614     for (i = 0; i < size; i++)
0615         buf1[i] = i + 1;
0616     test_free(buf1);
0617
0618     /* Try to get same address again -- this can take a while. */
0619     for (i = 0;; i++) {
0620         buf2 = test_alloc(test, size, GFP_KERNEL | __GFP_ZERO, ALLOCATE_ANY);
0621         if (buf1 == buf2)
0622             break;
0623         test_free(buf2);
0624
0625         if (kthread_should_stop() || (i == CONFIG_KFENCE_NUM_OBJECTS)) {
0626             kunit_warn(test, "giving up ... cannot get same object back\n");
0627             return;
0628         }
0629         cond_resched();
0630     }
0631
0632     for (i = 0; i < size; i++)
0633         KUNIT_EXPECT_EQ(test, buf2[i], (char)0);
0634
0635     test_free(buf2);
0636
0637     KUNIT_EXPECT_FALSE(test, report_available());
0638 }
0639
0640 static void test_invalid_access(struct kunit *test)
0641 {
0642     const struct expect_report expect = {
0643         .type = KFENCE_ERROR_INVALID,
0644         .fn = test_invalid_access,
0645         .addr = &__kfence_pool[10],
0646         .is_write = false,
0647     };
0648
0649     READ_ONCE(__kfence_pool[10]);
0650     KUNIT_EXPECT_TRUE(test, report_matches(&expect));
0651 }
0652
0653 /* Test SLAB_TYPESAFE_BY_RCU works. */
0654 static void test_memcache_typesafe_by_rcu(struct kunit *test)
0655 {
0656     const size_t size = 32;
0657     struct expect_report expect = {
0658         .type = KFENCE_ERROR_UAF,
0659         .fn = test_memcache_typesafe_by_rcu,
0660         .is_write = false,
0661     };
0662
0663     setup_test_cache(test, size, SLAB_TYPESAFE_BY_RCU, NULL);
0664     KUNIT_EXPECT_TRUE(test, test_cache); /* Want memcache. */
0665
0666     expect.addr = test_alloc(test, size, GFP_KERNEL, ALLOCATE_ANY);
0667     *expect.addr = 42;
0668
0669     rcu_read_lock();
0670     test_free(expect.addr);
0671     KUNIT_EXPECT_EQ(test, *expect.addr, (char)42);
0672     /*
0673      * Up to this point, memory should not have been freed yet, and
0674      * therefore there should be no KFENCE report from the above access.
0675      */
0676     rcu_read_unlock();
0677
0678     /* Above access to @expect.addr should not have generated a report! */
0679     KUNIT_EXPECT_FALSE(test, report_available());
0680
0681     /* Only after rcu_barrier() is the memory guaranteed to be freed. */
0682     rcu_barrier();
0683
0684     /* Expect use-after-free. */
0685     KUNIT_EXPECT_EQ(test, *expect.addr, (char)42);
0686     KUNIT_EXPECT_TRUE(test, report_matches(&expect));
0687 }
0688
0689 /* Test krealloc(). */
0690 static void test_krealloc(struct kunit *test)
0691 {
0692     const size_t size = 32;
0693     const struct expect_report expect = {
0694         .type = KFENCE_ERROR_UAF,
0695         .fn = test_krealloc,
0696         .addr = test_alloc(test, size, GFP_KERNEL, ALLOCATE_ANY),
0697         .is_write = false,
0698     };
0699     char *buf = expect.addr;
0700     int i;
0701
0702     KUNIT_EXPECT_FALSE(test, test_cache);
0703     KUNIT_EXPECT_EQ(test, ksize(buf), size); /* Precise size match after KFENCE alloc. */
0704     for (i = 0; i < size; i++)
0705         buf[i] = i + 1;
0706
0707     /* Check that we successfully change the size. */
0708     buf = krealloc(buf, size * 3, GFP_KERNEL); /* Grow. */
0709     /* Note: Might no longer be a KFENCE alloc. */
0710     KUNIT_EXPECT_GE(test, ksize(buf), size * 3);
0711     for (i = 0; i < size; i++)
0712         KUNIT_EXPECT_EQ(test, buf[i], (char)(i + 1));
0713     for (; i < size * 3; i++) /* Fill to extra bytes. */
0714         buf[i] = i + 1;
0715
0716     buf = krealloc(buf, size * 2, GFP_KERNEL); /* Shrink. */
0717     KUNIT_EXPECT_GE(test, ksize(buf), size * 2);
0718     for (i = 0; i < size * 2; i++)
0719         KUNIT_EXPECT_EQ(test, buf[i], (char)(i + 1));
0720
0721     buf = krealloc(buf, 0, GFP_KERNEL); /* Free. */
0722     KUNIT_EXPECT_EQ(test, (unsigned long)buf, (unsigned long)ZERO_SIZE_PTR);
0723     KUNIT_ASSERT_FALSE(test, report_available()); /* No reports yet! */
0724
0725     READ_ONCE(*expect.addr); /* Ensure krealloc() actually freed earlier KFENCE object. */
0726     KUNIT_ASSERT_TRUE(test, report_matches(&expect));
0727 }
0728
0729 /* Test that some objects from a bulk allocation belong to KFENCE pool. */
0730 static void test_memcache_alloc_bulk(struct kunit *test)
0731 {
0732     const size_t size = 32;
0733     bool pass = false;
0734     unsigned long timeout;
0735
0736     setup_test_cache(test, size, 0, NULL);
0737     KUNIT_EXPECT_TRUE(test, test_cache); /* Want memcache. */
0738     /*
0739      * 100x the sample interval should be more than enough to ensure we get
0740      * a KFENCE allocation eventually.
0741      */
0742     timeout = jiffies + msecs_to_jiffies(100 * kfence_sample_interval);
0743     do {
0744         void *objects[100];
0745         int i, num = kmem_cache_alloc_bulk(test_cache, GFP_ATOMIC, ARRAY_SIZE(objects),
0746                            objects);
0747         if (!num)
0748             continue;
0749         for (i = 0; i < ARRAY_SIZE(objects); i++) {
0750             if (is_kfence_address(objects[i])) {
0751                 pass = true;
0752                 break;
0753             }
0754         }
0755         kmem_cache_free_bulk(test_cache, num, objects);
0756         /*
0757          * kmem_cache_alloc_bulk() disables interrupts, and calling it
0758          * in a tight loop may not give KFENCE a chance to switch the
0759          * static branch. Call cond_resched() to let KFENCE chime in.
0760          */
0761         cond_resched();
0762     } while (!pass && time_before(jiffies, timeout));
0763
0764     KUNIT_EXPECT_TRUE(test, pass);
0765     KUNIT_EXPECT_FALSE(test, report_available());
0766 }
0767
0768 /*
0769  * KUnit does not provide a way to provide arguments to tests, and we encode
0770  * additional info in the name. Set up 2 tests per test case, one using the
0771  * default allocator, and another using a custom memcache (suffix '-memcache').
0772  */
0773 #define KFENCE_KUNIT_CASE(test_name)                        \
0774     { .run_case = test_name, .name = #test_name },              \
0775     { .run_case = test_name, .name = #test_name "-memcache" }
0776
0777 static struct kunit_case kfence_test_cases[] = {
0778     KFENCE_KUNIT_CASE(test_out_of_bounds_read),
0779     KFENCE_KUNIT_CASE(test_out_of_bounds_write),
0780     KFENCE_KUNIT_CASE(test_use_after_free_read),
0781     KFENCE_KUNIT_CASE(test_double_free),
0782     KFENCE_KUNIT_CASE(test_invalid_addr_free),
0783     KFENCE_KUNIT_CASE(test_corruption),
0784     KFENCE_KUNIT_CASE(test_free_bulk),
0785     KFENCE_KUNIT_CASE(test_init_on_free),
0786     KUNIT_CASE(test_kmalloc_aligned_oob_read),
0787     KUNIT_CASE(test_kmalloc_aligned_oob_write),
0788     KUNIT_CASE(test_shrink_memcache),
0789     KUNIT_CASE(test_memcache_ctor),
0790     KUNIT_CASE(test_invalid_access),
0791     KUNIT_CASE(test_gfpzero),
0792     KUNIT_CASE(test_memcache_typesafe_by_rcu),
0793     KUNIT_CASE(test_krealloc),
0794     KUNIT_CASE(test_memcache_alloc_bulk),
0795     {},
0796 };
0797
0798 /* ===== End test cases ===== */
0799
0800 static int test_init(struct kunit *test)
0801 {
0802     unsigned long flags;
0803     int i;
0804
0805     if (!__kfence_pool)
0806         return -EINVAL;
0807
0808     spin_lock_irqsave(&observed.lock, flags);
0809     for (i = 0; i < ARRAY_SIZE(observed.lines); i++)
0810         observed.lines[i][0] = '\0';
0811     observed.nlines = 0;
0812     spin_unlock_irqrestore(&observed.lock, flags);
0813
0814     /* Any test with 'memcache' in its name will want a memcache. */
0815     if (strstr(test->name, "memcache"))
0816         test->priv = TEST_PRIV_WANT_MEMCACHE;
0817     else
0818         test->priv = NULL;
0819
0820     return 0;
0821 }
0822
0823 static void test_exit(struct kunit *test)
0824 {
0825     test_cache_destroy();
0826 }
0827
0828 static void register_tracepoints(struct tracepoint *tp, void *ignore)
0829 {
0830     check_trace_callback_type_console(probe_console);
0831     if (!strcmp(tp->name, "console"))
0832         WARN_ON(tracepoint_probe_register(tp, probe_console, NULL));
0833 }
0834
0835 static void unregister_tracepoints(struct tracepoint *tp, void *ignore)
0836 {
0837     if (!strcmp(tp->name, "console"))
0838         tracepoint_probe_unregister(tp, probe_console, NULL);
0839 }
0840
0841 static int kfence_suite_init(struct kunit_suite *suite)
0842 {
0843     /*
0844      * Because we want to be able to build the test as a module, we need to
0845      * iterate through all known tracepoints, since the static registration
0846      * won't work here.
0847      */
0848     for_each_kernel_tracepoint(register_tracepoints, NULL);
0849     return 0;
0850 }
0851
0852 static void kfence_suite_exit(struct kunit_suite *suite)
0853 {
0854     for_each_kernel_tracepoint(unregister_tracepoints, NULL);
0855     tracepoint_synchronize_unregister();
0856 }
0857
0858 static struct kunit_suite kfence_test_suite = {
0859     .name = "kfence",
0860     .test_cases = kfence_test_cases,
0861     .init = test_init,
0862     .exit = test_exit,
0863     .suite_init = kfence_suite_init,
0864     .suite_exit = kfence_suite_exit,
0865 };
0866
0867 kunit_test_suites(&kfence_test_suite);
0868
0869 MODULE_LICENSE("GPL v2");
0870 MODULE_AUTHOR("Alexander Potapenko <glider@google.com>, Marco Elver <elver@google.com>");