Back to home page

LXR

 
 

    


0001 #include <linux/kernel.h>
0002 #include <linux/string.h>
0003 #include <linux/mm.h>
0004 #include <linux/highmem.h>
0005 #include <linux/page_ext.h>
0006 #include <linux/poison.h>
0007 #include <linux/ratelimit.h>
0008 
0009 static bool __page_poisoning_enabled __read_mostly;
0010 static bool want_page_poisoning __read_mostly;
0011 
0012 static int early_page_poison_param(char *buf)
0013 {
0014     if (!buf)
0015         return -EINVAL;
0016     return strtobool(buf, &want_page_poisoning);
0017 }
0018 early_param("page_poison", early_page_poison_param);
0019 
0020 bool page_poisoning_enabled(void)
0021 {
0022     return __page_poisoning_enabled;
0023 }
0024 
0025 static bool need_page_poisoning(void)
0026 {
0027     return want_page_poisoning;
0028 }
0029 
0030 static void init_page_poisoning(void)
0031 {
0032     /*
0033      * page poisoning is debug page alloc for some arches. If either
0034      * of those options are enabled, enable poisoning
0035      */
0036     if (!IS_ENABLED(CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC)) {
0037         if (!want_page_poisoning && !debug_pagealloc_enabled())
0038             return;
0039     } else {
0040         if (!want_page_poisoning)
0041             return;
0042     }
0043 
0044     __page_poisoning_enabled = true;
0045 }
0046 
0047 struct page_ext_operations page_poisoning_ops = {
0048     .need = need_page_poisoning,
0049     .init = init_page_poisoning,
0050 };
0051 
0052 static inline void set_page_poison(struct page *page)
0053 {
0054     struct page_ext *page_ext;
0055 
0056     page_ext = lookup_page_ext(page);
0057     if (unlikely(!page_ext))
0058         return;
0059 
0060     __set_bit(PAGE_EXT_DEBUG_POISON, &page_ext->flags);
0061 }
0062 
0063 static inline void clear_page_poison(struct page *page)
0064 {
0065     struct page_ext *page_ext;
0066 
0067     page_ext = lookup_page_ext(page);
0068     if (unlikely(!page_ext))
0069         return;
0070 
0071     __clear_bit(PAGE_EXT_DEBUG_POISON, &page_ext->flags);
0072 }
0073 
0074 bool page_is_poisoned(struct page *page)
0075 {
0076     struct page_ext *page_ext;
0077 
0078     page_ext = lookup_page_ext(page);
0079     if (unlikely(!page_ext))
0080         return false;
0081 
0082     return test_bit(PAGE_EXT_DEBUG_POISON, &page_ext->flags);
0083 }
0084 
0085 static void poison_page(struct page *page)
0086 {
0087     void *addr = kmap_atomic(page);
0088 
0089     set_page_poison(page);
0090     memset(addr, PAGE_POISON, PAGE_SIZE);
0091     kunmap_atomic(addr);
0092 }
0093 
0094 static void poison_pages(struct page *page, int n)
0095 {
0096     int i;
0097 
0098     for (i = 0; i < n; i++)
0099         poison_page(page + i);
0100 }
0101 
0102 static bool single_bit_flip(unsigned char a, unsigned char b)
0103 {
0104     unsigned char error = a ^ b;
0105 
0106     return error && !(error & (error - 1));
0107 }
0108 
0109 static void check_poison_mem(unsigned char *mem, size_t bytes)
0110 {
0111     static DEFINE_RATELIMIT_STATE(ratelimit, 5 * HZ, 10);
0112     unsigned char *start;
0113     unsigned char *end;
0114 
0115     if (IS_ENABLED(CONFIG_PAGE_POISONING_NO_SANITY))
0116         return;
0117 
0118     start = memchr_inv(mem, PAGE_POISON, bytes);
0119     if (!start)
0120         return;
0121 
0122     for (end = mem + bytes - 1; end > start; end--) {
0123         if (*end != PAGE_POISON)
0124             break;
0125     }
0126 
0127     if (!__ratelimit(&ratelimit))
0128         return;
0129     else if (start == end && single_bit_flip(*start, PAGE_POISON))
0130         pr_err("pagealloc: single bit error\n");
0131     else
0132         pr_err("pagealloc: memory corruption\n");
0133 
0134     print_hex_dump(KERN_ERR, "", DUMP_PREFIX_ADDRESS, 16, 1, start,
0135             end - start + 1, 1);
0136     dump_stack();
0137 }
0138 
0139 static void unpoison_page(struct page *page)
0140 {
0141     void *addr;
0142 
0143     if (!page_is_poisoned(page))
0144         return;
0145 
0146     addr = kmap_atomic(page);
0147     check_poison_mem(addr, PAGE_SIZE);
0148     clear_page_poison(page);
0149     kunmap_atomic(addr);
0150 }
0151 
0152 static void unpoison_pages(struct page *page, int n)
0153 {
0154     int i;
0155 
0156     for (i = 0; i < n; i++)
0157         unpoison_page(page + i);
0158 }
0159 
0160 void kernel_poison_pages(struct page *page, int numpages, int enable)
0161 {
0162     if (!page_poisoning_enabled())
0163         return;
0164 
0165     if (enable)
0166         unpoison_pages(page, numpages);
0167     else
0168         poison_pages(page, numpages);
0169 }
0170 
0171 #ifndef CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC
0172 void __kernel_map_pages(struct page *page, int numpages, int enable)
0173 {
0174     /* This function does nothing, all work is done via poison pages */
0175 }
0176 #endif