Back to home page

LXR

 
 

    


0001 /*
0002  *  bootmem - A boot-time physical memory allocator and configurator
0003  *
0004  *  Copyright (C) 1999 Ingo Molnar
0005  *                1999 Kanoj Sarcar, SGI
0006  *                2008 Johannes Weiner
0007  *
0008  * Access to this subsystem has to be serialized externally (which is true
0009  * for the boot process anyway).
0010  */
0011 #include <linux/init.h>
0012 #include <linux/pfn.h>
0013 #include <linux/slab.h>
0014 #include <linux/export.h>
0015 #include <linux/kmemleak.h>
0016 #include <linux/range.h>
0017 #include <linux/memblock.h>
0018 #include <linux/bootmem.h>
0019 
0020 #include <asm/bug.h>
0021 #include <asm/io.h>
0022 
0023 #include "internal.h"
0024 
0025 #ifndef CONFIG_HAVE_MEMBLOCK
0026 #error CONFIG_HAVE_MEMBLOCK not defined
0027 #endif
0028 
0029 #ifndef CONFIG_NEED_MULTIPLE_NODES
0030 struct pglist_data __refdata contig_page_data;
0031 EXPORT_SYMBOL(contig_page_data);
0032 #endif
0033 
0034 unsigned long max_low_pfn;
0035 unsigned long min_low_pfn;
0036 unsigned long max_pfn;
0037 unsigned long long max_possible_pfn;
0038 
0039 static void * __init __alloc_memory_core_early(int nid, u64 size, u64 align,
0040                     u64 goal, u64 limit)
0041 {
0042     void *ptr;
0043     u64 addr;
0044     ulong flags = choose_memblock_flags();
0045 
0046     if (limit > memblock.current_limit)
0047         limit = memblock.current_limit;
0048 
0049 again:
0050     addr = memblock_find_in_range_node(size, align, goal, limit, nid,
0051                        flags);
0052     if (!addr && (flags & MEMBLOCK_MIRROR)) {
0053         flags &= ~MEMBLOCK_MIRROR;
0054         pr_warn("Could not allocate %pap bytes of mirrored memory\n",
0055             &size);
0056         goto again;
0057     }
0058     if (!addr)
0059         return NULL;
0060 
0061     if (memblock_reserve(addr, size))
0062         return NULL;
0063 
0064     ptr = phys_to_virt(addr);
0065     memset(ptr, 0, size);
0066     /*
0067      * The min_count is set to 0 so that bootmem allocated blocks
0068      * are never reported as leaks.
0069      */
0070     kmemleak_alloc(ptr, size, 0, 0);
0071     return ptr;
0072 }
0073 
0074 /*
0075  * free_bootmem_late - free bootmem pages directly to page allocator
0076  * @addr: starting address of the range
0077  * @size: size of the range in bytes
0078  *
0079  * This is only useful when the bootmem allocator has already been torn
0080  * down, but we are still initializing the system.  Pages are given directly
0081  * to the page allocator, no bootmem metadata is updated because it is gone.
0082  */
0083 void __init free_bootmem_late(unsigned long addr, unsigned long size)
0084 {
0085     unsigned long cursor, end;
0086 
0087     kmemleak_free_part_phys(addr, size);
0088 
0089     cursor = PFN_UP(addr);
0090     end = PFN_DOWN(addr + size);
0091 
0092     for (; cursor < end; cursor++) {
0093         __free_pages_bootmem(pfn_to_page(cursor), cursor, 0);
0094         totalram_pages++;
0095     }
0096 }
0097 
0098 static void __init __free_pages_memory(unsigned long start, unsigned long end)
0099 {
0100     int order;
0101 
0102     while (start < end) {
0103         order = min(MAX_ORDER - 1UL, __ffs(start));
0104 
0105         while (start + (1UL << order) > end)
0106             order--;
0107 
0108         __free_pages_bootmem(pfn_to_page(start), start, order);
0109 
0110         start += (1UL << order);
0111     }
0112 }
0113 
0114 static unsigned long __init __free_memory_core(phys_addr_t start,
0115                  phys_addr_t end)
0116 {
0117     unsigned long start_pfn = PFN_UP(start);
0118     unsigned long end_pfn = min_t(unsigned long,
0119                       PFN_DOWN(end), max_low_pfn);
0120 
0121     if (start_pfn > end_pfn)
0122         return 0;
0123 
0124     __free_pages_memory(start_pfn, end_pfn);
0125 
0126     return end_pfn - start_pfn;
0127 }
0128 
0129 static unsigned long __init free_low_memory_core_early(void)
0130 {
0131     unsigned long count = 0;
0132     phys_addr_t start, end;
0133     u64 i;
0134 
0135     memblock_clear_hotplug(0, -1);
0136 
0137     for_each_reserved_mem_region(i, &start, &end)
0138         reserve_bootmem_region(start, end);
0139 
0140     /*
0141      * We need to use NUMA_NO_NODE instead of NODE_DATA(0)->node_id
0142      *  because in some case like Node0 doesn't have RAM installed
0143      *  low ram will be on Node1
0144      */
0145     for_each_free_mem_range(i, NUMA_NO_NODE, MEMBLOCK_NONE, &start, &end,
0146                 NULL)
0147         count += __free_memory_core(start, end);
0148 
0149 #ifdef CONFIG_ARCH_DISCARD_MEMBLOCK
0150     {
0151         phys_addr_t size;
0152 
0153         /* Free memblock.reserved array if it was allocated */
0154         size = get_allocated_memblock_reserved_regions_info(&start);
0155         if (size)
0156             count += __free_memory_core(start, start + size);
0157 
0158         /* Free memblock.memory array if it was allocated */
0159         size = get_allocated_memblock_memory_regions_info(&start);
0160         if (size)
0161             count += __free_memory_core(start, start + size);
0162     }
0163 #endif
0164 
0165     return count;
0166 }
0167 
0168 static int reset_managed_pages_done __initdata;
0169 
0170 void reset_node_managed_pages(pg_data_t *pgdat)
0171 {
0172     struct zone *z;
0173 
0174     for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++)
0175         z->managed_pages = 0;
0176 }
0177 
0178 void __init reset_all_zones_managed_pages(void)
0179 {
0180     struct pglist_data *pgdat;
0181 
0182     if (reset_managed_pages_done)
0183         return;
0184 
0185     for_each_online_pgdat(pgdat)
0186         reset_node_managed_pages(pgdat);
0187 
0188     reset_managed_pages_done = 1;
0189 }
0190 
0191 /**
0192  * free_all_bootmem - release free pages to the buddy allocator
0193  *
0194  * Returns the number of pages actually released.
0195  */
0196 unsigned long __init free_all_bootmem(void)
0197 {
0198     unsigned long pages;
0199 
0200     reset_all_zones_managed_pages();
0201 
0202     pages = free_low_memory_core_early();
0203     totalram_pages += pages;
0204 
0205     return pages;
0206 }
0207 
0208 /**
0209  * free_bootmem_node - mark a page range as usable
0210  * @pgdat: node the range resides on
0211  * @physaddr: starting address of the range
0212  * @size: size of the range in bytes
0213  *
0214  * Partial pages will be considered reserved and left as they are.
0215  *
0216  * The range must reside completely on the specified node.
0217  */
0218 void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
0219                   unsigned long size)
0220 {
0221     memblock_free(physaddr, size);
0222 }
0223 
0224 /**
0225  * free_bootmem - mark a page range as usable
0226  * @addr: starting address of the range
0227  * @size: size of the range in bytes
0228  *
0229  * Partial pages will be considered reserved and left as they are.
0230  *
0231  * The range must be contiguous but may span node boundaries.
0232  */
0233 void __init free_bootmem(unsigned long addr, unsigned long size)
0234 {
0235     memblock_free(addr, size);
0236 }
0237 
0238 static void * __init ___alloc_bootmem_nopanic(unsigned long size,
0239                     unsigned long align,
0240                     unsigned long goal,
0241                     unsigned long limit)
0242 {
0243     void *ptr;
0244 
0245     if (WARN_ON_ONCE(slab_is_available()))
0246         return kzalloc(size, GFP_NOWAIT);
0247 
0248 restart:
0249 
0250     ptr = __alloc_memory_core_early(NUMA_NO_NODE, size, align, goal, limit);
0251 
0252     if (ptr)
0253         return ptr;
0254 
0255     if (goal != 0) {
0256         goal = 0;
0257         goto restart;
0258     }
0259 
0260     return NULL;
0261 }
0262 
0263 /**
0264  * __alloc_bootmem_nopanic - allocate boot memory without panicking
0265  * @size: size of the request in bytes
0266  * @align: alignment of the region
0267  * @goal: preferred starting address of the region
0268  *
0269  * The goal is dropped if it can not be satisfied and the allocation will
0270  * fall back to memory below @goal.
0271  *
0272  * Allocation may happen on any node in the system.
0273  *
0274  * Returns NULL on failure.
0275  */
0276 void * __init __alloc_bootmem_nopanic(unsigned long size, unsigned long align,
0277                     unsigned long goal)
0278 {
0279     unsigned long limit = -1UL;
0280 
0281     return ___alloc_bootmem_nopanic(size, align, goal, limit);
0282 }
0283 
0284 static void * __init ___alloc_bootmem(unsigned long size, unsigned long align,
0285                     unsigned long goal, unsigned long limit)
0286 {
0287     void *mem = ___alloc_bootmem_nopanic(size, align, goal, limit);
0288 
0289     if (mem)
0290         return mem;
0291     /*
0292      * Whoops, we cannot satisfy the allocation request.
0293      */
0294     pr_alert("bootmem alloc of %lu bytes failed!\n", size);
0295     panic("Out of memory");
0296     return NULL;
0297 }
0298 
0299 /**
0300  * __alloc_bootmem - allocate boot memory
0301  * @size: size of the request in bytes
0302  * @align: alignment of the region
0303  * @goal: preferred starting address of the region
0304  *
0305  * The goal is dropped if it can not be satisfied and the allocation will
0306  * fall back to memory below @goal.
0307  *
0308  * Allocation may happen on any node in the system.
0309  *
0310  * The function panics if the request can not be satisfied.
0311  */
0312 void * __init __alloc_bootmem(unsigned long size, unsigned long align,
0313                   unsigned long goal)
0314 {
0315     unsigned long limit = -1UL;
0316 
0317     return ___alloc_bootmem(size, align, goal, limit);
0318 }
0319 
0320 void * __init ___alloc_bootmem_node_nopanic(pg_data_t *pgdat,
0321                            unsigned long size,
0322                            unsigned long align,
0323                            unsigned long goal,
0324                            unsigned long limit)
0325 {
0326     void *ptr;
0327 
0328 again:
0329     ptr = __alloc_memory_core_early(pgdat->node_id, size, align,
0330                     goal, limit);
0331     if (ptr)
0332         return ptr;
0333 
0334     ptr = __alloc_memory_core_early(NUMA_NO_NODE, size, align,
0335                     goal, limit);
0336     if (ptr)
0337         return ptr;
0338 
0339     if (goal) {
0340         goal = 0;
0341         goto again;
0342     }
0343 
0344     return NULL;
0345 }
0346 
0347 void * __init __alloc_bootmem_node_nopanic(pg_data_t *pgdat, unsigned long size,
0348                    unsigned long align, unsigned long goal)
0349 {
0350     if (WARN_ON_ONCE(slab_is_available()))
0351         return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
0352 
0353     return ___alloc_bootmem_node_nopanic(pgdat, size, align, goal, 0);
0354 }
0355 
0356 static void * __init ___alloc_bootmem_node(pg_data_t *pgdat, unsigned long size,
0357                     unsigned long align, unsigned long goal,
0358                     unsigned long limit)
0359 {
0360     void *ptr;
0361 
0362     ptr = ___alloc_bootmem_node_nopanic(pgdat, size, align, goal, limit);
0363     if (ptr)
0364         return ptr;
0365 
0366     pr_alert("bootmem alloc of %lu bytes failed!\n", size);
0367     panic("Out of memory");
0368     return NULL;
0369 }
0370 
0371 /**
0372  * __alloc_bootmem_node - allocate boot memory from a specific node
0373  * @pgdat: node to allocate from
0374  * @size: size of the request in bytes
0375  * @align: alignment of the region
0376  * @goal: preferred starting address of the region
0377  *
0378  * The goal is dropped if it can not be satisfied and the allocation will
0379  * fall back to memory below @goal.
0380  *
0381  * Allocation may fall back to any node in the system if the specified node
0382  * can not hold the requested memory.
0383  *
0384  * The function panics if the request can not be satisfied.
0385  */
0386 void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size,
0387                    unsigned long align, unsigned long goal)
0388 {
0389     if (WARN_ON_ONCE(slab_is_available()))
0390         return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
0391 
0392     return ___alloc_bootmem_node(pgdat, size, align, goal, 0);
0393 }
0394 
0395 void * __init __alloc_bootmem_node_high(pg_data_t *pgdat, unsigned long size,
0396                    unsigned long align, unsigned long goal)
0397 {
0398     return __alloc_bootmem_node(pgdat, size, align, goal);
0399 }
0400 
0401 
0402 /**
0403  * __alloc_bootmem_low - allocate low boot memory
0404  * @size: size of the request in bytes
0405  * @align: alignment of the region
0406  * @goal: preferred starting address of the region
0407  *
0408  * The goal is dropped if it can not be satisfied and the allocation will
0409  * fall back to memory below @goal.
0410  *
0411  * Allocation may happen on any node in the system.
0412  *
0413  * The function panics if the request can not be satisfied.
0414  */
0415 void * __init __alloc_bootmem_low(unsigned long size, unsigned long align,
0416                   unsigned long goal)
0417 {
0418     return ___alloc_bootmem(size, align, goal, ARCH_LOW_ADDRESS_LIMIT);
0419 }
0420 
0421 void * __init __alloc_bootmem_low_nopanic(unsigned long size,
0422                       unsigned long align,
0423                       unsigned long goal)
0424 {
0425     return ___alloc_bootmem_nopanic(size, align, goal,
0426                     ARCH_LOW_ADDRESS_LIMIT);
0427 }
0428 
0429 /**
0430  * __alloc_bootmem_low_node - allocate low boot memory from a specific node
0431  * @pgdat: node to allocate from
0432  * @size: size of the request in bytes
0433  * @align: alignment of the region
0434  * @goal: preferred starting address of the region
0435  *
0436  * The goal is dropped if it can not be satisfied and the allocation will
0437  * fall back to memory below @goal.
0438  *
0439  * Allocation may fall back to any node in the system if the specified node
0440  * can not hold the requested memory.
0441  *
0442  * The function panics if the request can not be satisfied.
0443  */
0444 void * __init __alloc_bootmem_low_node(pg_data_t *pgdat, unsigned long size,
0445                        unsigned long align, unsigned long goal)
0446 {
0447     if (WARN_ON_ONCE(slab_is_available()))
0448         return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
0449 
0450     return ___alloc_bootmem_node(pgdat, size, align, goal,
0451                      ARCH_LOW_ADDRESS_LIMIT);
0452 }