Back to home page

OSCL-LXR
 
 

     

0001 /*
0002  * This file is subject to the terms and conditions of the GNU General Public
0003  * License.  See the file "COPYING" in the main directory of this archive
0004  * for more details.
0005  *
0006  * Copyright (C) 1994 - 2000 Ralf Baechle
0007  * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
0008  * Kevin D. Kissell, kevink@mips.com and Carsten Langgaard, carstenl@mips.com
0009  * Copyright (C) 2000 MIPS Technologies, Inc.  All rights reserved.
0010  */
0011 #include <linux/bug.h>
0012 #include <linux/init.h>
0013 #include <linux/export.h>
0014 #include <linux/signal.h>
0015 #include <linux/sched.h>
0016 #include <linux/smp.h>
0017 #include <linux/kernel.h>
0018 #include <linux/errno.h>
0019 #include <linux/string.h>
0020 #include <linux/types.h>
0021 #include <linux/pagemap.h>
0022 #include <linux/ptrace.h>
0023 #include <linux/mman.h>
0024 #include <linux/mm.h>
0025 #include <linux/memblock.h>
0026 #include <linux/highmem.h>
0027 #include <linux/swap.h>
0028 #include <linux/proc_fs.h>
0029 #include <linux/pfn.h>
0030 #include <linux/hardirq.h>
0031 #include <linux/gfp.h>
0032 #include <linux/kcore.h>
0033 #include <linux/initrd.h>
0034 
0035 #include <asm/bootinfo.h>
0036 #include <asm/cachectl.h>
0037 #include <asm/cpu.h>
0038 #include <asm/dma.h>
0039 #include <asm/maar.h>
0040 #include <asm/mmu_context.h>
0041 #include <asm/sections.h>
0042 #include <asm/pgalloc.h>
0043 #include <asm/tlb.h>
0044 #include <asm/fixmap.h>
0045 
0046 /*
0047  * We have up to 8 empty zeroed pages so we can map one of the right colour
0048  * when needed.  This is necessary only on R4000 / R4400 SC and MC versions
0049  * where we have to avoid VCED / VECI exceptions for good performance at
0050  * any price.  Since page is never written to after the initialization we
0051  * don't have to care about aliases on other CPUs.
0052  */
0053 unsigned long empty_zero_page, zero_page_mask;
0054 EXPORT_SYMBOL_GPL(empty_zero_page);
0055 EXPORT_SYMBOL(zero_page_mask);
0056 
0057 /*
0058  * Not static inline because used by IP27 special magic initialization code
0059  */
0060 void setup_zero_pages(void)
0061 {
0062     unsigned int order, i;
0063     struct page *page;
0064 
0065     if (cpu_has_vce)
0066         order = 3;
0067     else
0068         order = 0;
0069 
0070     empty_zero_page = __get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
0071     if (!empty_zero_page)
0072         panic("Oh boy, that early out of memory?");
0073 
0074     page = virt_to_page((void *)empty_zero_page);
0075     split_page(page, order);
0076     for (i = 0; i < (1 << order); i++, page++)
0077         mark_page_reserved(page);
0078 
0079     zero_page_mask = ((PAGE_SIZE << order) - 1) & PAGE_MASK;
0080 }
0081 
0082 static void *__kmap_pgprot(struct page *page, unsigned long addr, pgprot_t prot)
0083 {
0084     enum fixed_addresses idx;
0085     unsigned int old_mmid;
0086     unsigned long vaddr, flags, entrylo;
0087     unsigned long old_ctx;
0088     pte_t pte;
0089     int tlbidx;
0090 
0091     BUG_ON(Page_dcache_dirty(page));
0092 
0093     preempt_disable();
0094     pagefault_disable();
0095     idx = (addr >> PAGE_SHIFT) & (FIX_N_COLOURS - 1);
0096     idx += in_interrupt() ? FIX_N_COLOURS : 0;
0097     vaddr = __fix_to_virt(FIX_CMAP_END - idx);
0098     pte = mk_pte(page, prot);
0099 #if defined(CONFIG_XPA)
0100     entrylo = pte_to_entrylo(pte.pte_high);
0101 #elif defined(CONFIG_PHYS_ADDR_T_64BIT) && defined(CONFIG_CPU_MIPS32)
0102     entrylo = pte.pte_high;
0103 #else
0104     entrylo = pte_to_entrylo(pte_val(pte));
0105 #endif
0106 
0107     local_irq_save(flags);
0108     old_ctx = read_c0_entryhi();
0109     write_c0_entryhi(vaddr & (PAGE_MASK << 1));
0110     write_c0_entrylo0(entrylo);
0111     write_c0_entrylo1(entrylo);
0112     if (cpu_has_mmid) {
0113         old_mmid = read_c0_memorymapid();
0114         write_c0_memorymapid(MMID_KERNEL_WIRED);
0115     }
0116 #ifdef CONFIG_XPA
0117     if (cpu_has_xpa) {
0118         entrylo = (pte.pte_low & _PFNX_MASK);
0119         writex_c0_entrylo0(entrylo);
0120         writex_c0_entrylo1(entrylo);
0121     }
0122 #endif
0123     tlbidx = num_wired_entries();
0124     write_c0_wired(tlbidx + 1);
0125     write_c0_index(tlbidx);
0126     mtc0_tlbw_hazard();
0127     tlb_write_indexed();
0128     tlbw_use_hazard();
0129     write_c0_entryhi(old_ctx);
0130     if (cpu_has_mmid)
0131         write_c0_memorymapid(old_mmid);
0132     local_irq_restore(flags);
0133 
0134     return (void*) vaddr;
0135 }
0136 
0137 void *kmap_coherent(struct page *page, unsigned long addr)
0138 {
0139     return __kmap_pgprot(page, addr, PAGE_KERNEL);
0140 }
0141 
0142 void *kmap_noncoherent(struct page *page, unsigned long addr)
0143 {
0144     return __kmap_pgprot(page, addr, PAGE_KERNEL_NC);
0145 }
0146 
0147 void kunmap_coherent(void)
0148 {
0149     unsigned int wired;
0150     unsigned long flags, old_ctx;
0151 
0152     local_irq_save(flags);
0153     old_ctx = read_c0_entryhi();
0154     wired = num_wired_entries() - 1;
0155     write_c0_wired(wired);
0156     write_c0_index(wired);
0157     write_c0_entryhi(UNIQUE_ENTRYHI(wired));
0158     write_c0_entrylo0(0);
0159     write_c0_entrylo1(0);
0160     mtc0_tlbw_hazard();
0161     tlb_write_indexed();
0162     tlbw_use_hazard();
0163     write_c0_entryhi(old_ctx);
0164     local_irq_restore(flags);
0165     pagefault_enable();
0166     preempt_enable();
0167 }
0168 
0169 void copy_user_highpage(struct page *to, struct page *from,
0170     unsigned long vaddr, struct vm_area_struct *vma)
0171 {
0172     void *vfrom, *vto;
0173 
0174     vto = kmap_atomic(to);
0175     if (cpu_has_dc_aliases &&
0176         page_mapcount(from) && !Page_dcache_dirty(from)) {
0177         vfrom = kmap_coherent(from, vaddr);
0178         copy_page(vto, vfrom);
0179         kunmap_coherent();
0180     } else {
0181         vfrom = kmap_atomic(from);
0182         copy_page(vto, vfrom);
0183         kunmap_atomic(vfrom);
0184     }
0185     if ((!cpu_has_ic_fills_f_dc) ||
0186         pages_do_alias((unsigned long)vto, vaddr & PAGE_MASK))
0187         flush_data_cache_page((unsigned long)vto);
0188     kunmap_atomic(vto);
0189     /* Make sure this page is cleared on other CPU's too before using it */
0190     smp_wmb();
0191 }
0192 
0193 void copy_to_user_page(struct vm_area_struct *vma,
0194     struct page *page, unsigned long vaddr, void *dst, const void *src,
0195     unsigned long len)
0196 {
0197     if (cpu_has_dc_aliases &&
0198         page_mapcount(page) && !Page_dcache_dirty(page)) {
0199         void *vto = kmap_coherent(page, vaddr) + (vaddr & ~PAGE_MASK);
0200         memcpy(vto, src, len);
0201         kunmap_coherent();
0202     } else {
0203         memcpy(dst, src, len);
0204         if (cpu_has_dc_aliases)
0205             SetPageDcacheDirty(page);
0206     }
0207     if (vma->vm_flags & VM_EXEC)
0208         flush_cache_page(vma, vaddr, page_to_pfn(page));
0209 }
0210 
0211 void copy_from_user_page(struct vm_area_struct *vma,
0212     struct page *page, unsigned long vaddr, void *dst, const void *src,
0213     unsigned long len)
0214 {
0215     if (cpu_has_dc_aliases &&
0216         page_mapcount(page) && !Page_dcache_dirty(page)) {
0217         void *vfrom = kmap_coherent(page, vaddr) + (vaddr & ~PAGE_MASK);
0218         memcpy(dst, vfrom, len);
0219         kunmap_coherent();
0220     } else {
0221         memcpy(dst, src, len);
0222         if (cpu_has_dc_aliases)
0223             SetPageDcacheDirty(page);
0224     }
0225 }
0226 EXPORT_SYMBOL_GPL(copy_from_user_page);
0227 
0228 void __init fixrange_init(unsigned long start, unsigned long end,
0229     pgd_t *pgd_base)
0230 {
0231 #ifdef CONFIG_HIGHMEM
0232     pgd_t *pgd;
0233     pud_t *pud;
0234     pmd_t *pmd;
0235     pte_t *pte;
0236     int i, j, k;
0237     unsigned long vaddr;
0238 
0239     vaddr = start;
0240     i = pgd_index(vaddr);
0241     j = pud_index(vaddr);
0242     k = pmd_index(vaddr);
0243     pgd = pgd_base + i;
0244 
0245     for ( ; (i < PTRS_PER_PGD) && (vaddr < end); pgd++, i++) {
0246         pud = (pud_t *)pgd;
0247         for ( ; (j < PTRS_PER_PUD) && (vaddr < end); pud++, j++) {
0248             pmd = (pmd_t *)pud;
0249             for (; (k < PTRS_PER_PMD) && (vaddr < end); pmd++, k++) {
0250                 if (pmd_none(*pmd)) {
0251                     pte = (pte_t *) memblock_alloc_low(PAGE_SIZE,
0252                                        PAGE_SIZE);
0253                     if (!pte)
0254                         panic("%s: Failed to allocate %lu bytes align=%lx\n",
0255                               __func__, PAGE_SIZE,
0256                               PAGE_SIZE);
0257 
0258                     set_pmd(pmd, __pmd((unsigned long)pte));
0259                     BUG_ON(pte != pte_offset_kernel(pmd, 0));
0260                 }
0261                 vaddr += PMD_SIZE;
0262             }
0263             k = 0;
0264         }
0265         j = 0;
0266     }
0267 #endif
0268 }
0269 
0270 struct maar_walk_info {
0271     struct maar_config cfg[16];
0272     unsigned int num_cfg;
0273 };
0274 
0275 static int maar_res_walk(unsigned long start_pfn, unsigned long nr_pages,
0276              void *data)
0277 {
0278     struct maar_walk_info *wi = data;
0279     struct maar_config *cfg = &wi->cfg[wi->num_cfg];
0280     unsigned int maar_align;
0281 
0282     /* MAAR registers hold physical addresses right shifted by 4 bits */
0283     maar_align = BIT(MIPS_MAAR_ADDR_SHIFT + 4);
0284 
0285     /* Fill in the MAAR config entry */
0286     cfg->lower = ALIGN(PFN_PHYS(start_pfn), maar_align);
0287     cfg->upper = ALIGN_DOWN(PFN_PHYS(start_pfn + nr_pages), maar_align) - 1;
0288     cfg->attrs = MIPS_MAAR_S;
0289 
0290     /* Ensure we don't overflow the cfg array */
0291     if (!WARN_ON(wi->num_cfg >= ARRAY_SIZE(wi->cfg)))
0292         wi->num_cfg++;
0293 
0294     return 0;
0295 }
0296 
0297 
0298 unsigned __weak platform_maar_init(unsigned num_pairs)
0299 {
0300     unsigned int num_configured;
0301     struct maar_walk_info wi;
0302 
0303     wi.num_cfg = 0;
0304     walk_system_ram_range(0, max_pfn, &wi, maar_res_walk);
0305 
0306     num_configured = maar_config(wi.cfg, wi.num_cfg, num_pairs);
0307     if (num_configured < wi.num_cfg)
0308         pr_warn("Not enough MAAR pairs (%u) for all memory regions (%u)\n",
0309             num_pairs, wi.num_cfg);
0310 
0311     return num_configured;
0312 }
0313 
0314 void maar_init(void)
0315 {
0316     unsigned num_maars, used, i;
0317     phys_addr_t lower, upper, attr;
0318     static struct {
0319         struct maar_config cfgs[3];
0320         unsigned used;
0321     } recorded = { { { 0 } }, 0 };
0322 
0323     if (!cpu_has_maar)
0324         return;
0325 
0326     /* Detect the number of MAARs */
0327     write_c0_maari(~0);
0328     back_to_back_c0_hazard();
0329     num_maars = read_c0_maari() + 1;
0330 
0331     /* MAARs should be in pairs */
0332     WARN_ON(num_maars % 2);
0333 
0334     /* Set MAARs using values we recorded already */
0335     if (recorded.used) {
0336         used = maar_config(recorded.cfgs, recorded.used, num_maars / 2);
0337         BUG_ON(used != recorded.used);
0338     } else {
0339         /* Configure the required MAARs */
0340         used = platform_maar_init(num_maars / 2);
0341     }
0342 
0343     /* Disable any further MAARs */
0344     for (i = (used * 2); i < num_maars; i++) {
0345         write_c0_maari(i);
0346         back_to_back_c0_hazard();
0347         write_c0_maar(0);
0348         back_to_back_c0_hazard();
0349     }
0350 
0351     if (recorded.used)
0352         return;
0353 
0354     pr_info("MAAR configuration:\n");
0355     for (i = 0; i < num_maars; i += 2) {
0356         write_c0_maari(i);
0357         back_to_back_c0_hazard();
0358         upper = read_c0_maar();
0359 #ifdef CONFIG_XPA
0360         upper |= (phys_addr_t)readx_c0_maar() << MIPS_MAARX_ADDR_SHIFT;
0361 #endif
0362 
0363         write_c0_maari(i + 1);
0364         back_to_back_c0_hazard();
0365         lower = read_c0_maar();
0366 #ifdef CONFIG_XPA
0367         lower |= (phys_addr_t)readx_c0_maar() << MIPS_MAARX_ADDR_SHIFT;
0368 #endif
0369 
0370         attr = lower & upper;
0371         lower = (lower & MIPS_MAAR_ADDR) << 4;
0372         upper = ((upper & MIPS_MAAR_ADDR) << 4) | 0xffff;
0373 
0374         pr_info("  [%d]: ", i / 2);
0375         if ((attr & MIPS_MAAR_V) != MIPS_MAAR_V) {
0376             pr_cont("disabled\n");
0377             continue;
0378         }
0379 
0380         pr_cont("%pa-%pa", &lower, &upper);
0381 
0382         if (attr & MIPS_MAAR_S)
0383             pr_cont(" speculate");
0384 
0385         pr_cont("\n");
0386 
0387         /* Record the setup for use on secondary CPUs */
0388         if (used <= ARRAY_SIZE(recorded.cfgs)) {
0389             recorded.cfgs[recorded.used].lower = lower;
0390             recorded.cfgs[recorded.used].upper = upper;
0391             recorded.cfgs[recorded.used].attrs = attr;
0392             recorded.used++;
0393         }
0394     }
0395 }
0396 
0397 #ifndef CONFIG_NUMA
0398 void __init paging_init(void)
0399 {
0400     unsigned long max_zone_pfns[MAX_NR_ZONES];
0401 
0402     pagetable_init();
0403 
0404 #ifdef CONFIG_ZONE_DMA
0405     max_zone_pfns[ZONE_DMA] = MAX_DMA_PFN;
0406 #endif
0407 #ifdef CONFIG_ZONE_DMA32
0408     max_zone_pfns[ZONE_DMA32] = MAX_DMA32_PFN;
0409 #endif
0410     max_zone_pfns[ZONE_NORMAL] = max_low_pfn;
0411 #ifdef CONFIG_HIGHMEM
0412     max_zone_pfns[ZONE_HIGHMEM] = highend_pfn;
0413 
0414     if (cpu_has_dc_aliases && max_low_pfn != highend_pfn) {
0415         printk(KERN_WARNING "This processor doesn't support highmem."
0416                " %ldk highmem ignored\n",
0417                (highend_pfn - max_low_pfn) << (PAGE_SHIFT - 10));
0418         max_zone_pfns[ZONE_HIGHMEM] = max_low_pfn;
0419     }
0420 #endif
0421 
0422     free_area_init(max_zone_pfns);
0423 }
0424 
0425 #ifdef CONFIG_64BIT
0426 static struct kcore_list kcore_kseg0;
0427 #endif
0428 
0429 static inline void __init mem_init_free_highmem(void)
0430 {
0431 #ifdef CONFIG_HIGHMEM
0432     unsigned long tmp;
0433 
0434     if (cpu_has_dc_aliases)
0435         return;
0436 
0437     for (tmp = highstart_pfn; tmp < highend_pfn; tmp++) {
0438         struct page *page = pfn_to_page(tmp);
0439 
0440         if (!memblock_is_memory(PFN_PHYS(tmp)))
0441             SetPageReserved(page);
0442         else
0443             free_highmem_page(page);
0444     }
0445 #endif
0446 }
0447 
0448 void __init mem_init(void)
0449 {
0450     /*
0451      * When _PFN_SHIFT is greater than PAGE_SHIFT we won't have enough PTE
0452      * bits to hold a full 32b physical address on MIPS32 systems.
0453      */
0454     BUILD_BUG_ON(IS_ENABLED(CONFIG_32BIT) && (_PFN_SHIFT > PAGE_SHIFT));
0455 
0456 #ifdef CONFIG_HIGHMEM
0457     max_mapnr = highend_pfn ? highend_pfn : max_low_pfn;
0458 #else
0459     max_mapnr = max_low_pfn;
0460 #endif
0461     high_memory = (void *) __va(max_low_pfn << PAGE_SHIFT);
0462 
0463     maar_init();
0464     memblock_free_all();
0465     setup_zero_pages(); /* Setup zeroed pages.  */
0466     mem_init_free_highmem();
0467 
0468 #ifdef CONFIG_64BIT
0469     if ((unsigned long) &_text > (unsigned long) CKSEG0)
0470         /* The -4 is a hack so that user tools don't have to handle
0471            the overflow.  */
0472         kclist_add(&kcore_kseg0, (void *) CKSEG0,
0473                 0x80000000 - 4, KCORE_TEXT);
0474 #endif
0475 }
0476 #endif /* !CONFIG_NUMA */
0477 
0478 void free_init_pages(const char *what, unsigned long begin, unsigned long end)
0479 {
0480     unsigned long pfn;
0481 
0482     for (pfn = PFN_UP(begin); pfn < PFN_DOWN(end); pfn++) {
0483         struct page *page = pfn_to_page(pfn);
0484         void *addr = phys_to_virt(PFN_PHYS(pfn));
0485 
0486         memset(addr, POISON_FREE_INITMEM, PAGE_SIZE);
0487         free_reserved_page(page);
0488     }
0489     printk(KERN_INFO "Freeing %s: %ldk freed\n", what, (end - begin) >> 10);
0490 }
0491 
0492 void (*free_init_pages_eva)(void *begin, void *end) = NULL;
0493 
0494 void __weak __init prom_free_prom_memory(void)
0495 {
0496     /* nothing to do */
0497 }
0498 
0499 void __ref free_initmem(void)
0500 {
0501     prom_free_prom_memory();
0502     /*
0503      * Let the platform define a specific function to free the
0504      * init section since EVA may have used any possible mapping
0505      * between virtual and physical addresses.
0506      */
0507     if (free_init_pages_eva)
0508         free_init_pages_eva((void *)&__init_begin, (void *)&__init_end);
0509     else
0510         free_initmem_default(POISON_FREE_INITMEM);
0511 }
0512 
0513 #ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
0514 unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;
0515 EXPORT_SYMBOL(__per_cpu_offset);
0516 
0517 static int __init pcpu_cpu_distance(unsigned int from, unsigned int to)
0518 {
0519     return node_distance(cpu_to_node(from), cpu_to_node(to));
0520 }
0521 
0522 static int __init pcpu_cpu_to_node(int cpu)
0523 {
0524     return cpu_to_node(cpu);
0525 }
0526 
0527 void __init setup_per_cpu_areas(void)
0528 {
0529     unsigned long delta;
0530     unsigned int cpu;
0531     int rc;
0532 
0533     /*
0534      * Always reserve area for module percpu variables.  That's
0535      * what the legacy allocator did.
0536      */
0537     rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE,
0538                     PERCPU_DYNAMIC_RESERVE, PAGE_SIZE,
0539                     pcpu_cpu_distance,
0540                     pcpu_cpu_to_node);
0541     if (rc < 0)
0542         panic("Failed to initialize percpu areas.");
0543 
0544     delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start;
0545     for_each_possible_cpu(cpu)
0546         __per_cpu_offset[cpu] = delta + pcpu_unit_offsets[cpu];
0547 }
0548 #endif
0549 
0550 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT
0551 unsigned long pgd_current[NR_CPUS];
0552 #endif
0553 
0554 /*
0555  * Align swapper_pg_dir in to 64K, allows its address to be loaded
0556  * with a single LUI instruction in the TLB handlers.  If we used
0557  * __aligned(64K), its size would get rounded up to the alignment
0558  * size, and waste space.  So we place it in its own section and align
0559  * it in the linker script.
0560  */
0561 pgd_t swapper_pg_dir[PTRS_PER_PGD] __section(".bss..swapper_pg_dir");
0562 #ifndef __PAGETABLE_PUD_FOLDED
0563 pud_t invalid_pud_table[PTRS_PER_PUD] __page_aligned_bss;
0564 #endif
0565 #ifndef __PAGETABLE_PMD_FOLDED
0566 pmd_t invalid_pmd_table[PTRS_PER_PMD] __page_aligned_bss;
0567 EXPORT_SYMBOL_GPL(invalid_pmd_table);
0568 #endif
0569 pte_t invalid_pte_table[PTRS_PER_PTE] __page_aligned_bss;
0570 EXPORT_SYMBOL(invalid_pte_table);
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!