powerpc/mm/pgtable_64.c

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  *  This file contains pgtable related functions for 64-bit machines.
0004  *
0005  *  Derived from arch/ppc64/mm/init.c
0006  *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
0007  *
0008  *  Modifications by Paul Mackerras (PowerMac) (paulus@samba.org)
0009  *  and Cort Dougan (PReP) (cort@cs.nmt.edu)
0010  *    Copyright (C) 1996 Paul Mackerras
0011  *
0012  *  Derived from "arch/i386/mm/init.c"
0013  *    Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
0014  *
0015  *  Dave Engebretsen <engebret@us.ibm.com>
0016  *      Rework for PPC64 port.
0017  */
0018
0019 #include <linux/signal.h>
0020 #include <linux/sched.h>
0021 #include <linux/kernel.h>
0022 #include <linux/errno.h>
0023 #include <linux/string.h>
0024 #include <linux/export.h>
0025 #include <linux/types.h>
0026 #include <linux/mman.h>
0027 #include <linux/mm.h>
0028 #include <linux/swap.h>
0029 #include <linux/stddef.h>
0030 #include <linux/vmalloc.h>
0031 #include <linux/slab.h>
0032 #include <linux/hugetlb.h>
0033
0034 #include <asm/page.h>
0035 #include <asm/mmu_context.h>
0036 #include <asm/mmu.h>
0037 #include <asm/smp.h>
0038 #include <asm/machdep.h>
0039 #include <asm/tlb.h>
0040 #include <asm/processor.h>
0041 #include <asm/cputable.h>
0042 #include <asm/sections.h>
0043 #include <asm/firmware.h>
0044 #include <asm/dma.h>
0045
0046 #include <mm/mmu_decl.h>
0047
0048
0049 #ifdef CONFIG_PPC_BOOK3S_64
0050 /*
0051  * partition table and process table for ISA 3.0
0052  */
0053 struct prtb_entry *process_tb;
0054 struct patb_entry *partition_tb;
0055 /*
0056  * page table size
0057  */
0058 unsigned long __pte_index_size;
0059 EXPORT_SYMBOL(__pte_index_size);
0060 unsigned long __pmd_index_size;
0061 EXPORT_SYMBOL(__pmd_index_size);
0062 unsigned long __pud_index_size;
0063 EXPORT_SYMBOL(__pud_index_size);
0064 unsigned long __pgd_index_size;
0065 EXPORT_SYMBOL(__pgd_index_size);
0066 unsigned long __pud_cache_index;
0067 EXPORT_SYMBOL(__pud_cache_index);
0068 unsigned long __pte_table_size;
0069 EXPORT_SYMBOL(__pte_table_size);
0070 unsigned long __pmd_table_size;
0071 EXPORT_SYMBOL(__pmd_table_size);
0072 unsigned long __pud_table_size;
0073 EXPORT_SYMBOL(__pud_table_size);
0074 unsigned long __pgd_table_size;
0075 EXPORT_SYMBOL(__pgd_table_size);
0076 unsigned long __pmd_val_bits;
0077 EXPORT_SYMBOL(__pmd_val_bits);
0078 unsigned long __pud_val_bits;
0079 EXPORT_SYMBOL(__pud_val_bits);
0080 unsigned long __pgd_val_bits;
0081 EXPORT_SYMBOL(__pgd_val_bits);
0082 unsigned long __kernel_virt_start;
0083 EXPORT_SYMBOL(__kernel_virt_start);
0084 unsigned long __vmalloc_start;
0085 EXPORT_SYMBOL(__vmalloc_start);
0086 unsigned long __vmalloc_end;
0087 EXPORT_SYMBOL(__vmalloc_end);
0088 unsigned long __kernel_io_start;
0089 EXPORT_SYMBOL(__kernel_io_start);
0090 unsigned long __kernel_io_end;
0091 struct page *vmemmap;
0092 EXPORT_SYMBOL(vmemmap);
0093 unsigned long __pte_frag_nr;
0094 EXPORT_SYMBOL(__pte_frag_nr);
0095 unsigned long __pte_frag_size_shift;
0096 EXPORT_SYMBOL(__pte_frag_size_shift);
0097 #endif
0098
0099 #ifndef __PAGETABLE_PUD_FOLDED
0100 /* 4 level page table */
0101 struct page *p4d_page(p4d_t p4d)
0102 {
0103     if (p4d_is_leaf(p4d)) {
0104         if (!IS_ENABLED(CONFIG_HAVE_ARCH_HUGE_VMAP))
0105             VM_WARN_ON(!p4d_huge(p4d));
0106         return pte_page(p4d_pte(p4d));
0107     }
0108     return virt_to_page(p4d_pgtable(p4d));
0109 }
0110 #endif
0111
0112 struct page *pud_page(pud_t pud)
0113 {
0114     if (pud_is_leaf(pud)) {
0115         if (!IS_ENABLED(CONFIG_HAVE_ARCH_HUGE_VMAP))
0116             VM_WARN_ON(!pud_huge(pud));
0117         return pte_page(pud_pte(pud));
0118     }
0119     return virt_to_page(pud_pgtable(pud));
0120 }
0121
0122 /*
0123  * For hugepage we have pfn in the pmd, we use PTE_RPN_SHIFT bits for flags
0124  * For PTE page, we have a PTE_FRAG_SIZE (4K) aligned virtual address.
0125  */
0126 struct page *pmd_page(pmd_t pmd)
0127 {
0128     if (pmd_is_leaf(pmd)) {
0129         /*
0130          * vmalloc_to_page may be called on any vmap address (not only
0131          * vmalloc), and it uses pmd_page() etc., when huge vmap is
0132          * enabled so these checks can't be used.
0133          */
0134         if (!IS_ENABLED(CONFIG_HAVE_ARCH_HUGE_VMAP))
0135             VM_WARN_ON(!(pmd_large(pmd) || pmd_huge(pmd)));
0136         return pte_page(pmd_pte(pmd));
0137     }
0138     return virt_to_page(pmd_page_vaddr(pmd));
0139 }
0140
0141 #ifdef CONFIG_STRICT_KERNEL_RWX
0142 void mark_rodata_ro(void)
0143 {
0144     if (!mmu_has_feature(MMU_FTR_KERNEL_RO)) {
0145         pr_warn("Warning: Unable to mark rodata read only on this CPU.\n");
0146         return;
0147     }
0148
0149     if (radix_enabled())
0150         radix__mark_rodata_ro();
0151     else
0152         hash__mark_rodata_ro();
0153
0154     // mark_initmem_nx() should have already run by now
0155     ptdump_check_wx();
0156 }
0157
0158 void mark_initmem_nx(void)
0159 {
0160     if (radix_enabled())
0161         radix__mark_initmem_nx();
0162     else
0163         hash__mark_initmem_nx();
0164 }
0165 #endif