mm/book3s64/pkeys.c

0001 // SPDX-License-Identifier: GPL-2.0+
0002 /*
0003  * PowerPC Memory Protection Keys management
0004  *
0005  * Copyright 2017, Ram Pai, IBM Corporation.
0006  */
0007
0008 #include <asm/mman.h>
0009 #include <asm/mmu_context.h>
0010 #include <asm/mmu.h>
0011 #include <asm/setup.h>
0012 #include <asm/smp.h>
0013 #include <asm/firmware.h>
0014
0015 #include <linux/pkeys.h>
0016 #include <linux/of_fdt.h>
0017
0018
0019 int  num_pkey;      /* Max number of pkeys supported */
0020 /*
0021  *  Keys marked in the reservation list cannot be allocated by  userspace
0022  */
0023 u32 reserved_allocation_mask __ro_after_init;
0024
0025 /* Bits set for the initially allocated keys */
0026 static u32 initial_allocation_mask __ro_after_init;
0027
0028 /*
0029  * Even if we allocate keys with sys_pkey_alloc(), we need to make sure
0030  * other thread still find the access denied using the same keys.
0031  */
0032 u64 default_amr __ro_after_init  = ~0x0UL;
0033 u64 default_iamr __ro_after_init = 0x5555555555555555UL;
0034 u64 default_uamor __ro_after_init;
0035 EXPORT_SYMBOL(default_amr);
0036 /*
0037  * Key used to implement PROT_EXEC mmap. Denies READ/WRITE
0038  * We pick key 2 because 0 is special key and 1 is reserved as per ISA.
0039  */
0040 static int execute_only_key = 2;
0041 static bool pkey_execute_disable_supported;
0042
0043
0044 #define AMR_BITS_PER_PKEY 2
0045 #define AMR_RD_BIT 0x1UL
0046 #define AMR_WR_BIT 0x2UL
0047 #define IAMR_EX_BIT 0x1UL
0048 #define PKEY_REG_BITS (sizeof(u64) * 8)
0049 #define pkeyshift(pkey) (PKEY_REG_BITS - ((pkey+1) * AMR_BITS_PER_PKEY))
0050
0051 static int __init dt_scan_storage_keys(unsigned long node,
0052                        const char *uname, int depth,
0053                        void *data)
0054 {
0055     const char *type = of_get_flat_dt_prop(node, "device_type", NULL);
0056     const __be32 *prop;
0057     int *pkeys_total = (int *) data;
0058
0059     /* We are scanning "cpu" nodes only */
0060     if (type == NULL || strcmp(type, "cpu") != 0)
0061         return 0;
0062
0063     prop = of_get_flat_dt_prop(node, "ibm,processor-storage-keys", NULL);
0064     if (!prop)
0065         return 0;
0066     *pkeys_total = be32_to_cpu(prop[0]);
0067     return 1;
0068 }
0069
0070 static int __init scan_pkey_feature(void)
0071 {
0072     int ret;
0073     int pkeys_total = 0;
0074
0075     /*
0076      * Pkey is not supported with Radix translation.
0077      */
0078     if (early_radix_enabled())
0079         return 0;
0080
0081     ret = of_scan_flat_dt(dt_scan_storage_keys, &pkeys_total);
0082     if (ret == 0) {
0083         /*
0084          * Let's assume 32 pkeys on P8/P9 bare metal, if its not defined by device
0085          * tree. We make this exception since some version of skiboot forgot to
0086          * expose this property on power8/9.
0087          */
0088         if (!firmware_has_feature(FW_FEATURE_LPAR)) {
0089             unsigned long pvr = mfspr(SPRN_PVR);
0090
0091             if (PVR_VER(pvr) == PVR_POWER8 || PVR_VER(pvr) == PVR_POWER8E ||
0092                 PVR_VER(pvr) == PVR_POWER8NVL || PVR_VER(pvr) == PVR_POWER9)
0093                 pkeys_total = 32;
0094         }
0095     }
0096
0097 #ifdef CONFIG_PPC_MEM_KEYS
0098     /*
0099      * Adjust the upper limit, based on the number of bits supported by
0100      * arch-neutral code.
0101      */
0102     pkeys_total = min_t(int, pkeys_total,
0103                 ((ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT) + 1));
0104 #endif
0105     return pkeys_total;
0106 }
0107
0108 void __init pkey_early_init_devtree(void)
0109 {
0110     int pkeys_total, i;
0111
0112 #ifdef CONFIG_PPC_MEM_KEYS
0113     /*
0114      * We define PKEY_DISABLE_EXECUTE in addition to the arch-neutral
0115      * generic defines for PKEY_DISABLE_ACCESS and PKEY_DISABLE_WRITE.
0116      * Ensure that the bits a distinct.
0117      */
0118     BUILD_BUG_ON(PKEY_DISABLE_EXECUTE &
0119              (PKEY_DISABLE_ACCESS | PKEY_DISABLE_WRITE));
0120
0121     /*
0122      * pkey_to_vmflag_bits() assumes that the pkey bits are contiguous
0123      * in the vmaflag. Make sure that is really the case.
0124      */
0125     BUILD_BUG_ON(__builtin_clzl(ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT) +
0126              __builtin_popcountl(ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT)
0127                 != (sizeof(u64) * BITS_PER_BYTE));
0128 #endif
0129     /*
0130      * Only P7 and above supports SPRN_AMR update with MSR[PR] = 1
0131      */
0132     if (!early_cpu_has_feature(CPU_FTR_ARCH_206))
0133         return;
0134
0135     /* scan the device tree for pkey feature */
0136     pkeys_total = scan_pkey_feature();
0137     if (!pkeys_total)
0138         goto out;
0139
0140     /* Allow all keys to be modified by default */
0141     default_uamor = ~0x0UL;
0142
0143     cur_cpu_spec->mmu_features |= MMU_FTR_PKEY;
0144
0145     /*
0146      * The device tree cannot be relied to indicate support for
0147      * execute_disable support. Instead we use a PVR check.
0148      */
0149     if (pvr_version_is(PVR_POWER7) || pvr_version_is(PVR_POWER7p))
0150         pkey_execute_disable_supported = false;
0151     else
0152         pkey_execute_disable_supported = true;
0153
0154 #ifdef CONFIG_PPC_4K_PAGES
0155     /*
0156      * The OS can manage only 8 pkeys due to its inability to represent them
0157      * in the Linux 4K PTE. Mark all other keys reserved.
0158      */
0159     num_pkey = min(8, pkeys_total);
0160 #else
0161     num_pkey = pkeys_total;
0162 #endif
0163
0164     if (unlikely(num_pkey <= execute_only_key) || !pkey_execute_disable_supported) {
0165         /*
0166          * Insufficient number of keys to support
0167          * execute only key. Mark it unavailable.
0168          */
0169         execute_only_key = -1;
0170     } else {
0171         /*
0172          * Mark the execute_only_pkey as not available for
0173          * user allocation via pkey_alloc.
0174          */
0175         reserved_allocation_mask |= (0x1 << execute_only_key);
0176
0177         /*
0178          * Deny READ/WRITE for execute_only_key.
0179          * Allow execute in IAMR.
0180          */
0181         default_amr  |= (0x3ul << pkeyshift(execute_only_key));
0182         default_iamr &= ~(0x1ul << pkeyshift(execute_only_key));
0183
0184         /*
0185          * Clear the uamor bits for this key.
0186          */
0187         default_uamor &= ~(0x3ul << pkeyshift(execute_only_key));
0188     }
0189
0190     if (unlikely(num_pkey <= 3)) {
0191         /*
0192          * Insufficient number of keys to support
0193          * KUAP/KUEP feature.
0194          */
0195         disable_kuep = true;
0196         disable_kuap = true;
0197         WARN(1, "Disabling kernel user protection due to low (%d) max supported keys\n", num_pkey);
0198     } else {
0199         /*  handle key which is used by kernel for KAUP */
0200         reserved_allocation_mask |= (0x1 << 3);
0201         /*
0202          * Mark access for kup_key in default amr so that
0203          * we continue to operate with that AMR in
0204          * copy_to/from_user().
0205          */
0206         default_amr   &= ~(0x3ul << pkeyshift(3));
0207         default_iamr  &= ~(0x1ul << pkeyshift(3));
0208         default_uamor &= ~(0x3ul << pkeyshift(3));
0209     }
0210
0211     /*
0212      * Allow access for only key 0. And prevent any other modification.
0213      */
0214     default_amr   &= ~(0x3ul << pkeyshift(0));
0215     default_iamr  &= ~(0x1ul << pkeyshift(0));
0216     default_uamor &= ~(0x3ul << pkeyshift(0));
0217     /*
0218      * key 0 is special in that we want to consider it an allocated
0219      * key which is preallocated. We don't allow changing AMR bits
0220      * w.r.t key 0. But one can pkey_free(key0)
0221      */
0222     initial_allocation_mask |= (0x1 << 0);
0223
0224     /*
0225      * key 1 is recommended not to be used. PowerISA(3.0) page 1015,
0226      * programming note.
0227      */
0228     reserved_allocation_mask |= (0x1 << 1);
0229     default_uamor &= ~(0x3ul << pkeyshift(1));
0230
0231     /*
0232      * Prevent the usage of OS reserved keys. Update UAMOR
0233      * for those keys. Also mark the rest of the bits in the
0234      * 32 bit mask as reserved.
0235      */
0236     for (i = num_pkey; i < 32 ; i++) {
0237         reserved_allocation_mask |= (0x1 << i);
0238         default_uamor &= ~(0x3ul << pkeyshift(i));
0239     }
0240     /*
0241      * Prevent the allocation of reserved keys too.
0242      */
0243     initial_allocation_mask |= reserved_allocation_mask;
0244
0245     pr_info("Enabling pkeys with max key count %d\n", num_pkey);
0246 out:
0247     /*
0248      * Setup uamor on boot cpu
0249      */
0250     mtspr(SPRN_UAMOR, default_uamor);
0251
0252     return;
0253 }
0254
0255 #ifdef CONFIG_PPC_KUEP
0256 void setup_kuep(bool disabled)
0257 {
0258     if (disabled)
0259         return;
0260     /*
0261      * On hash if PKEY feature is not enabled, disable KUAP too.
0262      */
0263     if (!early_radix_enabled() && !early_mmu_has_feature(MMU_FTR_PKEY))
0264         return;
0265
0266     if (smp_processor_id() == boot_cpuid) {
0267         pr_info("Activating Kernel Userspace Execution Prevention\n");
0268         cur_cpu_spec->mmu_features |= MMU_FTR_BOOK3S_KUEP;
0269     }
0270
0271     /*
0272      * Radix always uses key0 of the IAMR to determine if an access is
0273      * allowed. We set bit 0 (IBM bit 1) of key0, to prevent instruction
0274      * fetch.
0275      */
0276     mtspr(SPRN_IAMR, AMR_KUEP_BLOCKED);
0277     isync();
0278 }
0279 #endif
0280
0281 #ifdef CONFIG_PPC_KUAP
0282 void setup_kuap(bool disabled)
0283 {
0284     if (disabled)
0285         return;
0286     /*
0287      * On hash if PKEY feature is not enabled, disable KUAP too.
0288      */
0289     if (!early_radix_enabled() && !early_mmu_has_feature(MMU_FTR_PKEY))
0290         return;
0291
0292     if (smp_processor_id() == boot_cpuid) {
0293         pr_info("Activating Kernel Userspace Access Prevention\n");
0294         cur_cpu_spec->mmu_features |= MMU_FTR_BOOK3S_KUAP;
0295     }
0296
0297     /*
0298      * Set the default kernel AMR values on all cpus.
0299      */
0300     mtspr(SPRN_AMR, AMR_KUAP_BLOCKED);
0301     isync();
0302 }
0303 #endif
0304
0305 #ifdef CONFIG_PPC_MEM_KEYS
0306 void pkey_mm_init(struct mm_struct *mm)
0307 {
0308     if (!mmu_has_feature(MMU_FTR_PKEY))
0309         return;
0310     mm_pkey_allocation_map(mm) = initial_allocation_mask;
0311     mm->context.execute_only_pkey = execute_only_key;
0312 }
0313
0314 static inline void init_amr(int pkey, u8 init_bits)
0315 {
0316     u64 new_amr_bits = (((u64)init_bits & 0x3UL) << pkeyshift(pkey));
0317     u64 old_amr = current_thread_amr() & ~((u64)(0x3ul) << pkeyshift(pkey));
0318
0319     current->thread.regs->amr = old_amr | new_amr_bits;
0320 }
0321
0322 static inline void init_iamr(int pkey, u8 init_bits)
0323 {
0324     u64 new_iamr_bits = (((u64)init_bits & 0x1UL) << pkeyshift(pkey));
0325     u64 old_iamr = current_thread_iamr() & ~((u64)(0x1ul) << pkeyshift(pkey));
0326
0327     if (!likely(pkey_execute_disable_supported))
0328         return;
0329
0330     current->thread.regs->iamr = old_iamr | new_iamr_bits;
0331 }
0332
0333 /*
0334  * Set the access rights in AMR IAMR and UAMOR registers for @pkey to that
0335  * specified in @init_val.
0336  */
0337 int __arch_set_user_pkey_access(struct task_struct *tsk, int pkey,
0338                 unsigned long init_val)
0339 {
0340     u64 new_amr_bits = 0x0ul;
0341     u64 new_iamr_bits = 0x0ul;
0342     u64 pkey_bits, uamor_pkey_bits;
0343
0344     /*
0345      * Check whether the key is disabled by UAMOR.
0346      */
0347     pkey_bits = 0x3ul << pkeyshift(pkey);
0348     uamor_pkey_bits = (default_uamor & pkey_bits);
0349
0350     /*
0351      * Both the bits in UAMOR corresponding to the key should be set
0352      */
0353     if (uamor_pkey_bits != pkey_bits)
0354         return -EINVAL;
0355
0356     if (init_val & PKEY_DISABLE_EXECUTE) {
0357         if (!pkey_execute_disable_supported)
0358             return -EINVAL;
0359         new_iamr_bits |= IAMR_EX_BIT;
0360     }
0361     init_iamr(pkey, new_iamr_bits);
0362
0363     /* Set the bits we need in AMR: */
0364     if (init_val & PKEY_DISABLE_ACCESS)
0365         new_amr_bits |= AMR_RD_BIT | AMR_WR_BIT;
0366     else if (init_val & PKEY_DISABLE_WRITE)
0367         new_amr_bits |= AMR_WR_BIT;
0368
0369     init_amr(pkey, new_amr_bits);
0370     return 0;
0371 }
0372
0373 int execute_only_pkey(struct mm_struct *mm)
0374 {
0375     return mm->context.execute_only_pkey;
0376 }
0377
0378 static inline bool vma_is_pkey_exec_only(struct vm_area_struct *vma)
0379 {
0380     /* Do this check first since the vm_flags should be hot */
0381     if ((vma->vm_flags & VM_ACCESS_FLAGS) != VM_EXEC)
0382         return false;
0383
0384     return (vma_pkey(vma) == vma->vm_mm->context.execute_only_pkey);
0385 }
0386
0387 /*
0388  * This should only be called for *plain* mprotect calls.
0389  */
0390 int __arch_override_mprotect_pkey(struct vm_area_struct *vma, int prot,
0391                   int pkey)
0392 {
0393     /*
0394      * If the currently associated pkey is execute-only, but the requested
0395      * protection is not execute-only, move it back to the default pkey.
0396      */
0397     if (vma_is_pkey_exec_only(vma) && (prot != PROT_EXEC))
0398         return 0;
0399
0400     /*
0401      * The requested protection is execute-only. Hence let's use an
0402      * execute-only pkey.
0403      */
0404     if (prot == PROT_EXEC) {
0405         pkey = execute_only_pkey(vma->vm_mm);
0406         if (pkey > 0)
0407             return pkey;
0408     }
0409
0410     /* Nothing to override. */
0411     return vma_pkey(vma);
0412 }
0413
0414 static bool pkey_access_permitted(int pkey, bool write, bool execute)
0415 {
0416     int pkey_shift;
0417     u64 amr;
0418
0419     pkey_shift = pkeyshift(pkey);
0420     if (execute)
0421         return !(current_thread_iamr() & (IAMR_EX_BIT << pkey_shift));
0422
0423     amr = current_thread_amr();
0424     if (write)
0425         return !(amr & (AMR_WR_BIT << pkey_shift));
0426
0427     return !(amr & (AMR_RD_BIT << pkey_shift));
0428 }
0429
0430 bool arch_pte_access_permitted(u64 pte, bool write, bool execute)
0431 {
0432     if (!mmu_has_feature(MMU_FTR_PKEY))
0433         return true;
0434
0435     return pkey_access_permitted(pte_to_pkey_bits(pte), write, execute);
0436 }
0437
0438 /*
0439  * We only want to enforce protection keys on the current thread because we
0440  * effectively have no access to AMR/IAMR for other threads or any way to tell
0441  * which AMR/IAMR in a threaded process we could use.
0442  *
0443  * So do not enforce things if the VMA is not from the current mm, or if we are
0444  * in a kernel thread.
0445  */
0446 bool arch_vma_access_permitted(struct vm_area_struct *vma, bool write,
0447                    bool execute, bool foreign)
0448 {
0449     if (!mmu_has_feature(MMU_FTR_PKEY))
0450         return true;
0451     /*
0452      * Do not enforce our key-permissions on a foreign vma.
0453      */
0454     if (foreign || vma_is_foreign(vma))
0455         return true;
0456
0457     return pkey_access_permitted(vma_pkey(vma), write, execute);
0458 }
0459
0460 void arch_dup_pkeys(struct mm_struct *oldmm, struct mm_struct *mm)
0461 {
0462     if (!mmu_has_feature(MMU_FTR_PKEY))
0463         return;
0464
0465     /* Duplicate the oldmm pkey state in mm: */
0466     mm_pkey_allocation_map(mm) = mm_pkey_allocation_map(oldmm);
0467     mm->context.execute_only_pkey = oldmm->context.execute_only_pkey;
0468 }
0469
0470 #endif /* CONFIG_PPC_MEM_KEYS */