include/asm/kfence.h

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 /*
0003  * x86 KFENCE support.
0004  *
0005  * Copyright (C) 2020, Google LLC.
0006  */
0007
0008 #ifndef _ASM_X86_KFENCE_H
0009 #define _ASM_X86_KFENCE_H
0010
0011 #ifndef MODULE
0012
0013 #include <linux/bug.h>
0014 #include <linux/kfence.h>
0015
0016 #include <asm/pgalloc.h>
0017 #include <asm/pgtable.h>
0018 #include <asm/set_memory.h>
0019 #include <asm/tlbflush.h>
0020
0021 /* Force 4K pages for __kfence_pool. */
0022 static inline bool arch_kfence_init_pool(void)
0023 {
0024     unsigned long addr;
0025
0026     for (addr = (unsigned long)__kfence_pool; is_kfence_address((void *)addr);
0027          addr += PAGE_SIZE) {
0028         unsigned int level;
0029
0030         if (!lookup_address(addr, &level))
0031             return false;
0032
0033         if (level != PG_LEVEL_4K)
0034             set_memory_4k(addr, 1);
0035     }
0036
0037     return true;
0038 }
0039
0040 /* Protect the given page and flush TLB. */
0041 static inline bool kfence_protect_page(unsigned long addr, bool protect)
0042 {
0043     unsigned int level;
0044     pte_t *pte = lookup_address(addr, &level);
0045
0046     if (WARN_ON(!pte || level != PG_LEVEL_4K))
0047         return false;
0048
0049     /*
0050      * We need to avoid IPIs, as we may get KFENCE allocations or faults
0051      * with interrupts disabled. Therefore, the below is best-effort, and
0052      * does not flush TLBs on all CPUs. We can tolerate some inaccuracy;
0053      * lazy fault handling takes care of faults after the page is PRESENT.
0054      */
0055
0056     if (protect)
0057         set_pte(pte, __pte(pte_val(*pte) & ~_PAGE_PRESENT));
0058     else
0059         set_pte(pte, __pte(pte_val(*pte) | _PAGE_PRESENT));
0060
0061     /*
0062      * Flush this CPU's TLB, assuming whoever did the allocation/free is
0063      * likely to continue running on this CPU.
0064      */
0065     preempt_disable();
0066     flush_tlb_one_kernel(addr);
0067     preempt_enable();
0068     return true;
0069 }
0070
0071 #endif /* !MODULE */
0072
0073 #endif /* _ASM_X86_KFENCE_H */