the-tree/kernel/scs.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * Shadow Call Stack support.
0004  *
0005  * Copyright (C) 2019 Google LLC
0006  */
0007
0008 #include <linux/cpuhotplug.h>
0009 #include <linux/kasan.h>
0010 #include <linux/mm.h>
0011 #include <linux/scs.h>
0012 #include <linux/vmalloc.h>
0013 #include <linux/vmstat.h>
0014
0015 static void __scs_account(void *s, int account)
0016 {
0017     struct page *scs_page = vmalloc_to_page(s);
0018
0019     mod_node_page_state(page_pgdat(scs_page), NR_KERNEL_SCS_KB,
0020                 account * (SCS_SIZE / SZ_1K));
0021 }
0022
0023 /* Matches NR_CACHED_STACKS for VMAP_STACK */
0024 #define NR_CACHED_SCS 2
0025 static DEFINE_PER_CPU(void *, scs_cache[NR_CACHED_SCS]);
0026
0027 static void *__scs_alloc(int node)
0028 {
0029     int i;
0030     void *s;
0031
0032     for (i = 0; i < NR_CACHED_SCS; i++) {
0033         s = this_cpu_xchg(scs_cache[i], NULL);
0034         if (s) {
0035             s = kasan_unpoison_vmalloc(s, SCS_SIZE,
0036                            KASAN_VMALLOC_PROT_NORMAL);
0037             memset(s, 0, SCS_SIZE);
0038             goto out;
0039         }
0040     }
0041
0042     s = __vmalloc_node_range(SCS_SIZE, 1, VMALLOC_START, VMALLOC_END,
0043                     GFP_SCS, PAGE_KERNEL, 0, node,
0044                     __builtin_return_address(0));
0045
0046 out:
0047     return kasan_reset_tag(s);
0048 }
0049
0050 void *scs_alloc(int node)
0051 {
0052     void *s;
0053
0054     s = __scs_alloc(node);
0055     if (!s)
0056         return NULL;
0057
0058     *__scs_magic(s) = SCS_END_MAGIC;
0059
0060     /*
0061      * Poison the allocation to catch unintentional accesses to
0062      * the shadow stack when KASAN is enabled.
0063      */
0064     kasan_poison_vmalloc(s, SCS_SIZE);
0065     __scs_account(s, 1);
0066     return s;
0067 }
0068
0069 void scs_free(void *s)
0070 {
0071     int i;
0072
0073     __scs_account(s, -1);
0074
0075     /*
0076      * We cannot sleep as this can be called in interrupt context,
0077      * so use this_cpu_cmpxchg to update the cache, and vfree_atomic
0078      * to free the stack.
0079      */
0080
0081     for (i = 0; i < NR_CACHED_SCS; i++)
0082         if (this_cpu_cmpxchg(scs_cache[i], 0, s) == NULL)
0083             return;
0084
0085     kasan_unpoison_vmalloc(s, SCS_SIZE, KASAN_VMALLOC_PROT_NORMAL);
0086     vfree_atomic(s);
0087 }
0088
0089 static int scs_cleanup(unsigned int cpu)
0090 {
0091     int i;
0092     void **cache = per_cpu_ptr(scs_cache, cpu);
0093
0094     for (i = 0; i < NR_CACHED_SCS; i++) {
0095         vfree(cache[i]);
0096         cache[i] = NULL;
0097     }
0098
0099     return 0;
0100 }
0101
0102 void __init scs_init(void)
0103 {
0104     cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, "scs:scs_cache", NULL,
0105               scs_cleanup);
0106 }
0107
0108 int scs_prepare(struct task_struct *tsk, int node)
0109 {
0110     void *s = scs_alloc(node);
0111
0112     if (!s)
0113         return -ENOMEM;
0114
0115     task_scs(tsk) = task_scs_sp(tsk) = s;
0116     return 0;
0117 }
0118
0119 static void scs_check_usage(struct task_struct *tsk)
0120 {
0121     static unsigned long highest;
0122
0123     unsigned long *p, prev, curr = highest, used = 0;
0124
0125     if (!IS_ENABLED(CONFIG_DEBUG_STACK_USAGE))
0126         return;
0127
0128     for (p = task_scs(tsk); p < __scs_magic(tsk); ++p) {
0129         if (!READ_ONCE_NOCHECK(*p))
0130             break;
0131         used += sizeof(*p);
0132     }
0133
0134     while (used > curr) {
0135         prev = cmpxchg_relaxed(&highest, curr, used);
0136
0137         if (prev == curr) {
0138             pr_info("%s (%d): highest shadow stack usage: %lu bytes\n",
0139                 tsk->comm, task_pid_nr(tsk), used);
0140             break;
0141         }
0142
0143         curr = prev;
0144     }
0145 }
0146
0147 void scs_release(struct task_struct *tsk)
0148 {
0149     void *s = task_scs(tsk);
0150
0151     if (!s)
0152         return;
0153
0154     WARN(task_scs_end_corrupted(tsk),
0155          "corrupted shadow stack detected when freeing task\n");
0156     scs_check_usage(tsk);
0157     scs_free(s);
0158 }