arm/kernel/vdso.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Adapted from arm64 version.
0004  *
0005  * Copyright (C) 2012 ARM Limited
0006  * Copyright (C) 2015 Mentor Graphics Corporation.
0007  */
0008
0009 #include <linux/cache.h>
0010 #include <linux/elf.h>
0011 #include <linux/err.h>
0012 #include <linux/kernel.h>
0013 #include <linux/mm.h>
0014 #include <linux/of.h>
0015 #include <linux/printk.h>
0016 #include <linux/slab.h>
0017 #include <linux/timekeeper_internal.h>
0018 #include <linux/vmalloc.h>
0019 #include <asm/arch_timer.h>
0020 #include <asm/barrier.h>
0021 #include <asm/cacheflush.h>
0022 #include <asm/page.h>
0023 #include <asm/vdso.h>
0024 #include <asm/vdso_datapage.h>
0025 #include <clocksource/arm_arch_timer.h>
0026 #include <vdso/helpers.h>
0027 #include <vdso/vsyscall.h>
0028
0029 #define MAX_SYMNAME 64
0030
0031 static struct page **vdso_text_pagelist;
0032
0033 extern char vdso_start[], vdso_end[];
0034
0035 /* Total number of pages needed for the data and text portions of the VDSO. */
0036 unsigned int vdso_total_pages __ro_after_init;
0037
0038 /*
0039  * The VDSO data page.
0040  */
0041 static union vdso_data_store vdso_data_store __page_aligned_data;
0042 struct vdso_data *vdso_data = vdso_data_store.data;
0043
0044 static struct page *vdso_data_page __ro_after_init;
0045 static const struct vm_special_mapping vdso_data_mapping = {
0046     .name = "[vvar]",
0047     .pages = &vdso_data_page,
0048 };
0049
0050 static int vdso_mremap(const struct vm_special_mapping *sm,
0051         struct vm_area_struct *new_vma)
0052 {
0053     current->mm->context.vdso = new_vma->vm_start;
0054
0055     return 0;
0056 }
0057
0058 static struct vm_special_mapping vdso_text_mapping __ro_after_init = {
0059     .name = "[vdso]",
0060     .mremap = vdso_mremap,
0061 };
0062
0063 struct elfinfo {
0064     Elf32_Ehdr  *hdr;       /* ptr to ELF */
0065     Elf32_Sym   *dynsym;    /* ptr to .dynsym section */
0066     unsigned long   dynsymsize; /* size of .dynsym section */
0067     char        *dynstr;    /* ptr to .dynstr section */
0068 };
0069
0070 /* Cached result of boot-time check for whether the arch timer exists,
0071  * and if so, whether the virtual counter is useable.
0072  */
0073 bool cntvct_ok __ro_after_init;
0074
0075 static bool __init cntvct_functional(void)
0076 {
0077     struct device_node *np;
0078     bool ret = false;
0079
0080     if (!IS_ENABLED(CONFIG_ARM_ARCH_TIMER))
0081         goto out;
0082
0083     /* The arm_arch_timer core should export
0084      * arch_timer_use_virtual or similar so we don't have to do
0085      * this.
0086      */
0087     np = of_find_compatible_node(NULL, NULL, "arm,armv7-timer");
0088     if (!np)
0089         np = of_find_compatible_node(NULL, NULL, "arm,armv8-timer");
0090     if (!np)
0091         goto out_put;
0092
0093     if (of_property_read_bool(np, "arm,cpu-registers-not-fw-configured"))
0094         goto out_put;
0095
0096     ret = true;
0097
0098 out_put:
0099     of_node_put(np);
0100 out:
0101     return ret;
0102 }
0103
0104 static void * __init find_section(Elf32_Ehdr *ehdr, const char *name,
0105                   unsigned long *size)
0106 {
0107     Elf32_Shdr *sechdrs;
0108     unsigned int i;
0109     char *secnames;
0110
0111     /* Grab section headers and strings so we can tell who is who */
0112     sechdrs = (void *)ehdr + ehdr->e_shoff;
0113     secnames = (void *)ehdr + sechdrs[ehdr->e_shstrndx].sh_offset;
0114
0115     /* Find the section they want */
0116     for (i = 1; i < ehdr->e_shnum; i++) {
0117         if (strcmp(secnames + sechdrs[i].sh_name, name) == 0) {
0118             if (size)
0119                 *size = sechdrs[i].sh_size;
0120             return (void *)ehdr + sechdrs[i].sh_offset;
0121         }
0122     }
0123
0124     if (size)
0125         *size = 0;
0126     return NULL;
0127 }
0128
0129 static Elf32_Sym * __init find_symbol(struct elfinfo *lib, const char *symname)
0130 {
0131     unsigned int i;
0132
0133     for (i = 0; i < (lib->dynsymsize / sizeof(Elf32_Sym)); i++) {
0134         char name[MAX_SYMNAME], *c;
0135
0136         if (lib->dynsym[i].st_name == 0)
0137             continue;
0138         strlcpy(name, lib->dynstr + lib->dynsym[i].st_name,
0139             MAX_SYMNAME);
0140         c = strchr(name, '@');
0141         if (c)
0142             *c = 0;
0143         if (strcmp(symname, name) == 0)
0144             return &lib->dynsym[i];
0145     }
0146     return NULL;
0147 }
0148
0149 static void __init vdso_nullpatch_one(struct elfinfo *lib, const char *symname)
0150 {
0151     Elf32_Sym *sym;
0152
0153     sym = find_symbol(lib, symname);
0154     if (!sym)
0155         return;
0156
0157     sym->st_name = 0;
0158 }
0159
0160 static void __init patch_vdso(void *ehdr)
0161 {
0162     struct elfinfo einfo;
0163
0164     einfo = (struct elfinfo) {
0165         .hdr = ehdr,
0166     };
0167
0168     einfo.dynsym = find_section(einfo.hdr, ".dynsym", &einfo.dynsymsize);
0169     einfo.dynstr = find_section(einfo.hdr, ".dynstr", NULL);
0170
0171     /* If the virtual counter is absent or non-functional we don't
0172      * want programs to incur the slight additional overhead of
0173      * dispatching through the VDSO only to fall back to syscalls.
0174      */
0175     if (!cntvct_ok) {
0176         vdso_nullpatch_one(&einfo, "__vdso_gettimeofday");
0177         vdso_nullpatch_one(&einfo, "__vdso_clock_gettime");
0178         vdso_nullpatch_one(&einfo, "__vdso_clock_gettime64");
0179     }
0180 }
0181
0182 static int __init vdso_init(void)
0183 {
0184     unsigned int text_pages;
0185     int i;
0186
0187     if (memcmp(vdso_start, "\177ELF", 4)) {
0188         pr_err("VDSO is not a valid ELF object!\n");
0189         return -ENOEXEC;
0190     }
0191
0192     text_pages = (vdso_end - vdso_start) >> PAGE_SHIFT;
0193
0194     /* Allocate the VDSO text pagelist */
0195     vdso_text_pagelist = kcalloc(text_pages, sizeof(struct page *),
0196                      GFP_KERNEL);
0197     if (vdso_text_pagelist == NULL)
0198         return -ENOMEM;
0199
0200     /* Grab the VDSO data page. */
0201     vdso_data_page = virt_to_page(vdso_data);
0202
0203     /* Grab the VDSO text pages. */
0204     for (i = 0; i < text_pages; i++) {
0205         struct page *page;
0206
0207         page = virt_to_page(vdso_start + i * PAGE_SIZE);
0208         vdso_text_pagelist[i] = page;
0209     }
0210
0211     vdso_text_mapping.pages = vdso_text_pagelist;
0212
0213     vdso_total_pages = 1; /* for the data/vvar page */
0214     vdso_total_pages += text_pages;
0215
0216     cntvct_ok = cntvct_functional();
0217
0218     patch_vdso(vdso_start);
0219
0220     return 0;
0221 }
0222 arch_initcall(vdso_init);
0223
0224 static int install_vvar(struct mm_struct *mm, unsigned long addr)
0225 {
0226     struct vm_area_struct *vma;
0227
0228     vma = _install_special_mapping(mm, addr, PAGE_SIZE,
0229                        VM_READ | VM_MAYREAD,
0230                        &vdso_data_mapping);
0231
0232     return PTR_ERR_OR_ZERO(vma);
0233 }
0234
0235 /* assumes mmap_lock is write-locked */
0236 void arm_install_vdso(struct mm_struct *mm, unsigned long addr)
0237 {
0238     struct vm_area_struct *vma;
0239     unsigned long len;
0240
0241     mm->context.vdso = 0;
0242
0243     if (vdso_text_pagelist == NULL)
0244         return;
0245
0246     if (install_vvar(mm, addr))
0247         return;
0248
0249     /* Account for vvar page. */
0250     addr += PAGE_SIZE;
0251     len = (vdso_total_pages - 1) << PAGE_SHIFT;
0252
0253     vma = _install_special_mapping(mm, addr, len,
0254         VM_READ | VM_EXEC | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC,
0255         &vdso_text_mapping);
0256
0257     if (!IS_ERR(vma))
0258         mm->context.vdso = addr;
0259 }
0260