sparc/vdso/vma.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Set up the VMAs to tell the VM about the vDSO.
0004  * Copyright 2007 Andi Kleen, SUSE Labs.
0005  */
0006
0007 /*
0008  * Copyright (c) 2017 Oracle and/or its affiliates. All rights reserved.
0009  */
0010
0011 #include <linux/mm.h>
0012 #include <linux/err.h>
0013 #include <linux/sched.h>
0014 #include <linux/slab.h>
0015 #include <linux/init.h>
0016 #include <linux/linkage.h>
0017 #include <linux/random.h>
0018 #include <linux/elf.h>
0019 #include <asm/cacheflush.h>
0020 #include <asm/spitfire.h>
0021 #include <asm/vdso.h>
0022 #include <asm/vvar.h>
0023 #include <asm/page.h>
0024
0025 unsigned int __read_mostly vdso_enabled = 1;
0026
0027 static struct vm_special_mapping vvar_mapping = {
0028     .name = "[vvar]"
0029 };
0030
0031 #ifdef  CONFIG_SPARC64
0032 static struct vm_special_mapping vdso_mapping64 = {
0033     .name = "[vdso]"
0034 };
0035 #endif
0036
0037 #ifdef CONFIG_COMPAT
0038 static struct vm_special_mapping vdso_mapping32 = {
0039     .name = "[vdso]"
0040 };
0041 #endif
0042
0043 struct vvar_data *vvar_data;
0044
0045 struct vdso_elfinfo32 {
0046     Elf32_Ehdr  *hdr;
0047     Elf32_Sym   *dynsym;
0048     unsigned long   dynsymsize;
0049     const char  *dynstr;
0050     unsigned long   text;
0051 };
0052
0053 struct vdso_elfinfo64 {
0054     Elf64_Ehdr  *hdr;
0055     Elf64_Sym   *dynsym;
0056     unsigned long   dynsymsize;
0057     const char  *dynstr;
0058     unsigned long   text;
0059 };
0060
0061 struct vdso_elfinfo {
0062     union {
0063         struct vdso_elfinfo32 elf32;
0064         struct vdso_elfinfo64 elf64;
0065     } u;
0066 };
0067
0068 static void *one_section64(struct vdso_elfinfo64 *e, const char *name,
0069                unsigned long *size)
0070 {
0071     const char *snames;
0072     Elf64_Shdr *shdrs;
0073     unsigned int i;
0074
0075     shdrs = (void *)e->hdr + e->hdr->e_shoff;
0076     snames = (void *)e->hdr + shdrs[e->hdr->e_shstrndx].sh_offset;
0077     for (i = 1; i < e->hdr->e_shnum; i++) {
0078         if (!strcmp(snames+shdrs[i].sh_name, name)) {
0079             if (size)
0080                 *size = shdrs[i].sh_size;
0081             return (void *)e->hdr + shdrs[i].sh_offset;
0082         }
0083     }
0084     return NULL;
0085 }
0086
0087 static int find_sections64(const struct vdso_image *image, struct vdso_elfinfo *_e)
0088 {
0089     struct vdso_elfinfo64 *e = &_e->u.elf64;
0090
0091     e->hdr = image->data;
0092     e->dynsym = one_section64(e, ".dynsym", &e->dynsymsize);
0093     e->dynstr = one_section64(e, ".dynstr", NULL);
0094
0095     if (!e->dynsym || !e->dynstr) {
0096         pr_err("VDSO64: Missing symbol sections.\n");
0097         return -ENODEV;
0098     }
0099     return 0;
0100 }
0101
0102 static Elf64_Sym *find_sym64(const struct vdso_elfinfo64 *e, const char *name)
0103 {
0104     unsigned int i;
0105
0106     for (i = 0; i < (e->dynsymsize / sizeof(Elf64_Sym)); i++) {
0107         Elf64_Sym *s = &e->dynsym[i];
0108         if (s->st_name == 0)
0109             continue;
0110         if (!strcmp(e->dynstr + s->st_name, name))
0111             return s;
0112     }
0113     return NULL;
0114 }
0115
0116 static int patchsym64(struct vdso_elfinfo *_e, const char *orig,
0117               const char *new)
0118 {
0119     struct vdso_elfinfo64 *e = &_e->u.elf64;
0120     Elf64_Sym *osym = find_sym64(e, orig);
0121     Elf64_Sym *nsym = find_sym64(e, new);
0122
0123     if (!nsym || !osym) {
0124         pr_err("VDSO64: Missing symbols.\n");
0125         return -ENODEV;
0126     }
0127     osym->st_value = nsym->st_value;
0128     osym->st_size = nsym->st_size;
0129     osym->st_info = nsym->st_info;
0130     osym->st_other = nsym->st_other;
0131     osym->st_shndx = nsym->st_shndx;
0132
0133     return 0;
0134 }
0135
0136 static void *one_section32(struct vdso_elfinfo32 *e, const char *name,
0137                unsigned long *size)
0138 {
0139     const char *snames;
0140     Elf32_Shdr *shdrs;
0141     unsigned int i;
0142
0143     shdrs = (void *)e->hdr + e->hdr->e_shoff;
0144     snames = (void *)e->hdr + shdrs[e->hdr->e_shstrndx].sh_offset;
0145     for (i = 1; i < e->hdr->e_shnum; i++) {
0146         if (!strcmp(snames+shdrs[i].sh_name, name)) {
0147             if (size)
0148                 *size = shdrs[i].sh_size;
0149             return (void *)e->hdr + shdrs[i].sh_offset;
0150         }
0151     }
0152     return NULL;
0153 }
0154
0155 static int find_sections32(const struct vdso_image *image, struct vdso_elfinfo *_e)
0156 {
0157     struct vdso_elfinfo32 *e = &_e->u.elf32;
0158
0159     e->hdr = image->data;
0160     e->dynsym = one_section32(e, ".dynsym", &e->dynsymsize);
0161     e->dynstr = one_section32(e, ".dynstr", NULL);
0162
0163     if (!e->dynsym || !e->dynstr) {
0164         pr_err("VDSO32: Missing symbol sections.\n");
0165         return -ENODEV;
0166     }
0167     return 0;
0168 }
0169
0170 static Elf32_Sym *find_sym32(const struct vdso_elfinfo32 *e, const char *name)
0171 {
0172     unsigned int i;
0173
0174     for (i = 0; i < (e->dynsymsize / sizeof(Elf32_Sym)); i++) {
0175         Elf32_Sym *s = &e->dynsym[i];
0176         if (s->st_name == 0)
0177             continue;
0178         if (!strcmp(e->dynstr + s->st_name, name))
0179             return s;
0180     }
0181     return NULL;
0182 }
0183
0184 static int patchsym32(struct vdso_elfinfo *_e, const char *orig,
0185               const char *new)
0186 {
0187     struct vdso_elfinfo32 *e = &_e->u.elf32;
0188     Elf32_Sym *osym = find_sym32(e, orig);
0189     Elf32_Sym *nsym = find_sym32(e, new);
0190
0191     if (!nsym || !osym) {
0192         pr_err("VDSO32: Missing symbols.\n");
0193         return -ENODEV;
0194     }
0195     osym->st_value = nsym->st_value;
0196     osym->st_size = nsym->st_size;
0197     osym->st_info = nsym->st_info;
0198     osym->st_other = nsym->st_other;
0199     osym->st_shndx = nsym->st_shndx;
0200
0201     return 0;
0202 }
0203
0204 static int find_sections(const struct vdso_image *image, struct vdso_elfinfo *e,
0205              bool elf64)
0206 {
0207     if (elf64)
0208         return find_sections64(image, e);
0209     else
0210         return find_sections32(image, e);
0211 }
0212
0213 static int patch_one_symbol(struct vdso_elfinfo *e, const char *orig,
0214                 const char *new_target, bool elf64)
0215 {
0216     if (elf64)
0217         return patchsym64(e, orig, new_target);
0218     else
0219         return patchsym32(e, orig, new_target);
0220 }
0221
0222 static int stick_patch(const struct vdso_image *image, struct vdso_elfinfo *e, bool elf64)
0223 {
0224     int err;
0225
0226     err = find_sections(image, e, elf64);
0227     if (err)
0228         return err;
0229
0230     err = patch_one_symbol(e,
0231                    "__vdso_gettimeofday",
0232                    "__vdso_gettimeofday_stick", elf64);
0233     if (err)
0234         return err;
0235
0236     return patch_one_symbol(e,
0237                 "__vdso_clock_gettime",
0238                 "__vdso_clock_gettime_stick", elf64);
0239     return 0;
0240 }
0241
0242 /*
0243  * Allocate pages for the vdso and vvar, and copy in the vdso text from the
0244  * kernel image.
0245  */
0246 int __init init_vdso_image(const struct vdso_image *image,
0247                struct vm_special_mapping *vdso_mapping, bool elf64)
0248 {
0249     int cnpages = (image->size) / PAGE_SIZE;
0250     struct page *dp, **dpp = NULL;
0251     struct page *cp, **cpp = NULL;
0252     struct vdso_elfinfo ei;
0253     int i, dnpages = 0;
0254
0255     if (tlb_type != spitfire) {
0256         int err = stick_patch(image, &ei, elf64);
0257         if (err)
0258             return err;
0259     }
0260
0261     /*
0262      * First, the vdso text.  This is initialied data, an integral number of
0263      * pages long.
0264      */
0265     if (WARN_ON(image->size % PAGE_SIZE != 0))
0266         goto oom;
0267
0268     cpp = kcalloc(cnpages, sizeof(struct page *), GFP_KERNEL);
0269     vdso_mapping->pages = cpp;
0270
0271     if (!cpp)
0272         goto oom;
0273
0274     for (i = 0; i < cnpages; i++) {
0275         cp = alloc_page(GFP_KERNEL);
0276         if (!cp)
0277             goto oom;
0278         cpp[i] = cp;
0279         copy_page(page_address(cp), image->data + i * PAGE_SIZE);
0280     }
0281
0282     /*
0283      * Now the vvar page.  This is uninitialized data.
0284      */
0285
0286     if (vvar_data == NULL) {
0287         dnpages = (sizeof(struct vvar_data) / PAGE_SIZE) + 1;
0288         if (WARN_ON(dnpages != 1))
0289             goto oom;
0290         dpp = kcalloc(dnpages, sizeof(struct page *), GFP_KERNEL);
0291         vvar_mapping.pages = dpp;
0292
0293         if (!dpp)
0294             goto oom;
0295
0296         dp = alloc_page(GFP_KERNEL);
0297         if (!dp)
0298             goto oom;
0299
0300         dpp[0] = dp;
0301         vvar_data = page_address(dp);
0302         memset(vvar_data, 0, PAGE_SIZE);
0303
0304         vvar_data->seq = 0;
0305     }
0306
0307     return 0;
0308  oom:
0309     if (cpp != NULL) {
0310         for (i = 0; i < cnpages; i++) {
0311             if (cpp[i] != NULL)
0312                 __free_page(cpp[i]);
0313         }
0314         kfree(cpp);
0315         vdso_mapping->pages = NULL;
0316     }
0317
0318     if (dpp != NULL) {
0319         for (i = 0; i < dnpages; i++) {
0320             if (dpp[i] != NULL)
0321                 __free_page(dpp[i]);
0322         }
0323         kfree(dpp);
0324         vvar_mapping.pages = NULL;
0325     }
0326
0327     pr_warn("Cannot allocate vdso\n");
0328     vdso_enabled = 0;
0329     return -ENOMEM;
0330 }
0331
0332 static int __init init_vdso(void)
0333 {
0334     int err = 0;
0335 #ifdef CONFIG_SPARC64
0336     err = init_vdso_image(&vdso_image_64_builtin, &vdso_mapping64, true);
0337     if (err)
0338         return err;
0339 #endif
0340
0341 #ifdef CONFIG_COMPAT
0342     err = init_vdso_image(&vdso_image_32_builtin, &vdso_mapping32, false);
0343 #endif
0344     return err;
0345
0346 }
0347 subsys_initcall(init_vdso);
0348
0349 struct linux_binprm;
0350
0351 /* Shuffle the vdso up a bit, randomly. */
0352 static unsigned long vdso_addr(unsigned long start, unsigned int len)
0353 {
0354     unsigned int offset;
0355
0356     /* This loses some more bits than a modulo, but is cheaper */
0357     offset = get_random_int() & (PTRS_PER_PTE - 1);
0358     return start + (offset << PAGE_SHIFT);
0359 }
0360
0361 static int map_vdso(const struct vdso_image *image,
0362         struct vm_special_mapping *vdso_mapping)
0363 {
0364     struct mm_struct *mm = current->mm;
0365     struct vm_area_struct *vma;
0366     unsigned long text_start, addr = 0;
0367     int ret = 0;
0368
0369     mmap_write_lock(mm);
0370
0371     /*
0372      * First, get an unmapped region: then randomize it, and make sure that
0373      * region is free.
0374      */
0375     if (current->flags & PF_RANDOMIZE) {
0376         addr = get_unmapped_area(NULL, 0,
0377                      image->size - image->sym_vvar_start,
0378                      0, 0);
0379         if (IS_ERR_VALUE(addr)) {
0380             ret = addr;
0381             goto up_fail;
0382         }
0383         addr = vdso_addr(addr, image->size - image->sym_vvar_start);
0384     }
0385     addr = get_unmapped_area(NULL, addr,
0386                  image->size - image->sym_vvar_start, 0, 0);
0387     if (IS_ERR_VALUE(addr)) {
0388         ret = addr;
0389         goto up_fail;
0390     }
0391
0392     text_start = addr - image->sym_vvar_start;
0393     current->mm->context.vdso = (void __user *)text_start;
0394
0395     /*
0396      * MAYWRITE to allow gdb to COW and set breakpoints
0397      */
0398     vma = _install_special_mapping(mm,
0399                        text_start,
0400                        image->size,
0401                        VM_READ|VM_EXEC|
0402                        VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC,
0403                        vdso_mapping);
0404
0405     if (IS_ERR(vma)) {
0406         ret = PTR_ERR(vma);
0407         goto up_fail;
0408     }
0409
0410     vma = _install_special_mapping(mm,
0411                        addr,
0412                        -image->sym_vvar_start,
0413                        VM_READ|VM_MAYREAD,
0414                        &vvar_mapping);
0415
0416     if (IS_ERR(vma)) {
0417         ret = PTR_ERR(vma);
0418         do_munmap(mm, text_start, image->size, NULL);
0419     }
0420
0421 up_fail:
0422     if (ret)
0423         current->mm->context.vdso = NULL;
0424
0425     mmap_write_unlock(mm);
0426     return ret;
0427 }
0428
0429 int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
0430 {
0431
0432     if (!vdso_enabled)
0433         return 0;
0434
0435 #if defined CONFIG_COMPAT
0436     if (!(is_32bit_task()))
0437         return map_vdso(&vdso_image_64_builtin, &vdso_mapping64);
0438     else
0439         return map_vdso(&vdso_image_32_builtin, &vdso_mapping32);
0440 #else
0441     return map_vdso(&vdso_image_64_builtin, &vdso_mapping64);
0442 #endif
0443
0444 }
0445
0446 static __init int vdso_setup(char *s)
0447 {
0448     int err;
0449     unsigned long val;
0450
0451     err = kstrtoul(s, 10, &val);
0452     if (err)
0453         return err;
0454     vdso_enabled = val;
0455     return 0;
0456 }
0457 __setup("vdso=", vdso_setup);