sparc/kernel/adi_64.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /* adi_64.c: support for ADI (Application Data Integrity) feature on
0003  * sparc m7 and newer processors. This feature is also known as
0004  * SSM (Silicon Secured Memory).
0005  *
0006  * Copyright (C) 2016 Oracle and/or its affiliates. All rights reserved.
0007  * Author: Khalid Aziz (khalid.aziz@oracle.com)
0008  */
0009 #include <linux/init.h>
0010 #include <linux/slab.h>
0011 #include <linux/mm_types.h>
0012 #include <asm/mdesc.h>
0013 #include <asm/adi_64.h>
0014 #include <asm/mmu_64.h>
0015 #include <asm/pgtable_64.h>
0016
0017 /* Each page of storage for ADI tags can accommodate tags for 128
0018  * pages. When ADI enabled pages are being swapped out, it would be
0019  * prudent to allocate at least enough tag storage space to accommodate
0020  * SWAPFILE_CLUSTER number of pages. Allocate enough tag storage to
0021  * store tags for four SWAPFILE_CLUSTER pages to reduce need for
0022  * further allocations for same vma.
0023  */
0024 #define TAG_STORAGE_PAGES   8
0025
0026 struct adi_config adi_state;
0027 EXPORT_SYMBOL(adi_state);
0028
0029 /* mdesc_adi_init() : Parse machine description provided by the
0030  *  hypervisor to detect ADI capabilities
0031  *
0032  * Hypervisor reports ADI capabilities of platform in "hwcap-list" property
0033  * for "cpu" node. If the platform supports ADI, "hwcap-list" property
0034  * contains the keyword "adp". If the platform supports ADI, "platform"
0035  * node will contain "adp-blksz", "adp-nbits" and "ue-on-adp" properties
0036  * to describe the ADI capabilities.
0037  */
0038 void __init mdesc_adi_init(void)
0039 {
0040     struct mdesc_handle *hp = mdesc_grab();
0041     const char *prop;
0042     u64 pn, *val;
0043     int len;
0044
0045     if (!hp)
0046         goto adi_not_found;
0047
0048     pn = mdesc_node_by_name(hp, MDESC_NODE_NULL, "cpu");
0049     if (pn == MDESC_NODE_NULL)
0050         goto adi_not_found;
0051
0052     prop = mdesc_get_property(hp, pn, "hwcap-list", &len);
0053     if (!prop)
0054         goto adi_not_found;
0055
0056     /*
0057      * Look for "adp" keyword in hwcap-list which would indicate
0058      * ADI support
0059      */
0060     adi_state.enabled = false;
0061     while (len) {
0062         int plen;
0063
0064         if (!strcmp(prop, "adp")) {
0065             adi_state.enabled = true;
0066             break;
0067         }
0068
0069         plen = strlen(prop) + 1;
0070         prop += plen;
0071         len -= plen;
0072     }
0073
0074     if (!adi_state.enabled)
0075         goto adi_not_found;
0076
0077     /* Find the ADI properties in "platform" node. If all ADI
0078      * properties are not found, ADI support is incomplete and
0079      * do not enable ADI in the kernel.
0080      */
0081     pn = mdesc_node_by_name(hp, MDESC_NODE_NULL, "platform");
0082     if (pn == MDESC_NODE_NULL)
0083         goto adi_not_found;
0084
0085     val = (u64 *) mdesc_get_property(hp, pn, "adp-blksz", &len);
0086     if (!val)
0087         goto adi_not_found;
0088     adi_state.caps.blksz = *val;
0089
0090     val = (u64 *) mdesc_get_property(hp, pn, "adp-nbits", &len);
0091     if (!val)
0092         goto adi_not_found;
0093     adi_state.caps.nbits = *val;
0094
0095     val = (u64 *) mdesc_get_property(hp, pn, "ue-on-adp", &len);
0096     if (!val)
0097         goto adi_not_found;
0098     adi_state.caps.ue_on_adi = *val;
0099
0100     /* Some of the code to support swapping ADI tags is written
0101      * assumption that two ADI tags can fit inside one byte. If
0102      * this assumption is broken by a future architecture change,
0103      * that code will have to be revisited. If that were to happen,
0104      * disable ADI support so we do not get unpredictable results
0105      * with programs trying to use ADI and their pages getting
0106      * swapped out
0107      */
0108     if (adi_state.caps.nbits > 4) {
0109         pr_warn("WARNING: ADI tag size >4 on this platform. Disabling AADI support\n");
0110         adi_state.enabled = false;
0111     }
0112
0113     mdesc_release(hp);
0114     return;
0115
0116 adi_not_found:
0117     adi_state.enabled = false;
0118     adi_state.caps.blksz = 0;
0119     adi_state.caps.nbits = 0;
0120     if (hp)
0121         mdesc_release(hp);
0122 }
0123
0124 tag_storage_desc_t *find_tag_store(struct mm_struct *mm,
0125                    struct vm_area_struct *vma,
0126                    unsigned long addr)
0127 {
0128     tag_storage_desc_t *tag_desc = NULL;
0129     unsigned long i, max_desc, flags;
0130
0131     /* Check if this vma already has tag storage descriptor
0132      * allocated for it.
0133      */
0134     max_desc = PAGE_SIZE/sizeof(tag_storage_desc_t);
0135     if (mm->context.tag_store) {
0136         tag_desc = mm->context.tag_store;
0137         spin_lock_irqsave(&mm->context.tag_lock, flags);
0138         for (i = 0; i < max_desc; i++) {
0139             if ((addr >= tag_desc->start) &&
0140                 ((addr + PAGE_SIZE - 1) <= tag_desc->end))
0141                 break;
0142             tag_desc++;
0143         }
0144         spin_unlock_irqrestore(&mm->context.tag_lock, flags);
0145
0146         /* If no matching entries were found, this must be a
0147          * freshly allocated page
0148          */
0149         if (i >= max_desc)
0150             tag_desc = NULL;
0151     }
0152
0153     return tag_desc;
0154 }
0155
0156 tag_storage_desc_t *alloc_tag_store(struct mm_struct *mm,
0157                     struct vm_area_struct *vma,
0158                     unsigned long addr)
0159 {
0160     unsigned char *tags;
0161     unsigned long i, size, max_desc, flags;
0162     tag_storage_desc_t *tag_desc, *open_desc;
0163     unsigned long end_addr, hole_start, hole_end;
0164
0165     max_desc = PAGE_SIZE/sizeof(tag_storage_desc_t);
0166     open_desc = NULL;
0167     hole_start = 0;
0168     hole_end = ULONG_MAX;
0169     end_addr = addr + PAGE_SIZE - 1;
0170
0171     /* Check if this vma already has tag storage descriptor
0172      * allocated for it.
0173      */
0174     spin_lock_irqsave(&mm->context.tag_lock, flags);
0175     if (mm->context.tag_store) {
0176         tag_desc = mm->context.tag_store;
0177
0178         /* Look for a matching entry for this address. While doing
0179          * that, look for the first open slot as well and find
0180          * the hole in already allocated range where this request
0181          * will fit in.
0182          */
0183         for (i = 0; i < max_desc; i++) {
0184             if (tag_desc->tag_users == 0) {
0185                 if (open_desc == NULL)
0186                     open_desc = tag_desc;
0187             } else {
0188                 if ((addr >= tag_desc->start) &&
0189                     (tag_desc->end >= (addr + PAGE_SIZE - 1))) {
0190                     tag_desc->tag_users++;
0191                     goto out;
0192                 }
0193             }
0194             if ((tag_desc->start > end_addr) &&
0195                 (tag_desc->start < hole_end))
0196                 hole_end = tag_desc->start;
0197             if ((tag_desc->end < addr) &&
0198                 (tag_desc->end > hole_start))
0199                 hole_start = tag_desc->end;
0200             tag_desc++;
0201         }
0202
0203     } else {
0204         size = sizeof(tag_storage_desc_t)*max_desc;
0205         mm->context.tag_store = kzalloc(size, GFP_NOWAIT|__GFP_NOWARN);
0206         if (mm->context.tag_store == NULL) {
0207             tag_desc = NULL;
0208             goto out;
0209         }
0210         tag_desc = mm->context.tag_store;
0211         for (i = 0; i < max_desc; i++, tag_desc++)
0212             tag_desc->tag_users = 0;
0213         open_desc = mm->context.tag_store;
0214         i = 0;
0215     }
0216
0217     /* Check if we ran out of tag storage descriptors */
0218     if (open_desc == NULL) {
0219         tag_desc = NULL;
0220         goto out;
0221     }
0222
0223     /* Mark this tag descriptor slot in use and then initialize it */
0224     tag_desc = open_desc;
0225     tag_desc->tag_users = 1;
0226
0227     /* Tag storage has not been allocated for this vma and space
0228      * is available in tag storage descriptor. Since this page is
0229      * being swapped out, there is high probability subsequent pages
0230      * in the VMA will be swapped out as well. Allocate pages to
0231      * store tags for as many pages in this vma as possible but not
0232      * more than TAG_STORAGE_PAGES. Each byte in tag space holds
0233      * two ADI tags since each ADI tag is 4 bits. Each ADI tag
0234      * covers adi_blksize() worth of addresses. Check if the hole is
0235      * big enough to accommodate full address range for using
0236      * TAG_STORAGE_PAGES number of tag pages.
0237      */
0238     size = TAG_STORAGE_PAGES * PAGE_SIZE;
0239     end_addr = addr + (size*2*adi_blksize()) - 1;
0240     /* Check for overflow. If overflow occurs, allocate only one page */
0241     if (end_addr < addr) {
0242         size = PAGE_SIZE;
0243         end_addr = addr + (size*2*adi_blksize()) - 1;
0244         /* If overflow happens with the minimum tag storage
0245          * allocation as well, adjust ending address for this
0246          * tag storage.
0247          */
0248         if (end_addr < addr)
0249             end_addr = ULONG_MAX;
0250     }
0251     if (hole_end < end_addr) {
0252         /* Available hole is too small on the upper end of
0253          * address. Can we expand the range towards the lower
0254          * address and maximize use of this slot?
0255          */
0256         unsigned long tmp_addr;
0257
0258         end_addr = hole_end - 1;
0259         tmp_addr = end_addr - (size*2*adi_blksize()) + 1;
0260         /* Check for underflow. If underflow occurs, allocate
0261          * only one page for storing ADI tags
0262          */
0263         if (tmp_addr > addr) {
0264             size = PAGE_SIZE;
0265             tmp_addr = end_addr - (size*2*adi_blksize()) - 1;
0266             /* If underflow happens with the minimum tag storage
0267              * allocation as well, adjust starting address for
0268              * this tag storage.
0269              */
0270             if (tmp_addr > addr)
0271                 tmp_addr = 0;
0272         }
0273         if (tmp_addr < hole_start) {
0274             /* Available hole is restricted on lower address
0275              * end as well
0276              */
0277             tmp_addr = hole_start + 1;
0278         }
0279         addr = tmp_addr;
0280         size = (end_addr + 1 - addr)/(2*adi_blksize());
0281         size = (size + (PAGE_SIZE-adi_blksize()))/PAGE_SIZE;
0282         size = size * PAGE_SIZE;
0283     }
0284     tags = kzalloc(size, GFP_NOWAIT|__GFP_NOWARN);
0285     if (tags == NULL) {
0286         tag_desc->tag_users = 0;
0287         tag_desc = NULL;
0288         goto out;
0289     }
0290     tag_desc->start = addr;
0291     tag_desc->tags = tags;
0292     tag_desc->end = end_addr;
0293
0294 out:
0295     spin_unlock_irqrestore(&mm->context.tag_lock, flags);
0296     return tag_desc;
0297 }
0298
0299 void del_tag_store(tag_storage_desc_t *tag_desc, struct mm_struct *mm)
0300 {
0301     unsigned long flags;
0302     unsigned char *tags = NULL;
0303
0304     spin_lock_irqsave(&mm->context.tag_lock, flags);
0305     tag_desc->tag_users--;
0306     if (tag_desc->tag_users == 0) {
0307         tag_desc->start = tag_desc->end = 0;
0308         /* Do not free up the tag storage space allocated
0309          * by the first descriptor. This is persistent
0310          * emergency tag storage space for the task.
0311          */
0312         if (tag_desc != mm->context.tag_store) {
0313             tags = tag_desc->tags;
0314             tag_desc->tags = NULL;
0315         }
0316     }
0317     spin_unlock_irqrestore(&mm->context.tag_lock, flags);
0318     kfree(tags);
0319 }
0320
0321 #define tag_start(addr, tag_desc)       \
0322     ((tag_desc)->tags + ((addr - (tag_desc)->start)/(2*adi_blksize())))
0323
0324 /* Retrieve any saved ADI tags for the page being swapped back in and
0325  * restore these tags to the newly allocated physical page.
0326  */
0327 void adi_restore_tags(struct mm_struct *mm, struct vm_area_struct *vma,
0328               unsigned long addr, pte_t pte)
0329 {
0330     unsigned char *tag;
0331     tag_storage_desc_t *tag_desc;
0332     unsigned long paddr, tmp, version1, version2;
0333
0334     /* Check if the swapped out page has an ADI version
0335      * saved. If yes, restore version tag to the newly
0336      * allocated page.
0337      */
0338     tag_desc = find_tag_store(mm, vma, addr);
0339     if (tag_desc == NULL)
0340         return;
0341
0342     tag = tag_start(addr, tag_desc);
0343     paddr = pte_val(pte) & _PAGE_PADDR_4V;
0344     for (tmp = paddr; tmp < (paddr+PAGE_SIZE); tmp += adi_blksize()) {
0345         version1 = (*tag) >> 4;
0346         version2 = (*tag) & 0x0f;
0347         *tag++ = 0;
0348         asm volatile("stxa %0, [%1] %2\n\t"
0349             :
0350             : "r" (version1), "r" (tmp),
0351               "i" (ASI_MCD_REAL));
0352         tmp += adi_blksize();
0353         asm volatile("stxa %0, [%1] %2\n\t"
0354             :
0355             : "r" (version2), "r" (tmp),
0356               "i" (ASI_MCD_REAL));
0357     }
0358     asm volatile("membar #Sync\n\t");
0359
0360     /* Check and mark this tag space for release later if
0361      * the swapped in page was the last user of tag space
0362      */
0363     del_tag_store(tag_desc, mm);
0364 }
0365
0366 /* A page is about to be swapped out. Save any ADI tags associated with
0367  * this physical page so they can be restored later when the page is swapped
0368  * back in.
0369  */
0370 int adi_save_tags(struct mm_struct *mm, struct vm_area_struct *vma,
0371           unsigned long addr, pte_t oldpte)
0372 {
0373     unsigned char *tag;
0374     tag_storage_desc_t *tag_desc;
0375     unsigned long version1, version2, paddr, tmp;
0376
0377     tag_desc = alloc_tag_store(mm, vma, addr);
0378     if (tag_desc == NULL)
0379         return -1;
0380
0381     tag = tag_start(addr, tag_desc);
0382     paddr = pte_val(oldpte) & _PAGE_PADDR_4V;
0383     for (tmp = paddr; tmp < (paddr+PAGE_SIZE); tmp += adi_blksize()) {
0384         asm volatile("ldxa [%1] %2, %0\n\t"
0385                 : "=r" (version1)
0386                 : "r" (tmp), "i" (ASI_MCD_REAL));
0387         tmp += adi_blksize();
0388         asm volatile("ldxa [%1] %2, %0\n\t"
0389                 : "=r" (version2)
0390                 : "r" (tmp), "i" (ASI_MCD_REAL));
0391         *tag = (version1 << 4) | version2;
0392         tag++;
0393     }
0394
0395     return 0;
0396 }