mm/book3s64/iommu_api.c

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  *  IOMMU helpers in MMU context.
0004  *
0005  *  Copyright (C) 2015 IBM Corp. <aik@ozlabs.ru>
0006  */
0007
0008 #include <linux/sched/signal.h>
0009 #include <linux/slab.h>
0010 #include <linux/rculist.h>
0011 #include <linux/vmalloc.h>
0012 #include <linux/mutex.h>
0013 #include <linux/migrate.h>
0014 #include <linux/hugetlb.h>
0015 #include <linux/swap.h>
0016 #include <linux/sizes.h>
0017 #include <linux/mm.h>
0018 #include <asm/mmu_context.h>
0019 #include <asm/pte-walk.h>
0020 #include <linux/mm_inline.h>
0021
0022 static DEFINE_MUTEX(mem_list_mutex);
0023
0024 #define MM_IOMMU_TABLE_GROUP_PAGE_DIRTY 0x1
0025 #define MM_IOMMU_TABLE_GROUP_PAGE_MASK  ~(SZ_4K - 1)
0026
0027 struct mm_iommu_table_group_mem_t {
0028     struct list_head next;
0029     struct rcu_head rcu;
0030     unsigned long used;
0031     atomic64_t mapped;
0032     unsigned int pageshift;
0033     u64 ua;         /* userspace address */
0034     u64 entries;        /* number of entries in hpas/hpages[] */
0035     /*
0036      * in mm_iommu_get we temporarily use this to store
0037      * struct page address.
0038      *
0039      * We need to convert ua to hpa in real mode. Make it
0040      * simpler by storing physical address.
0041      */
0042     union {
0043         struct page **hpages;   /* vmalloc'ed */
0044         phys_addr_t *hpas;
0045     };
0046 #define MM_IOMMU_TABLE_INVALID_HPA  ((uint64_t)-1)
0047     u64 dev_hpa;        /* Device memory base address */
0048 };
0049
0050 bool mm_iommu_preregistered(struct mm_struct *mm)
0051 {
0052     return !list_empty(&mm->context.iommu_group_mem_list);
0053 }
0054 EXPORT_SYMBOL_GPL(mm_iommu_preregistered);
0055
0056 static long mm_iommu_do_alloc(struct mm_struct *mm, unsigned long ua,
0057                   unsigned long entries, unsigned long dev_hpa,
0058                   struct mm_iommu_table_group_mem_t **pmem)
0059 {
0060     struct mm_iommu_table_group_mem_t *mem, *mem2;
0061     long i, ret, locked_entries = 0, pinned = 0;
0062     unsigned int pageshift;
0063     unsigned long entry, chunk;
0064
0065     if (dev_hpa == MM_IOMMU_TABLE_INVALID_HPA) {
0066         ret = account_locked_vm(mm, entries, true);
0067         if (ret)
0068             return ret;
0069
0070         locked_entries = entries;
0071     }
0072
0073     mem = kzalloc(sizeof(*mem), GFP_KERNEL);
0074     if (!mem) {
0075         ret = -ENOMEM;
0076         goto unlock_exit;
0077     }
0078
0079     if (dev_hpa != MM_IOMMU_TABLE_INVALID_HPA) {
0080         mem->pageshift = __ffs(dev_hpa | (entries << PAGE_SHIFT));
0081         mem->dev_hpa = dev_hpa;
0082         goto good_exit;
0083     }
0084     mem->dev_hpa = MM_IOMMU_TABLE_INVALID_HPA;
0085
0086     /*
0087      * For a starting point for a maximum page size calculation
0088      * we use @ua and @entries natural alignment to allow IOMMU pages
0089      * smaller than huge pages but still bigger than PAGE_SIZE.
0090      */
0091     mem->pageshift = __ffs(ua | (entries << PAGE_SHIFT));
0092     mem->hpas = vzalloc(array_size(entries, sizeof(mem->hpas[0])));
0093     if (!mem->hpas) {
0094         kfree(mem);
0095         ret = -ENOMEM;
0096         goto unlock_exit;
0097     }
0098
0099     mmap_read_lock(mm);
0100     chunk = (1UL << (PAGE_SHIFT + MAX_ORDER - 1)) /
0101             sizeof(struct vm_area_struct *);
0102     chunk = min(chunk, entries);
0103     for (entry = 0; entry < entries; entry += chunk) {
0104         unsigned long n = min(entries - entry, chunk);
0105
0106         ret = pin_user_pages(ua + (entry << PAGE_SHIFT), n,
0107                 FOLL_WRITE | FOLL_LONGTERM,
0108                 mem->hpages + entry, NULL);
0109         if (ret == n) {
0110             pinned += n;
0111             continue;
0112         }
0113         if (ret > 0)
0114             pinned += ret;
0115         break;
0116     }
0117     mmap_read_unlock(mm);
0118     if (pinned != entries) {
0119         if (!ret)
0120             ret = -EFAULT;
0121         goto free_exit;
0122     }
0123
0124 good_exit:
0125     atomic64_set(&mem->mapped, 1);
0126     mem->used = 1;
0127     mem->ua = ua;
0128     mem->entries = entries;
0129
0130     mutex_lock(&mem_list_mutex);
0131
0132     list_for_each_entry_rcu(mem2, &mm->context.iommu_group_mem_list, next,
0133                 lockdep_is_held(&mem_list_mutex)) {
0134         /* Overlap? */
0135         if ((mem2->ua < (ua + (entries << PAGE_SHIFT))) &&
0136                 (ua < (mem2->ua +
0137                        (mem2->entries << PAGE_SHIFT)))) {
0138             ret = -EINVAL;
0139             mutex_unlock(&mem_list_mutex);
0140             goto free_exit;
0141         }
0142     }
0143
0144     if (mem->dev_hpa == MM_IOMMU_TABLE_INVALID_HPA) {
0145         /*
0146          * Allow to use larger than 64k IOMMU pages. Only do that
0147          * if we are backed by hugetlb. Skip device memory as it is not
0148          * backed with page structs.
0149          */
0150         pageshift = PAGE_SHIFT;
0151         for (i = 0; i < entries; ++i) {
0152             struct page *page = mem->hpages[i];
0153
0154             if ((mem->pageshift > PAGE_SHIFT) && PageHuge(page))
0155                 pageshift = page_shift(compound_head(page));
0156             mem->pageshift = min(mem->pageshift, pageshift);
0157             /*
0158              * We don't need struct page reference any more, switch
0159              * to physical address.
0160              */
0161             mem->hpas[i] = page_to_pfn(page) << PAGE_SHIFT;
0162         }
0163     }
0164
0165     list_add_rcu(&mem->next, &mm->context.iommu_group_mem_list);
0166
0167     mutex_unlock(&mem_list_mutex);
0168
0169     *pmem = mem;
0170
0171     return 0;
0172
0173 free_exit:
0174     /* free the references taken */
0175     unpin_user_pages(mem->hpages, pinned);
0176
0177     vfree(mem->hpas);
0178     kfree(mem);
0179
0180 unlock_exit:
0181     account_locked_vm(mm, locked_entries, false);
0182
0183     return ret;
0184 }
0185
0186 long mm_iommu_new(struct mm_struct *mm, unsigned long ua, unsigned long entries,
0187         struct mm_iommu_table_group_mem_t **pmem)
0188 {
0189     return mm_iommu_do_alloc(mm, ua, entries, MM_IOMMU_TABLE_INVALID_HPA,
0190             pmem);
0191 }
0192 EXPORT_SYMBOL_GPL(mm_iommu_new);
0193
0194 long mm_iommu_newdev(struct mm_struct *mm, unsigned long ua,
0195         unsigned long entries, unsigned long dev_hpa,
0196         struct mm_iommu_table_group_mem_t **pmem)
0197 {
0198     return mm_iommu_do_alloc(mm, ua, entries, dev_hpa, pmem);
0199 }
0200 EXPORT_SYMBOL_GPL(mm_iommu_newdev);
0201
0202 static void mm_iommu_unpin(struct mm_iommu_table_group_mem_t *mem)
0203 {
0204     long i;
0205     struct page *page = NULL;
0206
0207     if (!mem->hpas)
0208         return;
0209
0210     for (i = 0; i < mem->entries; ++i) {
0211         if (!mem->hpas[i])
0212             continue;
0213
0214         page = pfn_to_page(mem->hpas[i] >> PAGE_SHIFT);
0215         if (!page)
0216             continue;
0217
0218         if (mem->hpas[i] & MM_IOMMU_TABLE_GROUP_PAGE_DIRTY)
0219             SetPageDirty(page);
0220
0221         unpin_user_page(page);
0222
0223         mem->hpas[i] = 0;
0224     }
0225 }
0226
0227 static void mm_iommu_do_free(struct mm_iommu_table_group_mem_t *mem)
0228 {
0229
0230     mm_iommu_unpin(mem);
0231     vfree(mem->hpas);
0232     kfree(mem);
0233 }
0234
0235 static void mm_iommu_free(struct rcu_head *head)
0236 {
0237     struct mm_iommu_table_group_mem_t *mem = container_of(head,
0238             struct mm_iommu_table_group_mem_t, rcu);
0239
0240     mm_iommu_do_free(mem);
0241 }
0242
0243 static void mm_iommu_release(struct mm_iommu_table_group_mem_t *mem)
0244 {
0245     list_del_rcu(&mem->next);
0246     call_rcu(&mem->rcu, mm_iommu_free);
0247 }
0248
0249 long mm_iommu_put(struct mm_struct *mm, struct mm_iommu_table_group_mem_t *mem)
0250 {
0251     long ret = 0;
0252     unsigned long unlock_entries = 0;
0253
0254     mutex_lock(&mem_list_mutex);
0255
0256     if (mem->used == 0) {
0257         ret = -ENOENT;
0258         goto unlock_exit;
0259     }
0260
0261     --mem->used;
0262     /* There are still users, exit */
0263     if (mem->used)
0264         goto unlock_exit;
0265
0266     /* Are there still mappings? */
0267     if (atomic64_cmpxchg(&mem->mapped, 1, 0) != 1) {
0268         ++mem->used;
0269         ret = -EBUSY;
0270         goto unlock_exit;
0271     }
0272
0273     if (mem->dev_hpa == MM_IOMMU_TABLE_INVALID_HPA)
0274         unlock_entries = mem->entries;
0275
0276     /* @mapped became 0 so now mappings are disabled, release the region */
0277     mm_iommu_release(mem);
0278
0279 unlock_exit:
0280     mutex_unlock(&mem_list_mutex);
0281
0282     account_locked_vm(mm, unlock_entries, false);
0283
0284     return ret;
0285 }
0286 EXPORT_SYMBOL_GPL(mm_iommu_put);
0287
0288 struct mm_iommu_table_group_mem_t *mm_iommu_lookup(struct mm_struct *mm,
0289         unsigned long ua, unsigned long size)
0290 {
0291     struct mm_iommu_table_group_mem_t *mem, *ret = NULL;
0292
0293     rcu_read_lock();
0294     list_for_each_entry_rcu(mem, &mm->context.iommu_group_mem_list, next) {
0295         if ((mem->ua <= ua) &&
0296                 (ua + size <= mem->ua +
0297                  (mem->entries << PAGE_SHIFT))) {
0298             ret = mem;
0299             break;
0300         }
0301     }
0302     rcu_read_unlock();
0303
0304     return ret;
0305 }
0306 EXPORT_SYMBOL_GPL(mm_iommu_lookup);
0307
0308 struct mm_iommu_table_group_mem_t *mm_iommu_get(struct mm_struct *mm,
0309         unsigned long ua, unsigned long entries)
0310 {
0311     struct mm_iommu_table_group_mem_t *mem, *ret = NULL;
0312
0313     mutex_lock(&mem_list_mutex);
0314
0315     list_for_each_entry_rcu(mem, &mm->context.iommu_group_mem_list, next,
0316                 lockdep_is_held(&mem_list_mutex)) {
0317         if ((mem->ua == ua) && (mem->entries == entries)) {
0318             ret = mem;
0319             ++mem->used;
0320             break;
0321         }
0322     }
0323
0324     mutex_unlock(&mem_list_mutex);
0325
0326     return ret;
0327 }
0328 EXPORT_SYMBOL_GPL(mm_iommu_get);
0329
0330 long mm_iommu_ua_to_hpa(struct mm_iommu_table_group_mem_t *mem,
0331         unsigned long ua, unsigned int pageshift, unsigned long *hpa)
0332 {
0333     const long entry = (ua - mem->ua) >> PAGE_SHIFT;
0334     u64 *va;
0335
0336     if (entry >= mem->entries)
0337         return -EFAULT;
0338
0339     if (pageshift > mem->pageshift)
0340         return -EFAULT;
0341
0342     if (!mem->hpas) {
0343         *hpa = mem->dev_hpa + (ua - mem->ua);
0344         return 0;
0345     }
0346
0347     va = &mem->hpas[entry];
0348     *hpa = (*va & MM_IOMMU_TABLE_GROUP_PAGE_MASK) | (ua & ~PAGE_MASK);
0349
0350     return 0;
0351 }
0352 EXPORT_SYMBOL_GPL(mm_iommu_ua_to_hpa);
0353
0354 bool mm_iommu_is_devmem(struct mm_struct *mm, unsigned long hpa,
0355         unsigned int pageshift, unsigned long *size)
0356 {
0357     struct mm_iommu_table_group_mem_t *mem;
0358     unsigned long end;
0359
0360     rcu_read_lock();
0361     list_for_each_entry_rcu(mem, &mm->context.iommu_group_mem_list, next) {
0362         if (mem->dev_hpa == MM_IOMMU_TABLE_INVALID_HPA)
0363             continue;
0364
0365         end = mem->dev_hpa + (mem->entries << PAGE_SHIFT);
0366         if ((mem->dev_hpa <= hpa) && (hpa < end)) {
0367             /*
0368              * Since the IOMMU page size might be bigger than
0369              * PAGE_SIZE, the amount of preregistered memory
0370              * starting from @hpa might be smaller than 1<<pageshift
0371              * and the caller needs to distinguish this situation.
0372              */
0373             *size = min(1UL << pageshift, end - hpa);
0374             return true;
0375         }
0376     }
0377     rcu_read_unlock();
0378
0379     return false;
0380 }
0381 EXPORT_SYMBOL_GPL(mm_iommu_is_devmem);
0382
0383 long mm_iommu_mapped_inc(struct mm_iommu_table_group_mem_t *mem)
0384 {
0385     if (atomic64_inc_not_zero(&mem->mapped))
0386         return 0;
0387
0388     /* Last mm_iommu_put() has been called, no more mappings allowed() */
0389     return -ENXIO;
0390 }
0391 EXPORT_SYMBOL_GPL(mm_iommu_mapped_inc);
0392
0393 void mm_iommu_mapped_dec(struct mm_iommu_table_group_mem_t *mem)
0394 {
0395     atomic64_add_unless(&mem->mapped, -1, 1);
0396 }
0397 EXPORT_SYMBOL_GPL(mm_iommu_mapped_dec);
0398
0399 void mm_iommu_init(struct mm_struct *mm)
0400 {
0401     INIT_LIST_HEAD_RCU(&mm->context.iommu_group_mem_list);
0402 }