subdev/mmu/base.c

0001 /*
0002  * Copyright 2010 Red Hat Inc.
0003  *
0004  * Permission is hereby granted, free of charge, to any person obtaining a
0005  * copy of this software and associated documentation files (the "Software"),
0006  * to deal in the Software without restriction, including without limitation
0007  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
0008  * and/or sell copies of the Software, and to permit persons to whom the
0009  * Software is furnished to do so, subject to the following conditions:
0010  *
0011  * The above copyright notice and this permission notice shall be included in
0012  * all copies or substantial portions of the Software.
0013  *
0014  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
0015  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
0016  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
0017  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
0018  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
0019  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
0020  * OTHER DEALINGS IN THE SOFTWARE.
0021  *
0022  * Authors: Ben Skeggs
0023  */
0024 #include "ummu.h"
0025 #include "vmm.h"
0026
0027 #include <subdev/bar.h>
0028 #include <subdev/fb.h>
0029
0030 #include <nvif/if500d.h>
0031 #include <nvif/if900d.h>
0032
0033 struct nvkm_mmu_ptp {
0034     struct nvkm_mmu_pt *pt;
0035     struct list_head head;
0036     u8  shift;
0037     u16 mask;
0038     u16 free;
0039 };
0040
0041 static void
0042 nvkm_mmu_ptp_put(struct nvkm_mmu *mmu, bool force, struct nvkm_mmu_pt *pt)
0043 {
0044     const int slot = pt->base >> pt->ptp->shift;
0045     struct nvkm_mmu_ptp *ptp = pt->ptp;
0046
0047     /* If there were no free slots in the parent allocation before,
0048      * there will be now, so return PTP to the cache.
0049      */
0050     if (!ptp->free)
0051         list_add(&ptp->head, &mmu->ptp.list);
0052     ptp->free |= BIT(slot);
0053
0054     /* If there's no more sub-allocations, destroy PTP. */
0055     if (ptp->free == ptp->mask) {
0056         nvkm_mmu_ptc_put(mmu, force, &ptp->pt);
0057         list_del(&ptp->head);
0058         kfree(ptp);
0059     }
0060
0061     kfree(pt);
0062 }
0063
0064 static struct nvkm_mmu_pt *
0065 nvkm_mmu_ptp_get(struct nvkm_mmu *mmu, u32 size, bool zero)
0066 {
0067     struct nvkm_mmu_pt *pt;
0068     struct nvkm_mmu_ptp *ptp;
0069     int slot;
0070
0071     if (!(pt = kzalloc(sizeof(*pt), GFP_KERNEL)))
0072         return NULL;
0073
0074     ptp = list_first_entry_or_null(&mmu->ptp.list, typeof(*ptp), head);
0075     if (!ptp) {
0076         /* Need to allocate a new parent to sub-allocate from. */
0077         if (!(ptp = kmalloc(sizeof(*ptp), GFP_KERNEL))) {
0078             kfree(pt);
0079             return NULL;
0080         }
0081
0082         ptp->pt = nvkm_mmu_ptc_get(mmu, 0x1000, 0x1000, false);
0083         if (!ptp->pt) {
0084             kfree(ptp);
0085             kfree(pt);
0086             return NULL;
0087         }
0088
0089         ptp->shift = order_base_2(size);
0090         slot = nvkm_memory_size(ptp->pt->memory) >> ptp->shift;
0091         ptp->mask = (1 << slot) - 1;
0092         ptp->free = ptp->mask;
0093         list_add(&ptp->head, &mmu->ptp.list);
0094     }
0095     pt->ptp = ptp;
0096     pt->sub = true;
0097
0098     /* Sub-allocate from parent object, removing PTP from cache
0099      * if there's no more free slots left.
0100      */
0101     slot = __ffs(ptp->free);
0102     ptp->free &= ~BIT(slot);
0103     if (!ptp->free)
0104         list_del(&ptp->head);
0105
0106     pt->memory = pt->ptp->pt->memory;
0107     pt->base = slot << ptp->shift;
0108     pt->addr = pt->ptp->pt->addr + pt->base;
0109     return pt;
0110 }
0111
0112 struct nvkm_mmu_ptc {
0113     struct list_head head;
0114     struct list_head item;
0115     u32 size;
0116     u32 refs;
0117 };
0118
0119 static inline struct nvkm_mmu_ptc *
0120 nvkm_mmu_ptc_find(struct nvkm_mmu *mmu, u32 size)
0121 {
0122     struct nvkm_mmu_ptc *ptc;
0123
0124     list_for_each_entry(ptc, &mmu->ptc.list, head) {
0125         if (ptc->size == size)
0126             return ptc;
0127     }
0128
0129     ptc = kmalloc(sizeof(*ptc), GFP_KERNEL);
0130     if (ptc) {
0131         INIT_LIST_HEAD(&ptc->item);
0132         ptc->size = size;
0133         ptc->refs = 0;
0134         list_add(&ptc->head, &mmu->ptc.list);
0135     }
0136
0137     return ptc;
0138 }
0139
0140 void
0141 nvkm_mmu_ptc_put(struct nvkm_mmu *mmu, bool force, struct nvkm_mmu_pt **ppt)
0142 {
0143     struct nvkm_mmu_pt *pt = *ppt;
0144     if (pt) {
0145         /* Handle sub-allocated page tables. */
0146         if (pt->sub) {
0147             mutex_lock(&mmu->ptp.mutex);
0148             nvkm_mmu_ptp_put(mmu, force, pt);
0149             mutex_unlock(&mmu->ptp.mutex);
0150             return;
0151         }
0152
0153         /* Either cache or free the object. */
0154         mutex_lock(&mmu->ptc.mutex);
0155         if (pt->ptc->refs < 8 /* Heuristic. */ && !force) {
0156             list_add_tail(&pt->head, &pt->ptc->item);
0157             pt->ptc->refs++;
0158         } else {
0159             nvkm_memory_unref(&pt->memory);
0160             kfree(pt);
0161         }
0162         mutex_unlock(&mmu->ptc.mutex);
0163     }
0164 }
0165
0166 struct nvkm_mmu_pt *
0167 nvkm_mmu_ptc_get(struct nvkm_mmu *mmu, u32 size, u32 align, bool zero)
0168 {
0169     struct nvkm_mmu_ptc *ptc;
0170     struct nvkm_mmu_pt *pt;
0171     int ret;
0172
0173     /* Sub-allocated page table (ie. GP100 LPT). */
0174     if (align < 0x1000) {
0175         mutex_lock(&mmu->ptp.mutex);
0176         pt = nvkm_mmu_ptp_get(mmu, align, zero);
0177         mutex_unlock(&mmu->ptp.mutex);
0178         return pt;
0179     }
0180
0181     /* Lookup cache for this page table size. */
0182     mutex_lock(&mmu->ptc.mutex);
0183     ptc = nvkm_mmu_ptc_find(mmu, size);
0184     if (!ptc) {
0185         mutex_unlock(&mmu->ptc.mutex);
0186         return NULL;
0187     }
0188
0189     /* If there's a free PT in the cache, reuse it. */
0190     pt = list_first_entry_or_null(&ptc->item, typeof(*pt), head);
0191     if (pt) {
0192         if (zero)
0193             nvkm_fo64(pt->memory, 0, 0, size >> 3);
0194         list_del(&pt->head);
0195         ptc->refs--;
0196         mutex_unlock(&mmu->ptc.mutex);
0197         return pt;
0198     }
0199     mutex_unlock(&mmu->ptc.mutex);
0200
0201     /* No such luck, we need to allocate. */
0202     if (!(pt = kmalloc(sizeof(*pt), GFP_KERNEL)))
0203         return NULL;
0204     pt->ptc = ptc;
0205     pt->sub = false;
0206
0207     ret = nvkm_memory_new(mmu->subdev.device, NVKM_MEM_TARGET_INST,
0208                   size, align, zero, &pt->memory);
0209     if (ret) {
0210         kfree(pt);
0211         return NULL;
0212     }
0213
0214     pt->base = 0;
0215     pt->addr = nvkm_memory_addr(pt->memory);
0216     return pt;
0217 }
0218
0219 void
0220 nvkm_mmu_ptc_dump(struct nvkm_mmu *mmu)
0221 {
0222     struct nvkm_mmu_ptc *ptc;
0223     list_for_each_entry(ptc, &mmu->ptc.list, head) {
0224         struct nvkm_mmu_pt *pt, *tt;
0225         list_for_each_entry_safe(pt, tt, &ptc->item, head) {
0226             nvkm_memory_unref(&pt->memory);
0227             list_del(&pt->head);
0228             kfree(pt);
0229         }
0230     }
0231 }
0232
0233 static void
0234 nvkm_mmu_ptc_fini(struct nvkm_mmu *mmu)
0235 {
0236     struct nvkm_mmu_ptc *ptc, *ptct;
0237
0238     list_for_each_entry_safe(ptc, ptct, &mmu->ptc.list, head) {
0239         WARN_ON(!list_empty(&ptc->item));
0240         list_del(&ptc->head);
0241         kfree(ptc);
0242     }
0243 }
0244
0245 static void
0246 nvkm_mmu_ptc_init(struct nvkm_mmu *mmu)
0247 {
0248     mutex_init(&mmu->ptc.mutex);
0249     INIT_LIST_HEAD(&mmu->ptc.list);
0250     mutex_init(&mmu->ptp.mutex);
0251     INIT_LIST_HEAD(&mmu->ptp.list);
0252 }
0253
0254 static void
0255 nvkm_mmu_type(struct nvkm_mmu *mmu, int heap, u8 type)
0256 {
0257     if (heap >= 0 && !WARN_ON(mmu->type_nr == ARRAY_SIZE(mmu->type))) {
0258         mmu->type[mmu->type_nr].type = type | mmu->heap[heap].type;
0259         mmu->type[mmu->type_nr].heap = heap;
0260         mmu->type_nr++;
0261     }
0262 }
0263
0264 static int
0265 nvkm_mmu_heap(struct nvkm_mmu *mmu, u8 type, u64 size)
0266 {
0267     if (size) {
0268         if (!WARN_ON(mmu->heap_nr == ARRAY_SIZE(mmu->heap))) {
0269             mmu->heap[mmu->heap_nr].type = type;
0270             mmu->heap[mmu->heap_nr].size = size;
0271             return mmu->heap_nr++;
0272         }
0273     }
0274     return -EINVAL;
0275 }
0276
0277 static void
0278 nvkm_mmu_host(struct nvkm_mmu *mmu)
0279 {
0280     struct nvkm_device *device = mmu->subdev.device;
0281     u8 type = NVKM_MEM_KIND * !!mmu->func->kind_sys;
0282     int heap;
0283
0284     /* Non-mappable system memory. */
0285     heap = nvkm_mmu_heap(mmu, NVKM_MEM_HOST, ~0ULL);
0286     nvkm_mmu_type(mmu, heap, type);
0287
0288     /* Non-coherent, cached, system memory.
0289      *
0290      * Block-linear mappings of system memory must be done through
0291      * BAR1, and cannot be supported on systems where we're unable
0292      * to map BAR1 with write-combining.
0293      */
0294     type |= NVKM_MEM_MAPPABLE;
0295     if (!device->bar || device->bar->iomap_uncached)
0296         nvkm_mmu_type(mmu, heap, type & ~NVKM_MEM_KIND);
0297     else
0298         nvkm_mmu_type(mmu, heap, type);
0299
0300     /* Coherent, cached, system memory.
0301      *
0302      * Unsupported on systems that aren't able to support snooped
0303      * mappings, and also for block-linear mappings which must be
0304      * done through BAR1.
0305      */
0306     type |= NVKM_MEM_COHERENT;
0307     if (device->func->cpu_coherent)
0308         nvkm_mmu_type(mmu, heap, type & ~NVKM_MEM_KIND);
0309
0310     /* Uncached system memory. */
0311     nvkm_mmu_type(mmu, heap, type |= NVKM_MEM_UNCACHED);
0312 }
0313
0314 static void
0315 nvkm_mmu_vram(struct nvkm_mmu *mmu)
0316 {
0317     struct nvkm_device *device = mmu->subdev.device;
0318     struct nvkm_mm *mm = &device->fb->ram->vram;
0319     const u64 sizeN = nvkm_mm_heap_size(mm, NVKM_RAM_MM_NORMAL);
0320     const u64 sizeU = nvkm_mm_heap_size(mm, NVKM_RAM_MM_NOMAP);
0321     const u64 sizeM = nvkm_mm_heap_size(mm, NVKM_RAM_MM_MIXED);
0322     u8 type = NVKM_MEM_KIND * !!mmu->func->kind;
0323     u8 heap = NVKM_MEM_VRAM;
0324     int heapM, heapN, heapU;
0325
0326     /* Mixed-memory doesn't support compression or display. */
0327     heapM = nvkm_mmu_heap(mmu, heap, sizeM << NVKM_RAM_MM_SHIFT);
0328
0329     heap |= NVKM_MEM_COMP;
0330     heap |= NVKM_MEM_DISP;
0331     heapN = nvkm_mmu_heap(mmu, heap, sizeN << NVKM_RAM_MM_SHIFT);
0332     heapU = nvkm_mmu_heap(mmu, heap, sizeU << NVKM_RAM_MM_SHIFT);
0333
0334     /* Add non-mappable VRAM types first so that they're preferred
0335      * over anything else.  Mixed-memory will be slower than other
0336      * heaps, it's prioritised last.
0337      */
0338     nvkm_mmu_type(mmu, heapU, type);
0339     nvkm_mmu_type(mmu, heapN, type);
0340     nvkm_mmu_type(mmu, heapM, type);
0341
0342     /* Add host memory types next, under the assumption that users
0343      * wanting mappable memory want to use them as staging buffers
0344      * or the like.
0345      */
0346     nvkm_mmu_host(mmu);
0347
0348     /* Mappable VRAM types go last, as they're basically the worst
0349      * possible type to ask for unless there's no other choice.
0350      */
0351     if (device->bar) {
0352         /* Write-combined BAR1 access. */
0353         type |= NVKM_MEM_MAPPABLE;
0354         if (!device->bar->iomap_uncached) {
0355             nvkm_mmu_type(mmu, heapN, type);
0356             nvkm_mmu_type(mmu, heapM, type);
0357         }
0358
0359         /* Uncached BAR1 access. */
0360         type |= NVKM_MEM_COHERENT;
0361         type |= NVKM_MEM_UNCACHED;
0362         nvkm_mmu_type(mmu, heapN, type);
0363         nvkm_mmu_type(mmu, heapM, type);
0364     }
0365 }
0366
0367 static int
0368 nvkm_mmu_oneinit(struct nvkm_subdev *subdev)
0369 {
0370     struct nvkm_mmu *mmu = nvkm_mmu(subdev);
0371
0372     /* Determine available memory types. */
0373     if (mmu->subdev.device->fb && mmu->subdev.device->fb->ram)
0374         nvkm_mmu_vram(mmu);
0375     else
0376         nvkm_mmu_host(mmu);
0377
0378     if (mmu->func->vmm.global) {
0379         int ret = nvkm_vmm_new(subdev->device, 0, 0, NULL, 0, NULL,
0380                        "gart", &mmu->vmm);
0381         if (ret)
0382             return ret;
0383     }
0384
0385     return 0;
0386 }
0387
0388 static int
0389 nvkm_mmu_init(struct nvkm_subdev *subdev)
0390 {
0391     struct nvkm_mmu *mmu = nvkm_mmu(subdev);
0392     if (mmu->func->init)
0393         mmu->func->init(mmu);
0394     return 0;
0395 }
0396
0397 static void *
0398 nvkm_mmu_dtor(struct nvkm_subdev *subdev)
0399 {
0400     struct nvkm_mmu *mmu = nvkm_mmu(subdev);
0401
0402     nvkm_vmm_unref(&mmu->vmm);
0403
0404     nvkm_mmu_ptc_fini(mmu);
0405     mutex_destroy(&mmu->mutex);
0406     return mmu;
0407 }
0408
0409 static const struct nvkm_subdev_func
0410 nvkm_mmu = {
0411     .dtor = nvkm_mmu_dtor,
0412     .oneinit = nvkm_mmu_oneinit,
0413     .init = nvkm_mmu_init,
0414 };
0415
0416 void
0417 nvkm_mmu_ctor(const struct nvkm_mmu_func *func, struct nvkm_device *device,
0418           enum nvkm_subdev_type type, int inst, struct nvkm_mmu *mmu)
0419 {
0420     nvkm_subdev_ctor(&nvkm_mmu, device, type, inst, &mmu->subdev);
0421     mmu->func = func;
0422     mmu->dma_bits = func->dma_bits;
0423     nvkm_mmu_ptc_init(mmu);
0424     mutex_init(&mmu->mutex);
0425     mmu->user.ctor = nvkm_ummu_new;
0426     mmu->user.base = func->mmu.user;
0427 }
0428
0429 int
0430 nvkm_mmu_new_(const struct nvkm_mmu_func *func, struct nvkm_device *device,
0431           enum nvkm_subdev_type type, int inst, struct nvkm_mmu **pmmu)
0432 {
0433     if (!(*pmmu = kzalloc(sizeof(**pmmu), GFP_KERNEL)))
0434         return -ENOMEM;
0435     nvkm_mmu_ctor(func, device, type, inst, *pmmu);
0436     return 0;
0437 }