Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Tegra host1x Job
0004  *
0005  * Copyright (c) 2010-2015, NVIDIA Corporation.
0006  */
0007 
0008 #include <linux/dma-mapping.h>
0009 #include <linux/err.h>
0010 #include <linux/host1x.h>
0011 #include <linux/iommu.h>
0012 #include <linux/kref.h>
0013 #include <linux/module.h>
0014 #include <linux/scatterlist.h>
0015 #include <linux/slab.h>
0016 #include <linux/vmalloc.h>
0017 #include <trace/events/host1x.h>
0018 
0019 #include "channel.h"
0020 #include "dev.h"
0021 #include "job.h"
0022 #include "syncpt.h"
0023 
0024 #define HOST1X_WAIT_SYNCPT_OFFSET 0x8
0025 
0026 struct host1x_job *host1x_job_alloc(struct host1x_channel *ch,
0027                     u32 num_cmdbufs, u32 num_relocs,
0028                     bool skip_firewall)
0029 {
0030     struct host1x_job *job = NULL;
0031     unsigned int num_unpins = num_relocs;
0032     bool enable_firewall;
0033     u64 total;
0034     void *mem;
0035 
0036     enable_firewall = IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL) && !skip_firewall;
0037 
0038     if (!enable_firewall)
0039         num_unpins += num_cmdbufs;
0040 
0041     /* Check that we're not going to overflow */
0042     total = sizeof(struct host1x_job) +
0043         (u64)num_relocs * sizeof(struct host1x_reloc) +
0044         (u64)num_unpins * sizeof(struct host1x_job_unpin_data) +
0045         (u64)num_cmdbufs * sizeof(struct host1x_job_cmd) +
0046         (u64)num_unpins * sizeof(dma_addr_t) +
0047         (u64)num_unpins * sizeof(u32 *);
0048     if (total > ULONG_MAX)
0049         return NULL;
0050 
0051     mem = job = kzalloc(total, GFP_KERNEL);
0052     if (!job)
0053         return NULL;
0054 
0055     job->enable_firewall = enable_firewall;
0056 
0057     kref_init(&job->ref);
0058     job->channel = ch;
0059 
0060     /* Redistribute memory to the structs  */
0061     mem += sizeof(struct host1x_job);
0062     job->relocs = num_relocs ? mem : NULL;
0063     mem += num_relocs * sizeof(struct host1x_reloc);
0064     job->unpins = num_unpins ? mem : NULL;
0065     mem += num_unpins * sizeof(struct host1x_job_unpin_data);
0066     job->cmds = num_cmdbufs ? mem : NULL;
0067     mem += num_cmdbufs * sizeof(struct host1x_job_cmd);
0068     job->addr_phys = num_unpins ? mem : NULL;
0069 
0070     job->reloc_addr_phys = job->addr_phys;
0071     job->gather_addr_phys = &job->addr_phys[num_relocs];
0072 
0073     return job;
0074 }
0075 EXPORT_SYMBOL(host1x_job_alloc);
0076 
0077 struct host1x_job *host1x_job_get(struct host1x_job *job)
0078 {
0079     kref_get(&job->ref);
0080     return job;
0081 }
0082 EXPORT_SYMBOL(host1x_job_get);
0083 
0084 static void job_free(struct kref *ref)
0085 {
0086     struct host1x_job *job = container_of(ref, struct host1x_job, ref);
0087 
0088     if (job->release)
0089         job->release(job);
0090 
0091     if (job->waiter)
0092         host1x_intr_put_ref(job->syncpt->host, job->syncpt->id,
0093                     job->waiter, false);
0094 
0095     if (job->syncpt)
0096         host1x_syncpt_put(job->syncpt);
0097 
0098     kfree(job);
0099 }
0100 
0101 void host1x_job_put(struct host1x_job *job)
0102 {
0103     kref_put(&job->ref, job_free);
0104 }
0105 EXPORT_SYMBOL(host1x_job_put);
0106 
0107 void host1x_job_add_gather(struct host1x_job *job, struct host1x_bo *bo,
0108                unsigned int words, unsigned int offset)
0109 {
0110     struct host1x_job_gather *gather = &job->cmds[job->num_cmds].gather;
0111 
0112     gather->words = words;
0113     gather->bo = bo;
0114     gather->offset = offset;
0115 
0116     job->num_cmds++;
0117 }
0118 EXPORT_SYMBOL(host1x_job_add_gather);
0119 
0120 void host1x_job_add_wait(struct host1x_job *job, u32 id, u32 thresh,
0121              bool relative, u32 next_class)
0122 {
0123     struct host1x_job_cmd *cmd = &job->cmds[job->num_cmds];
0124 
0125     cmd->is_wait = true;
0126     cmd->wait.id = id;
0127     cmd->wait.threshold = thresh;
0128     cmd->wait.next_class = next_class;
0129     cmd->wait.relative = relative;
0130 
0131     job->num_cmds++;
0132 }
0133 EXPORT_SYMBOL(host1x_job_add_wait);
0134 
0135 static unsigned int pin_job(struct host1x *host, struct host1x_job *job)
0136 {
0137     unsigned long mask = HOST1X_RELOC_READ | HOST1X_RELOC_WRITE;
0138     struct host1x_client *client = job->client;
0139     struct device *dev = client->dev;
0140     struct host1x_job_gather *g;
0141     unsigned int i;
0142     int err;
0143 
0144     job->num_unpins = 0;
0145 
0146     for (i = 0; i < job->num_relocs; i++) {
0147         struct host1x_reloc *reloc = &job->relocs[i];
0148         enum dma_data_direction direction;
0149         struct host1x_bo_mapping *map;
0150         struct host1x_bo *bo;
0151 
0152         reloc->target.bo = host1x_bo_get(reloc->target.bo);
0153         if (!reloc->target.bo) {
0154             err = -EINVAL;
0155             goto unpin;
0156         }
0157 
0158         bo = reloc->target.bo;
0159 
0160         switch (reloc->flags & mask) {
0161         case HOST1X_RELOC_READ:
0162             direction = DMA_TO_DEVICE;
0163             break;
0164 
0165         case HOST1X_RELOC_WRITE:
0166             direction = DMA_FROM_DEVICE;
0167             break;
0168 
0169         case HOST1X_RELOC_READ | HOST1X_RELOC_WRITE:
0170             direction = DMA_BIDIRECTIONAL;
0171             break;
0172 
0173         default:
0174             err = -EINVAL;
0175             goto unpin;
0176         }
0177 
0178         map = host1x_bo_pin(dev, bo, direction, NULL);
0179         if (IS_ERR(map)) {
0180             err = PTR_ERR(map);
0181             goto unpin;
0182         }
0183 
0184         /*
0185          * host1x clients are generally not able to do scatter-gather themselves, so fail
0186          * if the buffer is discontiguous and we fail to map its SG table to a single
0187          * contiguous chunk of I/O virtual memory.
0188          */
0189         if (map->chunks > 1) {
0190             err = -EINVAL;
0191             goto unpin;
0192         }
0193 
0194         job->addr_phys[job->num_unpins] = map->phys;
0195         job->unpins[job->num_unpins].map = map;
0196         job->num_unpins++;
0197     }
0198 
0199     /*
0200      * We will copy gathers BO content later, so there is no need to
0201      * hold and pin them.
0202      */
0203     if (job->enable_firewall)
0204         return 0;
0205 
0206     for (i = 0; i < job->num_cmds; i++) {
0207         struct host1x_bo_mapping *map;
0208         size_t gather_size = 0;
0209         struct scatterlist *sg;
0210         unsigned long shift;
0211         struct iova *alloc;
0212         unsigned int j;
0213 
0214         if (job->cmds[i].is_wait)
0215             continue;
0216 
0217         g = &job->cmds[i].gather;
0218 
0219         g->bo = host1x_bo_get(g->bo);
0220         if (!g->bo) {
0221             err = -EINVAL;
0222             goto unpin;
0223         }
0224 
0225         map = host1x_bo_pin(host->dev, g->bo, DMA_TO_DEVICE, NULL);
0226         if (IS_ERR(map)) {
0227             err = PTR_ERR(map);
0228             goto unpin;
0229         }
0230 
0231         if (host->domain) {
0232             for_each_sgtable_sg(map->sgt, sg, j)
0233                 gather_size += sg->length;
0234 
0235             gather_size = iova_align(&host->iova, gather_size);
0236 
0237             shift = iova_shift(&host->iova);
0238             alloc = alloc_iova(&host->iova, gather_size >> shift,
0239                        host->iova_end >> shift, true);
0240             if (!alloc) {
0241                 err = -ENOMEM;
0242                 goto put;
0243             }
0244 
0245             err = iommu_map_sgtable(host->domain, iova_dma_addr(&host->iova, alloc),
0246                         map->sgt, IOMMU_READ);
0247             if (err == 0) {
0248                 __free_iova(&host->iova, alloc);
0249                 err = -EINVAL;
0250                 goto put;
0251             }
0252 
0253             map->phys = iova_dma_addr(&host->iova, alloc);
0254             map->size = gather_size;
0255         }
0256 
0257         job->addr_phys[job->num_unpins] = map->phys;
0258         job->unpins[job->num_unpins].map = map;
0259         job->num_unpins++;
0260 
0261         job->gather_addr_phys[i] = map->phys;
0262     }
0263 
0264     return 0;
0265 
0266 put:
0267     host1x_bo_put(g->bo);
0268 unpin:
0269     host1x_job_unpin(job);
0270     return err;
0271 }
0272 
0273 static int do_relocs(struct host1x_job *job, struct host1x_job_gather *g)
0274 {
0275     void *cmdbuf_addr = NULL;
0276     struct host1x_bo *cmdbuf = g->bo;
0277     unsigned int i;
0278 
0279     /* pin & patch the relocs for one gather */
0280     for (i = 0; i < job->num_relocs; i++) {
0281         struct host1x_reloc *reloc = &job->relocs[i];
0282         u32 reloc_addr = (job->reloc_addr_phys[i] +
0283                   reloc->target.offset) >> reloc->shift;
0284         u32 *target;
0285 
0286         /* skip all other gathers */
0287         if (cmdbuf != reloc->cmdbuf.bo)
0288             continue;
0289 
0290         if (job->enable_firewall) {
0291             target = (u32 *)job->gather_copy_mapped +
0292                     reloc->cmdbuf.offset / sizeof(u32) +
0293                         g->offset / sizeof(u32);
0294             goto patch_reloc;
0295         }
0296 
0297         if (!cmdbuf_addr) {
0298             cmdbuf_addr = host1x_bo_mmap(cmdbuf);
0299 
0300             if (unlikely(!cmdbuf_addr)) {
0301                 pr_err("Could not map cmdbuf for relocation\n");
0302                 return -ENOMEM;
0303             }
0304         }
0305 
0306         target = cmdbuf_addr + reloc->cmdbuf.offset;
0307 patch_reloc:
0308         *target = reloc_addr;
0309     }
0310 
0311     if (cmdbuf_addr)
0312         host1x_bo_munmap(cmdbuf, cmdbuf_addr);
0313 
0314     return 0;
0315 }
0316 
0317 static bool check_reloc(struct host1x_reloc *reloc, struct host1x_bo *cmdbuf,
0318             unsigned int offset)
0319 {
0320     offset *= sizeof(u32);
0321 
0322     if (reloc->cmdbuf.bo != cmdbuf || reloc->cmdbuf.offset != offset)
0323         return false;
0324 
0325     /* relocation shift value validation isn't implemented yet */
0326     if (reloc->shift)
0327         return false;
0328 
0329     return true;
0330 }
0331 
0332 struct host1x_firewall {
0333     struct host1x_job *job;
0334     struct device *dev;
0335 
0336     unsigned int num_relocs;
0337     struct host1x_reloc *reloc;
0338 
0339     struct host1x_bo *cmdbuf;
0340     unsigned int offset;
0341 
0342     u32 words;
0343     u32 class;
0344     u32 reg;
0345     u32 mask;
0346     u32 count;
0347 };
0348 
0349 static int check_register(struct host1x_firewall *fw, unsigned long offset)
0350 {
0351     if (!fw->job->is_addr_reg)
0352         return 0;
0353 
0354     if (fw->job->is_addr_reg(fw->dev, fw->class, offset)) {
0355         if (!fw->num_relocs)
0356             return -EINVAL;
0357 
0358         if (!check_reloc(fw->reloc, fw->cmdbuf, fw->offset))
0359             return -EINVAL;
0360 
0361         fw->num_relocs--;
0362         fw->reloc++;
0363     }
0364 
0365     return 0;
0366 }
0367 
0368 static int check_class(struct host1x_firewall *fw, u32 class)
0369 {
0370     if (!fw->job->is_valid_class) {
0371         if (fw->class != class)
0372             return -EINVAL;
0373     } else {
0374         if (!fw->job->is_valid_class(fw->class))
0375             return -EINVAL;
0376     }
0377 
0378     return 0;
0379 }
0380 
0381 static int check_mask(struct host1x_firewall *fw)
0382 {
0383     u32 mask = fw->mask;
0384     u32 reg = fw->reg;
0385     int ret;
0386 
0387     while (mask) {
0388         if (fw->words == 0)
0389             return -EINVAL;
0390 
0391         if (mask & 1) {
0392             ret = check_register(fw, reg);
0393             if (ret < 0)
0394                 return ret;
0395 
0396             fw->words--;
0397             fw->offset++;
0398         }
0399         mask >>= 1;
0400         reg++;
0401     }
0402 
0403     return 0;
0404 }
0405 
0406 static int check_incr(struct host1x_firewall *fw)
0407 {
0408     u32 count = fw->count;
0409     u32 reg = fw->reg;
0410     int ret;
0411 
0412     while (count) {
0413         if (fw->words == 0)
0414             return -EINVAL;
0415 
0416         ret = check_register(fw, reg);
0417         if (ret < 0)
0418             return ret;
0419 
0420         reg++;
0421         fw->words--;
0422         fw->offset++;
0423         count--;
0424     }
0425 
0426     return 0;
0427 }
0428 
0429 static int check_nonincr(struct host1x_firewall *fw)
0430 {
0431     u32 count = fw->count;
0432     int ret;
0433 
0434     while (count) {
0435         if (fw->words == 0)
0436             return -EINVAL;
0437 
0438         ret = check_register(fw, fw->reg);
0439         if (ret < 0)
0440             return ret;
0441 
0442         fw->words--;
0443         fw->offset++;
0444         count--;
0445     }
0446 
0447     return 0;
0448 }
0449 
0450 static int validate(struct host1x_firewall *fw, struct host1x_job_gather *g)
0451 {
0452     u32 *cmdbuf_base = (u32 *)fw->job->gather_copy_mapped +
0453         (g->offset / sizeof(u32));
0454     u32 job_class = fw->class;
0455     int err = 0;
0456 
0457     fw->words = g->words;
0458     fw->cmdbuf = g->bo;
0459     fw->offset = 0;
0460 
0461     while (fw->words && !err) {
0462         u32 word = cmdbuf_base[fw->offset];
0463         u32 opcode = (word & 0xf0000000) >> 28;
0464 
0465         fw->mask = 0;
0466         fw->reg = 0;
0467         fw->count = 0;
0468         fw->words--;
0469         fw->offset++;
0470 
0471         switch (opcode) {
0472         case 0:
0473             fw->class = word >> 6 & 0x3ff;
0474             fw->mask = word & 0x3f;
0475             fw->reg = word >> 16 & 0xfff;
0476             err = check_class(fw, job_class);
0477             if (!err)
0478                 err = check_mask(fw);
0479             if (err)
0480                 goto out;
0481             break;
0482         case 1:
0483             fw->reg = word >> 16 & 0xfff;
0484             fw->count = word & 0xffff;
0485             err = check_incr(fw);
0486             if (err)
0487                 goto out;
0488             break;
0489 
0490         case 2:
0491             fw->reg = word >> 16 & 0xfff;
0492             fw->count = word & 0xffff;
0493             err = check_nonincr(fw);
0494             if (err)
0495                 goto out;
0496             break;
0497 
0498         case 3:
0499             fw->mask = word & 0xffff;
0500             fw->reg = word >> 16 & 0xfff;
0501             err = check_mask(fw);
0502             if (err)
0503                 goto out;
0504             break;
0505         case 4:
0506         case 14:
0507             break;
0508         default:
0509             err = -EINVAL;
0510             break;
0511         }
0512     }
0513 
0514 out:
0515     return err;
0516 }
0517 
0518 static inline int copy_gathers(struct device *host, struct host1x_job *job,
0519                    struct device *dev)
0520 {
0521     struct host1x_firewall fw;
0522     size_t size = 0;
0523     size_t offset = 0;
0524     unsigned int i;
0525 
0526     fw.job = job;
0527     fw.dev = dev;
0528     fw.reloc = job->relocs;
0529     fw.num_relocs = job->num_relocs;
0530     fw.class = job->class;
0531 
0532     for (i = 0; i < job->num_cmds; i++) {
0533         struct host1x_job_gather *g;
0534 
0535         if (job->cmds[i].is_wait)
0536             continue;
0537 
0538         g = &job->cmds[i].gather;
0539 
0540         size += g->words * sizeof(u32);
0541     }
0542 
0543     /*
0544      * Try a non-blocking allocation from a higher priority pools first,
0545      * as awaiting for the allocation here is a major performance hit.
0546      */
0547     job->gather_copy_mapped = dma_alloc_wc(host, size, &job->gather_copy,
0548                            GFP_NOWAIT);
0549 
0550     /* the higher priority allocation failed, try the generic-blocking */
0551     if (!job->gather_copy_mapped)
0552         job->gather_copy_mapped = dma_alloc_wc(host, size,
0553                                &job->gather_copy,
0554                                GFP_KERNEL);
0555     if (!job->gather_copy_mapped)
0556         return -ENOMEM;
0557 
0558     job->gather_copy_size = size;
0559 
0560     for (i = 0; i < job->num_cmds; i++) {
0561         struct host1x_job_gather *g;
0562         void *gather;
0563 
0564         if (job->cmds[i].is_wait)
0565             continue;
0566         g = &job->cmds[i].gather;
0567 
0568         /* Copy the gather */
0569         gather = host1x_bo_mmap(g->bo);
0570         memcpy(job->gather_copy_mapped + offset, gather + g->offset,
0571                g->words * sizeof(u32));
0572         host1x_bo_munmap(g->bo, gather);
0573 
0574         /* Store the location in the buffer */
0575         g->base = job->gather_copy;
0576         g->offset = offset;
0577 
0578         /* Validate the job */
0579         if (validate(&fw, g))
0580             return -EINVAL;
0581 
0582         offset += g->words * sizeof(u32);
0583     }
0584 
0585     /* No relocs should remain at this point */
0586     if (fw.num_relocs)
0587         return -EINVAL;
0588 
0589     return 0;
0590 }
0591 
0592 int host1x_job_pin(struct host1x_job *job, struct device *dev)
0593 {
0594     int err;
0595     unsigned int i, j;
0596     struct host1x *host = dev_get_drvdata(dev->parent);
0597 
0598     /* pin memory */
0599     err = pin_job(host, job);
0600     if (err)
0601         goto out;
0602 
0603     if (job->enable_firewall) {
0604         err = copy_gathers(host->dev, job, dev);
0605         if (err)
0606             goto out;
0607     }
0608 
0609     /* patch gathers */
0610     for (i = 0; i < job->num_cmds; i++) {
0611         struct host1x_job_gather *g;
0612 
0613         if (job->cmds[i].is_wait)
0614             continue;
0615         g = &job->cmds[i].gather;
0616 
0617         /* process each gather mem only once */
0618         if (g->handled)
0619             continue;
0620 
0621         /* copy_gathers() sets gathers base if firewall is enabled */
0622         if (!job->enable_firewall)
0623             g->base = job->gather_addr_phys[i];
0624 
0625         for (j = i + 1; j < job->num_cmds; j++) {
0626             if (!job->cmds[j].is_wait &&
0627                 job->cmds[j].gather.bo == g->bo) {
0628                 job->cmds[j].gather.handled = true;
0629                 job->cmds[j].gather.base = g->base;
0630             }
0631         }
0632 
0633         err = do_relocs(job, g);
0634         if (err)
0635             break;
0636     }
0637 
0638 out:
0639     if (err)
0640         host1x_job_unpin(job);
0641     wmb();
0642 
0643     return err;
0644 }
0645 EXPORT_SYMBOL(host1x_job_pin);
0646 
0647 void host1x_job_unpin(struct host1x_job *job)
0648 {
0649     struct host1x *host = dev_get_drvdata(job->channel->dev->parent);
0650     unsigned int i;
0651 
0652     for (i = 0; i < job->num_unpins; i++) {
0653         struct host1x_bo_mapping *map = job->unpins[i].map;
0654         struct host1x_bo *bo = map->bo;
0655 
0656         if (!job->enable_firewall && map->size && host->domain) {
0657             iommu_unmap(host->domain, job->addr_phys[i], map->size);
0658             free_iova(&host->iova, iova_pfn(&host->iova, job->addr_phys[i]));
0659         }
0660 
0661         host1x_bo_unpin(map);
0662         host1x_bo_put(bo);
0663     }
0664 
0665     job->num_unpins = 0;
0666 
0667     if (job->gather_copy_size)
0668         dma_free_wc(host->dev, job->gather_copy_size,
0669                 job->gather_copy_mapped, job->gather_copy);
0670 }
0671 EXPORT_SYMBOL(host1x_job_unpin);
0672 
0673 /*
0674  * Debug routine used to dump job entries
0675  */
0676 void host1x_job_dump(struct device *dev, struct host1x_job *job)
0677 {
0678     dev_dbg(dev, "    SYNCPT_ID   %d\n", job->syncpt->id);
0679     dev_dbg(dev, "    SYNCPT_VAL  %d\n", job->syncpt_end);
0680     dev_dbg(dev, "    FIRST_GET   0x%x\n", job->first_get);
0681     dev_dbg(dev, "    TIMEOUT     %d\n", job->timeout);
0682     dev_dbg(dev, "    NUM_SLOTS   %d\n", job->num_slots);
0683     dev_dbg(dev, "    NUM_HANDLES %d\n", job->num_unpins);
0684 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!