drm/msm/msm_iommu.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Copyright (C) 2013 Red Hat
0004  * Author: Rob Clark <robdclark@gmail.com>
0005  */
0006
0007 #include <linux/adreno-smmu-priv.h>
0008 #include <linux/io-pgtable.h>
0009 #include "msm_drv.h"
0010 #include "msm_mmu.h"
0011
0012 struct msm_iommu {
0013     struct msm_mmu base;
0014     struct iommu_domain *domain;
0015     atomic_t pagetables;
0016 };
0017
0018 #define to_msm_iommu(x) container_of(x, struct msm_iommu, base)
0019
0020 struct msm_iommu_pagetable {
0021     struct msm_mmu base;
0022     struct msm_mmu *parent;
0023     struct io_pgtable_ops *pgtbl_ops;
0024     phys_addr_t ttbr;
0025     u32 asid;
0026 };
0027 static struct msm_iommu_pagetable *to_pagetable(struct msm_mmu *mmu)
0028 {
0029     return container_of(mmu, struct msm_iommu_pagetable, base);
0030 }
0031
0032 static int msm_iommu_pagetable_unmap(struct msm_mmu *mmu, u64 iova,
0033         size_t size)
0034 {
0035     struct msm_iommu_pagetable *pagetable = to_pagetable(mmu);
0036     struct io_pgtable_ops *ops = pagetable->pgtbl_ops;
0037     size_t unmapped = 0;
0038
0039     /* Unmap the block one page at a time */
0040     while (size) {
0041         unmapped += ops->unmap(ops, iova, 4096, NULL);
0042         iova += 4096;
0043         size -= 4096;
0044     }
0045
0046     iommu_flush_iotlb_all(to_msm_iommu(pagetable->parent)->domain);
0047
0048     return (unmapped == size) ? 0 : -EINVAL;
0049 }
0050
0051 static int msm_iommu_pagetable_map(struct msm_mmu *mmu, u64 iova,
0052         struct sg_table *sgt, size_t len, int prot)
0053 {
0054     struct msm_iommu_pagetable *pagetable = to_pagetable(mmu);
0055     struct io_pgtable_ops *ops = pagetable->pgtbl_ops;
0056     struct scatterlist *sg;
0057     size_t mapped = 0;
0058     u64 addr = iova;
0059     unsigned int i;
0060
0061     for_each_sgtable_sg(sgt, sg, i) {
0062         size_t size = sg->length;
0063         phys_addr_t phys = sg_phys(sg);
0064
0065         /* Map the block one page at a time */
0066         while (size) {
0067             if (ops->map(ops, addr, phys, 4096, prot, GFP_KERNEL)) {
0068                 msm_iommu_pagetable_unmap(mmu, iova, mapped);
0069                 return -EINVAL;
0070             }
0071
0072             phys += 4096;
0073             addr += 4096;
0074             size -= 4096;
0075             mapped += 4096;
0076         }
0077     }
0078
0079     return 0;
0080 }
0081
0082 static void msm_iommu_pagetable_destroy(struct msm_mmu *mmu)
0083 {
0084     struct msm_iommu_pagetable *pagetable = to_pagetable(mmu);
0085     struct msm_iommu *iommu = to_msm_iommu(pagetable->parent);
0086     struct adreno_smmu_priv *adreno_smmu =
0087         dev_get_drvdata(pagetable->parent->dev);
0088
0089     /*
0090      * If this is the last attached pagetable for the parent,
0091      * disable TTBR0 in the arm-smmu driver
0092      */
0093     if (atomic_dec_return(&iommu->pagetables) == 0)
0094         adreno_smmu->set_ttbr0_cfg(adreno_smmu->cookie, NULL);
0095
0096     free_io_pgtable_ops(pagetable->pgtbl_ops);
0097     kfree(pagetable);
0098 }
0099
0100 int msm_iommu_pagetable_params(struct msm_mmu *mmu,
0101         phys_addr_t *ttbr, int *asid)
0102 {
0103     struct msm_iommu_pagetable *pagetable;
0104
0105     if (mmu->type != MSM_MMU_IOMMU_PAGETABLE)
0106         return -EINVAL;
0107
0108     pagetable = to_pagetable(mmu);
0109
0110     if (ttbr)
0111         *ttbr = pagetable->ttbr;
0112
0113     if (asid)
0114         *asid = pagetable->asid;
0115
0116     return 0;
0117 }
0118
0119 static const struct msm_mmu_funcs pagetable_funcs = {
0120         .map = msm_iommu_pagetable_map,
0121         .unmap = msm_iommu_pagetable_unmap,
0122         .destroy = msm_iommu_pagetable_destroy,
0123 };
0124
0125 static void msm_iommu_tlb_flush_all(void *cookie)
0126 {
0127 }
0128
0129 static void msm_iommu_tlb_flush_walk(unsigned long iova, size_t size,
0130         size_t granule, void *cookie)
0131 {
0132 }
0133
0134 static void msm_iommu_tlb_add_page(struct iommu_iotlb_gather *gather,
0135         unsigned long iova, size_t granule, void *cookie)
0136 {
0137 }
0138
0139 static const struct iommu_flush_ops null_tlb_ops = {
0140     .tlb_flush_all = msm_iommu_tlb_flush_all,
0141     .tlb_flush_walk = msm_iommu_tlb_flush_walk,
0142     .tlb_add_page = msm_iommu_tlb_add_page,
0143 };
0144
0145 static int msm_fault_handler(struct iommu_domain *domain, struct device *dev,
0146         unsigned long iova, int flags, void *arg);
0147
0148 struct msm_mmu *msm_iommu_pagetable_create(struct msm_mmu *parent)
0149 {
0150     struct adreno_smmu_priv *adreno_smmu = dev_get_drvdata(parent->dev);
0151     struct msm_iommu *iommu = to_msm_iommu(parent);
0152     struct msm_iommu_pagetable *pagetable;
0153     const struct io_pgtable_cfg *ttbr1_cfg = NULL;
0154     struct io_pgtable_cfg ttbr0_cfg;
0155     int ret;
0156
0157     /* Get the pagetable configuration from the domain */
0158     if (adreno_smmu->cookie)
0159         ttbr1_cfg = adreno_smmu->get_ttbr1_cfg(adreno_smmu->cookie);
0160     if (!ttbr1_cfg)
0161         return ERR_PTR(-ENODEV);
0162
0163     /*
0164      * Defer setting the fault handler until we have a valid adreno_smmu
0165      * to avoid accidentially installing a GPU specific fault handler for
0166      * the display's iommu
0167      */
0168     iommu_set_fault_handler(iommu->domain, msm_fault_handler, iommu);
0169
0170     pagetable = kzalloc(sizeof(*pagetable), GFP_KERNEL);
0171     if (!pagetable)
0172         return ERR_PTR(-ENOMEM);
0173
0174     msm_mmu_init(&pagetable->base, parent->dev, &pagetable_funcs,
0175         MSM_MMU_IOMMU_PAGETABLE);
0176
0177     /* Clone the TTBR1 cfg as starting point for TTBR0 cfg: */
0178     ttbr0_cfg = *ttbr1_cfg;
0179
0180     /* The incoming cfg will have the TTBR1 quirk enabled */
0181     ttbr0_cfg.quirks &= ~IO_PGTABLE_QUIRK_ARM_TTBR1;
0182     ttbr0_cfg.tlb = &null_tlb_ops;
0183
0184     pagetable->pgtbl_ops = alloc_io_pgtable_ops(ARM_64_LPAE_S1,
0185         &ttbr0_cfg, iommu->domain);
0186
0187     if (!pagetable->pgtbl_ops) {
0188         kfree(pagetable);
0189         return ERR_PTR(-ENOMEM);
0190     }
0191
0192     /*
0193      * If this is the first pagetable that we've allocated, send it back to
0194      * the arm-smmu driver as a trigger to set up TTBR0
0195      */
0196     if (atomic_inc_return(&iommu->pagetables) == 1) {
0197         /* Enable stall on iommu fault: */
0198         adreno_smmu->set_stall(adreno_smmu->cookie, true);
0199
0200         ret = adreno_smmu->set_ttbr0_cfg(adreno_smmu->cookie, &ttbr0_cfg);
0201         if (ret) {
0202             free_io_pgtable_ops(pagetable->pgtbl_ops);
0203             kfree(pagetable);
0204             return ERR_PTR(ret);
0205         }
0206     }
0207
0208     /* Needed later for TLB flush */
0209     pagetable->parent = parent;
0210     pagetable->ttbr = ttbr0_cfg.arm_lpae_s1_cfg.ttbr;
0211
0212     /*
0213      * TODO we would like each set of page tables to have a unique ASID
0214      * to optimize TLB invalidation.  But iommu_flush_iotlb_all() will
0215      * end up flushing the ASID used for TTBR1 pagetables, which is not
0216      * what we want.  So for now just use the same ASID as TTBR1.
0217      */
0218     pagetable->asid = 0;
0219
0220     return &pagetable->base;
0221 }
0222
0223 static int msm_fault_handler(struct iommu_domain *domain, struct device *dev,
0224         unsigned long iova, int flags, void *arg)
0225 {
0226     struct msm_iommu *iommu = arg;
0227     struct adreno_smmu_priv *adreno_smmu = dev_get_drvdata(iommu->base.dev);
0228     struct adreno_smmu_fault_info info, *ptr = NULL;
0229
0230     if (adreno_smmu->get_fault_info) {
0231         adreno_smmu->get_fault_info(adreno_smmu->cookie, &info);
0232         ptr = &info;
0233     }
0234
0235     if (iommu->base.handler)
0236         return iommu->base.handler(iommu->base.arg, iova, flags, ptr);
0237
0238     pr_warn_ratelimited("*** fault: iova=%16lx, flags=%d\n", iova, flags);
0239     return 0;
0240 }
0241
0242 static void msm_iommu_resume_translation(struct msm_mmu *mmu)
0243 {
0244     struct adreno_smmu_priv *adreno_smmu = dev_get_drvdata(mmu->dev);
0245
0246     adreno_smmu->resume_translation(adreno_smmu->cookie, true);
0247 }
0248
0249 static void msm_iommu_detach(struct msm_mmu *mmu)
0250 {
0251     struct msm_iommu *iommu = to_msm_iommu(mmu);
0252
0253     iommu_detach_device(iommu->domain, mmu->dev);
0254 }
0255
0256 static int msm_iommu_map(struct msm_mmu *mmu, uint64_t iova,
0257         struct sg_table *sgt, size_t len, int prot)
0258 {
0259     struct msm_iommu *iommu = to_msm_iommu(mmu);
0260     size_t ret;
0261
0262     /* The arm-smmu driver expects the addresses to be sign extended */
0263     if (iova & BIT_ULL(48))
0264         iova |= GENMASK_ULL(63, 49);
0265
0266     ret = iommu_map_sgtable(iommu->domain, iova, sgt, prot);
0267     WARN_ON(!ret);
0268
0269     return (ret == len) ? 0 : -EINVAL;
0270 }
0271
0272 static int msm_iommu_unmap(struct msm_mmu *mmu, uint64_t iova, size_t len)
0273 {
0274     struct msm_iommu *iommu = to_msm_iommu(mmu);
0275
0276     if (iova & BIT_ULL(48))
0277         iova |= GENMASK_ULL(63, 49);
0278
0279     iommu_unmap(iommu->domain, iova, len);
0280
0281     return 0;
0282 }
0283
0284 static void msm_iommu_destroy(struct msm_mmu *mmu)
0285 {
0286     struct msm_iommu *iommu = to_msm_iommu(mmu);
0287     iommu_domain_free(iommu->domain);
0288     kfree(iommu);
0289 }
0290
0291 static const struct msm_mmu_funcs funcs = {
0292         .detach = msm_iommu_detach,
0293         .map = msm_iommu_map,
0294         .unmap = msm_iommu_unmap,
0295         .destroy = msm_iommu_destroy,
0296         .resume_translation = msm_iommu_resume_translation,
0297 };
0298
0299 struct msm_mmu *msm_iommu_new(struct device *dev, struct iommu_domain *domain)
0300 {
0301     struct msm_iommu *iommu;
0302     int ret;
0303
0304     if (!domain)
0305         return ERR_PTR(-ENODEV);
0306
0307     iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
0308     if (!iommu)
0309         return ERR_PTR(-ENOMEM);
0310
0311     iommu->domain = domain;
0312     msm_mmu_init(&iommu->base, dev, &funcs, MSM_MMU_IOMMU);
0313
0314     atomic_set(&iommu->pagetables, 0);
0315
0316     ret = iommu_attach_device(iommu->domain, dev);
0317     if (ret) {
0318         kfree(iommu);
0319         return ERR_PTR(ret);
0320     }
0321
0322     return &iommu->base;
0323 }