arm/arm-smmu/arm-smmu-qcom.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Copyright (c) 2019, The Linux Foundation. All rights reserved.
0004  */
0005
0006 #include <linux/acpi.h>
0007 #include <linux/adreno-smmu-priv.h>
0008 #include <linux/delay.h>
0009 #include <linux/of_device.h>
0010 #include <linux/qcom_scm.h>
0011
0012 #include "arm-smmu.h"
0013 #include "arm-smmu-qcom.h"
0014
0015 #define QCOM_DUMMY_VAL  -1
0016
0017 static struct qcom_smmu *to_qcom_smmu(struct arm_smmu_device *smmu)
0018 {
0019     return container_of(smmu, struct qcom_smmu, smmu);
0020 }
0021
0022 static void qcom_smmu_tlb_sync(struct arm_smmu_device *smmu, int page,
0023                 int sync, int status)
0024 {
0025     unsigned int spin_cnt, delay;
0026     u32 reg;
0027
0028     arm_smmu_writel(smmu, page, sync, QCOM_DUMMY_VAL);
0029     for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) {
0030         for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) {
0031             reg = arm_smmu_readl(smmu, page, status);
0032             if (!(reg & ARM_SMMU_sTLBGSTATUS_GSACTIVE))
0033                 return;
0034             cpu_relax();
0035         }
0036         udelay(delay);
0037     }
0038
0039     qcom_smmu_tlb_sync_debug(smmu);
0040 }
0041
0042 static void qcom_adreno_smmu_write_sctlr(struct arm_smmu_device *smmu, int idx,
0043         u32 reg)
0044 {
0045     struct qcom_smmu *qsmmu = to_qcom_smmu(smmu);
0046
0047     /*
0048      * On the GPU device we want to process subsequent transactions after a
0049      * fault to keep the GPU from hanging
0050      */
0051     reg |= ARM_SMMU_SCTLR_HUPCF;
0052
0053     if (qsmmu->stall_enabled & BIT(idx))
0054         reg |= ARM_SMMU_SCTLR_CFCFG;
0055
0056     arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg);
0057 }
0058
0059 static void qcom_adreno_smmu_get_fault_info(const void *cookie,
0060         struct adreno_smmu_fault_info *info)
0061 {
0062     struct arm_smmu_domain *smmu_domain = (void *)cookie;
0063     struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
0064     struct arm_smmu_device *smmu = smmu_domain->smmu;
0065
0066     info->fsr = arm_smmu_cb_read(smmu, cfg->cbndx, ARM_SMMU_CB_FSR);
0067     info->fsynr0 = arm_smmu_cb_read(smmu, cfg->cbndx, ARM_SMMU_CB_FSYNR0);
0068     info->fsynr1 = arm_smmu_cb_read(smmu, cfg->cbndx, ARM_SMMU_CB_FSYNR1);
0069     info->far = arm_smmu_cb_readq(smmu, cfg->cbndx, ARM_SMMU_CB_FAR);
0070     info->cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(cfg->cbndx));
0071     info->ttbr0 = arm_smmu_cb_readq(smmu, cfg->cbndx, ARM_SMMU_CB_TTBR0);
0072     info->contextidr = arm_smmu_cb_read(smmu, cfg->cbndx, ARM_SMMU_CB_CONTEXTIDR);
0073 }
0074
0075 static void qcom_adreno_smmu_set_stall(const void *cookie, bool enabled)
0076 {
0077     struct arm_smmu_domain *smmu_domain = (void *)cookie;
0078     struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
0079     struct qcom_smmu *qsmmu = to_qcom_smmu(smmu_domain->smmu);
0080
0081     if (enabled)
0082         qsmmu->stall_enabled |= BIT(cfg->cbndx);
0083     else
0084         qsmmu->stall_enabled &= ~BIT(cfg->cbndx);
0085 }
0086
0087 static void qcom_adreno_smmu_resume_translation(const void *cookie, bool terminate)
0088 {
0089     struct arm_smmu_domain *smmu_domain = (void *)cookie;
0090     struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
0091     struct arm_smmu_device *smmu = smmu_domain->smmu;
0092     u32 reg = 0;
0093
0094     if (terminate)
0095         reg |= ARM_SMMU_RESUME_TERMINATE;
0096
0097     arm_smmu_cb_write(smmu, cfg->cbndx, ARM_SMMU_CB_RESUME, reg);
0098 }
0099
0100 #define QCOM_ADRENO_SMMU_GPU_SID 0
0101
0102 static bool qcom_adreno_smmu_is_gpu_device(struct device *dev)
0103 {
0104     struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
0105     int i;
0106
0107     /*
0108      * The GPU will always use SID 0 so that is a handy way to uniquely
0109      * identify it and configure it for per-instance pagetables
0110      */
0111     for (i = 0; i < fwspec->num_ids; i++) {
0112         u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
0113
0114         if (sid == QCOM_ADRENO_SMMU_GPU_SID)
0115             return true;
0116     }
0117
0118     return false;
0119 }
0120
0121 static const struct io_pgtable_cfg *qcom_adreno_smmu_get_ttbr1_cfg(
0122         const void *cookie)
0123 {
0124     struct arm_smmu_domain *smmu_domain = (void *)cookie;
0125     struct io_pgtable *pgtable =
0126         io_pgtable_ops_to_pgtable(smmu_domain->pgtbl_ops);
0127     return &pgtable->cfg;
0128 }
0129
0130 /*
0131  * Local implementation to configure TTBR0 with the specified pagetable config.
0132  * The GPU driver will call this to enable TTBR0 when per-instance pagetables
0133  * are active
0134  */
0135
0136 static int qcom_adreno_smmu_set_ttbr0_cfg(const void *cookie,
0137         const struct io_pgtable_cfg *pgtbl_cfg)
0138 {
0139     struct arm_smmu_domain *smmu_domain = (void *)cookie;
0140     struct io_pgtable *pgtable = io_pgtable_ops_to_pgtable(smmu_domain->pgtbl_ops);
0141     struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
0142     struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx];
0143
0144     /* The domain must have split pagetables already enabled */
0145     if (cb->tcr[0] & ARM_SMMU_TCR_EPD1)
0146         return -EINVAL;
0147
0148     /* If the pagetable config is NULL, disable TTBR0 */
0149     if (!pgtbl_cfg) {
0150         /* Do nothing if it is already disabled */
0151         if ((cb->tcr[0] & ARM_SMMU_TCR_EPD0))
0152             return -EINVAL;
0153
0154         /* Set TCR to the original configuration */
0155         cb->tcr[0] = arm_smmu_lpae_tcr(&pgtable->cfg);
0156         cb->ttbr[0] = FIELD_PREP(ARM_SMMU_TTBRn_ASID, cb->cfg->asid);
0157     } else {
0158         u32 tcr = cb->tcr[0];
0159
0160         /* Don't call this again if TTBR0 is already enabled */
0161         if (!(cb->tcr[0] & ARM_SMMU_TCR_EPD0))
0162             return -EINVAL;
0163
0164         tcr |= arm_smmu_lpae_tcr(pgtbl_cfg);
0165         tcr &= ~(ARM_SMMU_TCR_EPD0 | ARM_SMMU_TCR_EPD1);
0166
0167         cb->tcr[0] = tcr;
0168         cb->ttbr[0] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
0169         cb->ttbr[0] |= FIELD_PREP(ARM_SMMU_TTBRn_ASID, cb->cfg->asid);
0170     }
0171
0172     arm_smmu_write_context_bank(smmu_domain->smmu, cb->cfg->cbndx);
0173
0174     return 0;
0175 }
0176
0177 static int qcom_adreno_smmu_alloc_context_bank(struct arm_smmu_domain *smmu_domain,
0178                            struct arm_smmu_device *smmu,
0179                            struct device *dev, int start)
0180 {
0181     int count;
0182
0183     /*
0184      * Assign context bank 0 to the GPU device so the GPU hardware can
0185      * switch pagetables
0186      */
0187     if (qcom_adreno_smmu_is_gpu_device(dev)) {
0188         start = 0;
0189         count = 1;
0190     } else {
0191         start = 1;
0192         count = smmu->num_context_banks;
0193     }
0194
0195     return __arm_smmu_alloc_bitmap(smmu->context_map, start, count);
0196 }
0197
0198 static bool qcom_adreno_can_do_ttbr1(struct arm_smmu_device *smmu)
0199 {
0200     const struct device_node *np = smmu->dev->of_node;
0201
0202     if (of_device_is_compatible(np, "qcom,msm8996-smmu-v2"))
0203         return false;
0204
0205     return true;
0206 }
0207
0208 static int qcom_adreno_smmu_init_context(struct arm_smmu_domain *smmu_domain,
0209         struct io_pgtable_cfg *pgtbl_cfg, struct device *dev)
0210 {
0211     struct adreno_smmu_priv *priv;
0212
0213     smmu_domain->cfg.flush_walk_prefer_tlbiasid = true;
0214
0215     /* Only enable split pagetables for the GPU device (SID 0) */
0216     if (!qcom_adreno_smmu_is_gpu_device(dev))
0217         return 0;
0218
0219     /*
0220      * All targets that use the qcom,adreno-smmu compatible string *should*
0221      * be AARCH64 stage 1 but double check because the arm-smmu code assumes
0222      * that is the case when the TTBR1 quirk is enabled
0223      */
0224     if (qcom_adreno_can_do_ttbr1(smmu_domain->smmu) &&
0225         (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) &&
0226         (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64))
0227         pgtbl_cfg->quirks |= IO_PGTABLE_QUIRK_ARM_TTBR1;
0228
0229     /*
0230      * Initialize private interface with GPU:
0231      */
0232
0233     priv = dev_get_drvdata(dev);
0234     priv->cookie = smmu_domain;
0235     priv->get_ttbr1_cfg = qcom_adreno_smmu_get_ttbr1_cfg;
0236     priv->set_ttbr0_cfg = qcom_adreno_smmu_set_ttbr0_cfg;
0237     priv->get_fault_info = qcom_adreno_smmu_get_fault_info;
0238     priv->set_stall = qcom_adreno_smmu_set_stall;
0239     priv->resume_translation = qcom_adreno_smmu_resume_translation;
0240
0241     return 0;
0242 }
0243
0244 static const struct of_device_id qcom_smmu_client_of_match[] __maybe_unused = {
0245     { .compatible = "qcom,adreno" },
0246     { .compatible = "qcom,mdp4" },
0247     { .compatible = "qcom,mdss" },
0248     { .compatible = "qcom,sc7180-mdss" },
0249     { .compatible = "qcom,sc7180-mss-pil" },
0250     { .compatible = "qcom,sc7280-mdss" },
0251     { .compatible = "qcom,sc7280-mss-pil" },
0252     { .compatible = "qcom,sc8180x-mdss" },
0253     { .compatible = "qcom,sm8250-mdss" },
0254     { .compatible = "qcom,sdm845-mdss" },
0255     { .compatible = "qcom,sdm845-mss-pil" },
0256     { }
0257 };
0258
0259 static int qcom_smmu_init_context(struct arm_smmu_domain *smmu_domain,
0260         struct io_pgtable_cfg *pgtbl_cfg, struct device *dev)
0261 {
0262     smmu_domain->cfg.flush_walk_prefer_tlbiasid = true;
0263
0264     return 0;
0265 }
0266
0267 static int qcom_smmu_cfg_probe(struct arm_smmu_device *smmu)
0268 {
0269     unsigned int last_s2cr = ARM_SMMU_GR0_S2CR(smmu->num_mapping_groups - 1);
0270     struct qcom_smmu *qsmmu = to_qcom_smmu(smmu);
0271     u32 reg;
0272     u32 smr;
0273     int i;
0274
0275     /*
0276      * With some firmware versions writes to S2CR of type FAULT are
0277      * ignored, and writing BYPASS will end up written as FAULT in the
0278      * register. Perform a write to S2CR to detect if this is the case and
0279      * if so reserve a context bank to emulate bypass streams.
0280      */
0281     reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, S2CR_TYPE_BYPASS) |
0282           FIELD_PREP(ARM_SMMU_S2CR_CBNDX, 0xff) |
0283           FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, S2CR_PRIVCFG_DEFAULT);
0284     arm_smmu_gr0_write(smmu, last_s2cr, reg);
0285     reg = arm_smmu_gr0_read(smmu, last_s2cr);
0286     if (FIELD_GET(ARM_SMMU_S2CR_TYPE, reg) != S2CR_TYPE_BYPASS) {
0287         qsmmu->bypass_quirk = true;
0288         qsmmu->bypass_cbndx = smmu->num_context_banks - 1;
0289
0290         set_bit(qsmmu->bypass_cbndx, smmu->context_map);
0291
0292         arm_smmu_cb_write(smmu, qsmmu->bypass_cbndx, ARM_SMMU_CB_SCTLR, 0);
0293
0294         reg = FIELD_PREP(ARM_SMMU_CBAR_TYPE, CBAR_TYPE_S1_TRANS_S2_BYPASS);
0295         arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(qsmmu->bypass_cbndx), reg);
0296     }
0297
0298     for (i = 0; i < smmu->num_mapping_groups; i++) {
0299         smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
0300
0301         if (FIELD_GET(ARM_SMMU_SMR_VALID, smr)) {
0302             /* Ignore valid bit for SMR mask extraction. */
0303             smr &= ~ARM_SMMU_SMR_VALID;
0304             smmu->smrs[i].id = FIELD_GET(ARM_SMMU_SMR_ID, smr);
0305             smmu->smrs[i].mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr);
0306             smmu->smrs[i].valid = true;
0307
0308             smmu->s2crs[i].type = S2CR_TYPE_BYPASS;
0309             smmu->s2crs[i].privcfg = S2CR_PRIVCFG_DEFAULT;
0310             smmu->s2crs[i].cbndx = 0xff;
0311         }
0312     }
0313
0314     return 0;
0315 }
0316
0317 static void qcom_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
0318 {
0319     struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
0320     struct qcom_smmu *qsmmu = to_qcom_smmu(smmu);
0321     u32 cbndx = s2cr->cbndx;
0322     u32 type = s2cr->type;
0323     u32 reg;
0324
0325     if (qsmmu->bypass_quirk) {
0326         if (type == S2CR_TYPE_BYPASS) {
0327             /*
0328              * Firmware with quirky S2CR handling will substitute
0329              * BYPASS writes with FAULT, so point the stream to the
0330              * reserved context bank and ask for translation on the
0331              * stream
0332              */
0333             type = S2CR_TYPE_TRANS;
0334             cbndx = qsmmu->bypass_cbndx;
0335         } else if (type == S2CR_TYPE_FAULT) {
0336             /*
0337              * Firmware with quirky S2CR handling will ignore FAULT
0338              * writes, so trick it to write FAULT by asking for a
0339              * BYPASS.
0340              */
0341             type = S2CR_TYPE_BYPASS;
0342             cbndx = 0xff;
0343         }
0344     }
0345
0346     reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, type) |
0347           FIELD_PREP(ARM_SMMU_S2CR_CBNDX, cbndx) |
0348           FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, s2cr->privcfg);
0349     arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg);
0350 }
0351
0352 static int qcom_smmu_def_domain_type(struct device *dev)
0353 {
0354     const struct of_device_id *match =
0355         of_match_device(qcom_smmu_client_of_match, dev);
0356
0357     return match ? IOMMU_DOMAIN_IDENTITY : 0;
0358 }
0359
0360 static int qcom_sdm845_smmu500_reset(struct arm_smmu_device *smmu)
0361 {
0362     int ret;
0363
0364     /*
0365      * To address performance degradation in non-real time clients,
0366      * such as USB and UFS, turn off wait-for-safe on sdm845 based boards,
0367      * such as MTP and db845, whose firmwares implement secure monitor
0368      * call handlers to turn on/off the wait-for-safe logic.
0369      */
0370     ret = qcom_scm_qsmmu500_wait_safe_toggle(0);
0371     if (ret)
0372         dev_warn(smmu->dev, "Failed to turn off SAFE logic\n");
0373
0374     return ret;
0375 }
0376
0377 static int qcom_smmu500_reset(struct arm_smmu_device *smmu)
0378 {
0379     const struct device_node *np = smmu->dev->of_node;
0380
0381     arm_mmu500_reset(smmu);
0382
0383     if (of_device_is_compatible(np, "qcom,sdm845-smmu-500"))
0384         return qcom_sdm845_smmu500_reset(smmu);
0385
0386     return 0;
0387 }
0388
0389 static const struct arm_smmu_impl qcom_smmu_impl = {
0390     .init_context = qcom_smmu_init_context,
0391     .cfg_probe = qcom_smmu_cfg_probe,
0392     .def_domain_type = qcom_smmu_def_domain_type,
0393     .reset = qcom_smmu500_reset,
0394     .write_s2cr = qcom_smmu_write_s2cr,
0395     .tlb_sync = qcom_smmu_tlb_sync,
0396 };
0397
0398 static const struct arm_smmu_impl qcom_adreno_smmu_impl = {
0399     .init_context = qcom_adreno_smmu_init_context,
0400     .def_domain_type = qcom_smmu_def_domain_type,
0401     .reset = qcom_smmu500_reset,
0402     .alloc_context_bank = qcom_adreno_smmu_alloc_context_bank,
0403     .write_sctlr = qcom_adreno_smmu_write_sctlr,
0404     .tlb_sync = qcom_smmu_tlb_sync,
0405 };
0406
0407 static struct arm_smmu_device *qcom_smmu_create(struct arm_smmu_device *smmu,
0408         const struct arm_smmu_impl *impl)
0409 {
0410     struct qcom_smmu *qsmmu;
0411
0412     /* Check to make sure qcom_scm has finished probing */
0413     if (!qcom_scm_is_available())
0414         return ERR_PTR(-EPROBE_DEFER);
0415
0416     qsmmu = devm_krealloc(smmu->dev, smmu, sizeof(*qsmmu), GFP_KERNEL);
0417     if (!qsmmu)
0418         return ERR_PTR(-ENOMEM);
0419
0420     qsmmu->smmu.impl = impl;
0421     qsmmu->cfg = qcom_smmu_impl_data(smmu);
0422
0423     return &qsmmu->smmu;
0424 }
0425
0426 static const struct of_device_id __maybe_unused qcom_smmu_impl_of_match[] = {
0427     { .compatible = "qcom,msm8998-smmu-v2" },
0428     { .compatible = "qcom,qcm2290-smmu-500" },
0429     { .compatible = "qcom,sc7180-smmu-500" },
0430     { .compatible = "qcom,sc7280-smmu-500" },
0431     { .compatible = "qcom,sc8180x-smmu-500" },
0432     { .compatible = "qcom,sc8280xp-smmu-500" },
0433     { .compatible = "qcom,sdm630-smmu-v2" },
0434     { .compatible = "qcom,sdm845-smmu-500" },
0435     { .compatible = "qcom,sm6125-smmu-500" },
0436     { .compatible = "qcom,sm6350-smmu-500" },
0437     { .compatible = "qcom,sm6375-smmu-500" },
0438     { .compatible = "qcom,sm8150-smmu-500" },
0439     { .compatible = "qcom,sm8250-smmu-500" },
0440     { .compatible = "qcom,sm8350-smmu-500" },
0441     { .compatible = "qcom,sm8450-smmu-500" },
0442     { }
0443 };
0444
0445 #ifdef CONFIG_ACPI
0446 static struct acpi_platform_list qcom_acpi_platlist[] = {
0447     { "LENOVO", "CB-01   ", 0x8180, ACPI_SIG_IORT, equal, "QCOM SMMU" },
0448     { "QCOM  ", "QCOMEDK2", 0x8180, ACPI_SIG_IORT, equal, "QCOM SMMU" },
0449     { }
0450 };
0451 #endif
0452
0453 struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu)
0454 {
0455     const struct device_node *np = smmu->dev->of_node;
0456
0457 #ifdef CONFIG_ACPI
0458     if (np == NULL) {
0459         /* Match platform for ACPI boot */
0460         if (acpi_match_platform_list(qcom_acpi_platlist) >= 0)
0461             return qcom_smmu_create(smmu, &qcom_smmu_impl);
0462     }
0463 #endif
0464
0465     /*
0466      * Do not change this order of implementation, i.e., first adreno
0467      * smmu impl and then apss smmu since we can have both implementing
0468      * arm,mmu-500 in which case we will miss setting adreno smmu specific
0469      * features if the order is changed.
0470      */
0471     if (of_device_is_compatible(np, "qcom,adreno-smmu"))
0472         return qcom_smmu_create(smmu, &qcom_adreno_smmu_impl);
0473
0474     if (of_match_node(qcom_smmu_impl_of_match, np))
0475         return qcom_smmu_create(smmu, &qcom_smmu_impl);
0476
0477     return smmu;
0478 }