drivers/dma/milbeaut-xdmac.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 //
0003 // Copyright (C) 2019 Linaro Ltd.
0004 // Copyright (C) 2019 Socionext Inc.
0005
0006 #include <linux/bits.h>
0007 #include <linux/dma-mapping.h>
0008 #include <linux/dmaengine.h>
0009 #include <linux/interrupt.h>
0010 #include <linux/iopoll.h>
0011 #include <linux/list.h>
0012 #include <linux/module.h>
0013 #include <linux/of_dma.h>
0014 #include <linux/platform_device.h>
0015 #include <linux/slab.h>
0016 #include <linux/types.h>
0017 #include <linux/bitfield.h>
0018
0019 #include "virt-dma.h"
0020
0021 /* global register */
0022 #define M10V_XDACS 0x00
0023
0024 /* channel local register */
0025 #define M10V_XDTBC 0x10
0026 #define M10V_XDSSA 0x14
0027 #define M10V_XDDSA 0x18
0028 #define M10V_XDSAC 0x1C
0029 #define M10V_XDDAC 0x20
0030 #define M10V_XDDCC 0x24
0031 #define M10V_XDDES 0x28
0032 #define M10V_XDDPC 0x2C
0033 #define M10V_XDDSD 0x30
0034
0035 #define M10V_XDACS_XE BIT(28)
0036
0037 #define M10V_DEFBS  0x3
0038 #define M10V_DEFBL  0xf
0039
0040 #define M10V_XDSAC_SBS  GENMASK(17, 16)
0041 #define M10V_XDSAC_SBL  GENMASK(11, 8)
0042
0043 #define M10V_XDDAC_DBS  GENMASK(17, 16)
0044 #define M10V_XDDAC_DBL  GENMASK(11, 8)
0045
0046 #define M10V_XDDES_CE   BIT(28)
0047 #define M10V_XDDES_SE   BIT(24)
0048 #define M10V_XDDES_SA   BIT(15)
0049 #define M10V_XDDES_TF   GENMASK(23, 20)
0050 #define M10V_XDDES_EI   BIT(1)
0051 #define M10V_XDDES_TI   BIT(0)
0052
0053 #define M10V_XDDSD_IS_MASK  GENMASK(3, 0)
0054 #define M10V_XDDSD_IS_NORMAL    0x8
0055
0056 #define MLB_XDMAC_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
0057                  BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
0058                  BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \
0059                  BIT(DMA_SLAVE_BUSWIDTH_8_BYTES))
0060
0061 struct milbeaut_xdmac_desc {
0062     struct virt_dma_desc vd;
0063     size_t len;
0064     dma_addr_t src;
0065     dma_addr_t dst;
0066 };
0067
0068 struct milbeaut_xdmac_chan {
0069     struct virt_dma_chan vc;
0070     struct milbeaut_xdmac_desc *md;
0071     void __iomem *reg_ch_base;
0072 };
0073
0074 struct milbeaut_xdmac_device {
0075     struct dma_device ddev;
0076     void __iomem *reg_base;
0077     struct milbeaut_xdmac_chan channels[];
0078 };
0079
0080 static struct milbeaut_xdmac_chan *
0081 to_milbeaut_xdmac_chan(struct virt_dma_chan *vc)
0082 {
0083     return container_of(vc, struct milbeaut_xdmac_chan, vc);
0084 }
0085
0086 static struct milbeaut_xdmac_desc *
0087 to_milbeaut_xdmac_desc(struct virt_dma_desc *vd)
0088 {
0089     return container_of(vd, struct milbeaut_xdmac_desc, vd);
0090 }
0091
0092 /* mc->vc.lock must be held by caller */
0093 static struct milbeaut_xdmac_desc *
0094 milbeaut_xdmac_next_desc(struct milbeaut_xdmac_chan *mc)
0095 {
0096     struct virt_dma_desc *vd;
0097
0098     vd = vchan_next_desc(&mc->vc);
0099     if (!vd) {
0100         mc->md = NULL;
0101         return NULL;
0102     }
0103
0104     list_del(&vd->node);
0105
0106     mc->md = to_milbeaut_xdmac_desc(vd);
0107
0108     return mc->md;
0109 }
0110
0111 /* mc->vc.lock must be held by caller */
0112 static void milbeaut_chan_start(struct milbeaut_xdmac_chan *mc,
0113                 struct milbeaut_xdmac_desc *md)
0114 {
0115     u32 val;
0116
0117     /* Setup the channel */
0118     val = md->len - 1;
0119     writel_relaxed(val, mc->reg_ch_base + M10V_XDTBC);
0120
0121     val = md->src;
0122     writel_relaxed(val, mc->reg_ch_base + M10V_XDSSA);
0123
0124     val = md->dst;
0125     writel_relaxed(val, mc->reg_ch_base + M10V_XDDSA);
0126
0127     val = readl_relaxed(mc->reg_ch_base + M10V_XDSAC);
0128     val &= ~(M10V_XDSAC_SBS | M10V_XDSAC_SBL);
0129     val |= FIELD_PREP(M10V_XDSAC_SBS, M10V_DEFBS) |
0130         FIELD_PREP(M10V_XDSAC_SBL, M10V_DEFBL);
0131     writel_relaxed(val, mc->reg_ch_base + M10V_XDSAC);
0132
0133     val = readl_relaxed(mc->reg_ch_base + M10V_XDDAC);
0134     val &= ~(M10V_XDDAC_DBS | M10V_XDDAC_DBL);
0135     val |= FIELD_PREP(M10V_XDDAC_DBS, M10V_DEFBS) |
0136         FIELD_PREP(M10V_XDDAC_DBL, M10V_DEFBL);
0137     writel_relaxed(val, mc->reg_ch_base + M10V_XDDAC);
0138
0139     /* Start the channel */
0140     val = readl_relaxed(mc->reg_ch_base + M10V_XDDES);
0141     val &= ~(M10V_XDDES_CE | M10V_XDDES_SE | M10V_XDDES_TF |
0142          M10V_XDDES_EI | M10V_XDDES_TI);
0143     val |= FIELD_PREP(M10V_XDDES_CE, 1) | FIELD_PREP(M10V_XDDES_SE, 1) |
0144         FIELD_PREP(M10V_XDDES_TF, 1) | FIELD_PREP(M10V_XDDES_EI, 1) |
0145         FIELD_PREP(M10V_XDDES_TI, 1);
0146     writel_relaxed(val, mc->reg_ch_base + M10V_XDDES);
0147 }
0148
0149 /* mc->vc.lock must be held by caller */
0150 static void milbeaut_xdmac_start(struct milbeaut_xdmac_chan *mc)
0151 {
0152     struct milbeaut_xdmac_desc *md;
0153
0154     md = milbeaut_xdmac_next_desc(mc);
0155     if (md)
0156         milbeaut_chan_start(mc, md);
0157 }
0158
0159 static irqreturn_t milbeaut_xdmac_interrupt(int irq, void *dev_id)
0160 {
0161     struct milbeaut_xdmac_chan *mc = dev_id;
0162     struct milbeaut_xdmac_desc *md;
0163     u32 val;
0164
0165     spin_lock(&mc->vc.lock);
0166
0167     /* Ack and Stop */
0168     val = FIELD_PREP(M10V_XDDSD_IS_MASK, 0x0);
0169     writel_relaxed(val, mc->reg_ch_base + M10V_XDDSD);
0170
0171     md = mc->md;
0172     if (!md)
0173         goto out;
0174
0175     vchan_cookie_complete(&md->vd);
0176
0177     milbeaut_xdmac_start(mc);
0178 out:
0179     spin_unlock(&mc->vc.lock);
0180     return IRQ_HANDLED;
0181 }
0182
0183 static void milbeaut_xdmac_free_chan_resources(struct dma_chan *chan)
0184 {
0185     vchan_free_chan_resources(to_virt_chan(chan));
0186 }
0187
0188 static struct dma_async_tx_descriptor *
0189 milbeaut_xdmac_prep_memcpy(struct dma_chan *chan, dma_addr_t dst,
0190                dma_addr_t src, size_t len, unsigned long flags)
0191 {
0192     struct virt_dma_chan *vc = to_virt_chan(chan);
0193     struct milbeaut_xdmac_desc *md;
0194
0195     md = kzalloc(sizeof(*md), GFP_NOWAIT);
0196     if (!md)
0197         return NULL;
0198
0199     md->len = len;
0200     md->src = src;
0201     md->dst = dst;
0202
0203     return vchan_tx_prep(vc, &md->vd, flags);
0204 }
0205
0206 static int milbeaut_xdmac_terminate_all(struct dma_chan *chan)
0207 {
0208     struct virt_dma_chan *vc = to_virt_chan(chan);
0209     struct milbeaut_xdmac_chan *mc = to_milbeaut_xdmac_chan(vc);
0210     unsigned long flags;
0211     u32 val;
0212
0213     LIST_HEAD(head);
0214
0215     spin_lock_irqsave(&vc->lock, flags);
0216
0217     /* Halt the channel */
0218     val = readl(mc->reg_ch_base + M10V_XDDES);
0219     val &= ~M10V_XDDES_CE;
0220     val |= FIELD_PREP(M10V_XDDES_CE, 0);
0221     writel(val, mc->reg_ch_base + M10V_XDDES);
0222
0223     if (mc->md) {
0224         vchan_terminate_vdesc(&mc->md->vd);
0225         mc->md = NULL;
0226     }
0227
0228     vchan_get_all_descriptors(vc, &head);
0229
0230     spin_unlock_irqrestore(&vc->lock, flags);
0231
0232     vchan_dma_desc_free_list(vc, &head);
0233
0234     return 0;
0235 }
0236
0237 static void milbeaut_xdmac_synchronize(struct dma_chan *chan)
0238 {
0239     vchan_synchronize(to_virt_chan(chan));
0240 }
0241
0242 static void milbeaut_xdmac_issue_pending(struct dma_chan *chan)
0243 {
0244     struct virt_dma_chan *vc = to_virt_chan(chan);
0245     struct milbeaut_xdmac_chan *mc = to_milbeaut_xdmac_chan(vc);
0246     unsigned long flags;
0247
0248     spin_lock_irqsave(&vc->lock, flags);
0249
0250     if (vchan_issue_pending(vc) && !mc->md)
0251         milbeaut_xdmac_start(mc);
0252
0253     spin_unlock_irqrestore(&vc->lock, flags);
0254 }
0255
0256 static void milbeaut_xdmac_desc_free(struct virt_dma_desc *vd)
0257 {
0258     kfree(to_milbeaut_xdmac_desc(vd));
0259 }
0260
0261 static int milbeaut_xdmac_chan_init(struct platform_device *pdev,
0262                     struct milbeaut_xdmac_device *mdev,
0263                     int chan_id)
0264 {
0265     struct device *dev = &pdev->dev;
0266     struct milbeaut_xdmac_chan *mc = &mdev->channels[chan_id];
0267     char *irq_name;
0268     int irq, ret;
0269
0270     irq = platform_get_irq(pdev, chan_id);
0271     if (irq < 0)
0272         return irq;
0273
0274     irq_name = devm_kasprintf(dev, GFP_KERNEL, "milbeaut-xdmac-%d",
0275                   chan_id);
0276     if (!irq_name)
0277         return -ENOMEM;
0278
0279     ret = devm_request_irq(dev, irq, milbeaut_xdmac_interrupt,
0280                    IRQF_SHARED, irq_name, mc);
0281     if (ret)
0282         return ret;
0283
0284     mc->reg_ch_base = mdev->reg_base + chan_id * 0x30;
0285
0286     mc->vc.desc_free = milbeaut_xdmac_desc_free;
0287     vchan_init(&mc->vc, &mdev->ddev);
0288
0289     return 0;
0290 }
0291
0292 static void enable_xdmac(struct milbeaut_xdmac_device *mdev)
0293 {
0294     unsigned int val;
0295
0296     val = readl(mdev->reg_base + M10V_XDACS);
0297     val |= M10V_XDACS_XE;
0298     writel(val, mdev->reg_base + M10V_XDACS);
0299 }
0300
0301 static void disable_xdmac(struct milbeaut_xdmac_device *mdev)
0302 {
0303     unsigned int val;
0304
0305     val = readl(mdev->reg_base + M10V_XDACS);
0306     val &= ~M10V_XDACS_XE;
0307     writel(val, mdev->reg_base + M10V_XDACS);
0308 }
0309
0310 static int milbeaut_xdmac_probe(struct platform_device *pdev)
0311 {
0312     struct device *dev = &pdev->dev;
0313     struct milbeaut_xdmac_device *mdev;
0314     struct dma_device *ddev;
0315     int nr_chans, ret, i;
0316
0317     nr_chans = platform_irq_count(pdev);
0318     if (nr_chans < 0)
0319         return nr_chans;
0320
0321     mdev = devm_kzalloc(dev, struct_size(mdev, channels, nr_chans),
0322                 GFP_KERNEL);
0323     if (!mdev)
0324         return -ENOMEM;
0325
0326     mdev->reg_base = devm_platform_ioremap_resource(pdev, 0);
0327     if (IS_ERR(mdev->reg_base))
0328         return PTR_ERR(mdev->reg_base);
0329
0330     ddev = &mdev->ddev;
0331     ddev->dev = dev;
0332     dma_cap_set(DMA_MEMCPY, ddev->cap_mask);
0333     ddev->src_addr_widths = MLB_XDMAC_BUSWIDTHS;
0334     ddev->dst_addr_widths = MLB_XDMAC_BUSWIDTHS;
0335     ddev->device_free_chan_resources = milbeaut_xdmac_free_chan_resources;
0336     ddev->device_prep_dma_memcpy = milbeaut_xdmac_prep_memcpy;
0337     ddev->device_terminate_all = milbeaut_xdmac_terminate_all;
0338     ddev->device_synchronize = milbeaut_xdmac_synchronize;
0339     ddev->device_tx_status = dma_cookie_status;
0340     ddev->device_issue_pending = milbeaut_xdmac_issue_pending;
0341     INIT_LIST_HEAD(&ddev->channels);
0342
0343     for (i = 0; i < nr_chans; i++) {
0344         ret = milbeaut_xdmac_chan_init(pdev, mdev, i);
0345         if (ret)
0346             return ret;
0347     }
0348
0349     enable_xdmac(mdev);
0350
0351     ret = dma_async_device_register(ddev);
0352     if (ret)
0353         goto disable_xdmac;
0354
0355     ret = of_dma_controller_register(dev->of_node,
0356                      of_dma_simple_xlate, mdev);
0357     if (ret)
0358         goto unregister_dmac;
0359
0360     platform_set_drvdata(pdev, mdev);
0361
0362     return 0;
0363
0364 unregister_dmac:
0365     dma_async_device_unregister(ddev);
0366 disable_xdmac:
0367     disable_xdmac(mdev);
0368     return ret;
0369 }
0370
0371 static int milbeaut_xdmac_remove(struct platform_device *pdev)
0372 {
0373     struct milbeaut_xdmac_device *mdev = platform_get_drvdata(pdev);
0374     struct dma_chan *chan;
0375     int ret;
0376
0377     /*
0378      * Before reaching here, almost all descriptors have been freed by the
0379      * ->device_free_chan_resources() hook. However, each channel might
0380      * be still holding one descriptor that was on-flight at that moment.
0381      * Terminate it to make sure this hardware is no longer running. Then,
0382      * free the channel resources once again to avoid memory leak.
0383      */
0384     list_for_each_entry(chan, &mdev->ddev.channels, device_node) {
0385         ret = dmaengine_terminate_sync(chan);
0386         if (ret)
0387             return ret;
0388         milbeaut_xdmac_free_chan_resources(chan);
0389     }
0390
0391     of_dma_controller_free(pdev->dev.of_node);
0392     dma_async_device_unregister(&mdev->ddev);
0393
0394     disable_xdmac(mdev);
0395
0396     return 0;
0397 }
0398
0399 static const struct of_device_id milbeaut_xdmac_match[] = {
0400     { .compatible = "socionext,milbeaut-m10v-xdmac" },
0401     { /* sentinel */ }
0402 };
0403 MODULE_DEVICE_TABLE(of, milbeaut_xdmac_match);
0404
0405 static struct platform_driver milbeaut_xdmac_driver = {
0406     .probe = milbeaut_xdmac_probe,
0407     .remove = milbeaut_xdmac_remove,
0408     .driver = {
0409         .name = "milbeaut-m10v-xdmac",
0410         .of_match_table = milbeaut_xdmac_match,
0411     },
0412 };
0413 module_platform_driver(milbeaut_xdmac_driver);
0414
0415 MODULE_DESCRIPTION("Milbeaut XDMAC DmaEngine driver");
0416 MODULE_LICENSE("GPL v2");