Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * Renesas RZ/G2L DMA Controller Driver
0004  *
0005  * Based on imx-dma.c
0006  *
0007  * Copyright (C) 2021 Renesas Electronics Corp.
0008  * Copyright 2010 Sascha Hauer, Pengutronix <s.hauer@pengutronix.de>
0009  * Copyright 2012 Javier Martin, Vista Silicon <javier.martin@vista-silicon.com>
0010  */
0011 
0012 #include <linux/dma-mapping.h>
0013 #include <linux/dmaengine.h>
0014 #include <linux/interrupt.h>
0015 #include <linux/iopoll.h>
0016 #include <linux/list.h>
0017 #include <linux/module.h>
0018 #include <linux/of.h>
0019 #include <linux/of_dma.h>
0020 #include <linux/of_platform.h>
0021 #include <linux/platform_device.h>
0022 #include <linux/pm_runtime.h>
0023 #include <linux/slab.h>
0024 #include <linux/spinlock.h>
0025 
0026 #include "../dmaengine.h"
0027 #include "../virt-dma.h"
0028 
0029 enum  rz_dmac_prep_type {
0030     RZ_DMAC_DESC_MEMCPY,
0031     RZ_DMAC_DESC_SLAVE_SG,
0032 };
0033 
0034 struct rz_lmdesc {
0035     u32 header;
0036     u32 sa;
0037     u32 da;
0038     u32 tb;
0039     u32 chcfg;
0040     u32 chitvl;
0041     u32 chext;
0042     u32 nxla;
0043 };
0044 
0045 struct rz_dmac_desc {
0046     struct virt_dma_desc vd;
0047     dma_addr_t src;
0048     dma_addr_t dest;
0049     size_t len;
0050     struct list_head node;
0051     enum dma_transfer_direction direction;
0052     enum rz_dmac_prep_type type;
0053     /* For slave sg */
0054     struct scatterlist *sg;
0055     unsigned int sgcount;
0056 };
0057 
0058 #define to_rz_dmac_desc(d)  container_of(d, struct rz_dmac_desc, vd)
0059 
0060 struct rz_dmac_chan {
0061     struct virt_dma_chan vc;
0062     void __iomem *ch_base;
0063     void __iomem *ch_cmn_base;
0064     unsigned int index;
0065     int irq;
0066     struct rz_dmac_desc *desc;
0067     int descs_allocated;
0068 
0069     enum dma_slave_buswidth src_word_size;
0070     enum dma_slave_buswidth dst_word_size;
0071     dma_addr_t src_per_address;
0072     dma_addr_t dst_per_address;
0073 
0074     u32 chcfg;
0075     u32 chctrl;
0076     int mid_rid;
0077 
0078     struct list_head ld_free;
0079     struct list_head ld_queue;
0080     struct list_head ld_active;
0081 
0082     struct {
0083         struct rz_lmdesc *base;
0084         struct rz_lmdesc *head;
0085         struct rz_lmdesc *tail;
0086         dma_addr_t base_dma;
0087     } lmdesc;
0088 };
0089 
0090 #define to_rz_dmac_chan(c)  container_of(c, struct rz_dmac_chan, vc.chan)
0091 
0092 struct rz_dmac {
0093     struct dma_device engine;
0094     struct device *dev;
0095     void __iomem *base;
0096     void __iomem *ext_base;
0097 
0098     unsigned int n_channels;
0099     struct rz_dmac_chan *channels;
0100 
0101     DECLARE_BITMAP(modules, 1024);
0102 };
0103 
0104 #define to_rz_dmac(d)   container_of(d, struct rz_dmac, engine)
0105 
0106 /*
0107  * -----------------------------------------------------------------------------
0108  * Registers
0109  */
0110 
0111 #define CHSTAT              0x0024
0112 #define CHCTRL              0x0028
0113 #define CHCFG               0x002c
0114 #define NXLA                0x0038
0115 
0116 #define DCTRL               0x0000
0117 
0118 #define EACH_CHANNEL_OFFSET     0x0040
0119 #define CHANNEL_0_7_OFFSET      0x0000
0120 #define CHANNEL_0_7_COMMON_BASE     0x0300
0121 #define CHANNEL_8_15_OFFSET     0x0400
0122 #define CHANNEL_8_15_COMMON_BASE    0x0700
0123 
0124 #define CHSTAT_ER           BIT(4)
0125 #define CHSTAT_EN           BIT(0)
0126 
0127 #define CHCTRL_CLRINTMSK        BIT(17)
0128 #define CHCTRL_CLRSUS           BIT(9)
0129 #define CHCTRL_CLRTC            BIT(6)
0130 #define CHCTRL_CLREND           BIT(5)
0131 #define CHCTRL_CLRRQ            BIT(4)
0132 #define CHCTRL_SWRST            BIT(3)
0133 #define CHCTRL_STG          BIT(2)
0134 #define CHCTRL_CLREN            BIT(1)
0135 #define CHCTRL_SETEN            BIT(0)
0136 #define CHCTRL_DEFAULT          (CHCTRL_CLRINTMSK | CHCTRL_CLRSUS | \
0137                      CHCTRL_CLRTC | CHCTRL_CLREND | \
0138                      CHCTRL_CLRRQ | CHCTRL_SWRST | \
0139                      CHCTRL_CLREN)
0140 
0141 #define CHCFG_DMS           BIT(31)
0142 #define CHCFG_DEM           BIT(24)
0143 #define CHCFG_DAD           BIT(21)
0144 #define CHCFG_SAD           BIT(20)
0145 #define CHCFG_REQD          BIT(3)
0146 #define CHCFG_SEL(bits)         ((bits) & 0x07)
0147 #define CHCFG_MEM_COPY          (0x80400008)
0148 #define CHCFG_FILL_DDS(a)       (((a) << 16) & GENMASK(19, 16))
0149 #define CHCFG_FILL_SDS(a)       (((a) << 12) & GENMASK(15, 12))
0150 #define CHCFG_FILL_TM(a)        (((a) & BIT(5)) << 22)
0151 #define CHCFG_FILL_AM(a)        (((a) & GENMASK(4, 2)) << 6)
0152 #define CHCFG_FILL_LVL(a)       (((a) & BIT(1)) << 5)
0153 #define CHCFG_FILL_HIEN(a)      (((a) & BIT(0)) << 5)
0154 
0155 #define MID_RID_MASK            GENMASK(9, 0)
0156 #define CHCFG_MASK          GENMASK(15, 10)
0157 #define CHCFG_DS_INVALID        0xFF
0158 #define DCTRL_LVINT         BIT(1)
0159 #define DCTRL_PR            BIT(0)
0160 #define DCTRL_DEFAULT           (DCTRL_LVINT | DCTRL_PR)
0161 
0162 /* LINK MODE DESCRIPTOR */
0163 #define HEADER_LV           BIT(0)
0164 
0165 #define RZ_DMAC_MAX_CHAN_DESCRIPTORS    16
0166 #define RZ_DMAC_MAX_CHANNELS        16
0167 #define DMAC_NR_LMDESC          64
0168 
0169 /*
0170  * -----------------------------------------------------------------------------
0171  * Device access
0172  */
0173 
0174 static void rz_dmac_writel(struct rz_dmac *dmac, unsigned int val,
0175                unsigned int offset)
0176 {
0177     writel(val, dmac->base + offset);
0178 }
0179 
0180 static void rz_dmac_ext_writel(struct rz_dmac *dmac, unsigned int val,
0181                    unsigned int offset)
0182 {
0183     writel(val, dmac->ext_base + offset);
0184 }
0185 
0186 static u32 rz_dmac_ext_readl(struct rz_dmac *dmac, unsigned int offset)
0187 {
0188     return readl(dmac->ext_base + offset);
0189 }
0190 
0191 static void rz_dmac_ch_writel(struct rz_dmac_chan *channel, unsigned int val,
0192                   unsigned int offset, int which)
0193 {
0194     if (which)
0195         writel(val, channel->ch_base + offset);
0196     else
0197         writel(val, channel->ch_cmn_base + offset);
0198 }
0199 
0200 static u32 rz_dmac_ch_readl(struct rz_dmac_chan *channel,
0201                 unsigned int offset, int which)
0202 {
0203     if (which)
0204         return readl(channel->ch_base + offset);
0205     else
0206         return readl(channel->ch_cmn_base + offset);
0207 }
0208 
0209 /*
0210  * -----------------------------------------------------------------------------
0211  * Initialization
0212  */
0213 
0214 static void rz_lmdesc_setup(struct rz_dmac_chan *channel,
0215                 struct rz_lmdesc *lmdesc)
0216 {
0217     u32 nxla;
0218 
0219     channel->lmdesc.base = lmdesc;
0220     channel->lmdesc.head = lmdesc;
0221     channel->lmdesc.tail = lmdesc;
0222     nxla = channel->lmdesc.base_dma;
0223     while (lmdesc < (channel->lmdesc.base + (DMAC_NR_LMDESC - 1))) {
0224         lmdesc->header = 0;
0225         nxla += sizeof(*lmdesc);
0226         lmdesc->nxla = nxla;
0227         lmdesc++;
0228     }
0229 
0230     lmdesc->header = 0;
0231     lmdesc->nxla = channel->lmdesc.base_dma;
0232 }
0233 
0234 /*
0235  * -----------------------------------------------------------------------------
0236  * Descriptors preparation
0237  */
0238 
0239 static void rz_dmac_lmdesc_recycle(struct rz_dmac_chan *channel)
0240 {
0241     struct rz_lmdesc *lmdesc = channel->lmdesc.head;
0242 
0243     while (!(lmdesc->header & HEADER_LV)) {
0244         lmdesc->header = 0;
0245         lmdesc++;
0246         if (lmdesc >= (channel->lmdesc.base + DMAC_NR_LMDESC))
0247             lmdesc = channel->lmdesc.base;
0248     }
0249     channel->lmdesc.head = lmdesc;
0250 }
0251 
0252 static void rz_dmac_enable_hw(struct rz_dmac_chan *channel)
0253 {
0254     struct dma_chan *chan = &channel->vc.chan;
0255     struct rz_dmac *dmac = to_rz_dmac(chan->device);
0256     unsigned long flags;
0257     u32 nxla;
0258     u32 chctrl;
0259     u32 chstat;
0260 
0261     dev_dbg(dmac->dev, "%s channel %d\n", __func__, channel->index);
0262 
0263     local_irq_save(flags);
0264 
0265     rz_dmac_lmdesc_recycle(channel);
0266 
0267     nxla = channel->lmdesc.base_dma +
0268         (sizeof(struct rz_lmdesc) * (channel->lmdesc.head -
0269                          channel->lmdesc.base));
0270 
0271     chstat = rz_dmac_ch_readl(channel, CHSTAT, 1);
0272     if (!(chstat & CHSTAT_EN)) {
0273         chctrl = (channel->chctrl | CHCTRL_SETEN);
0274         rz_dmac_ch_writel(channel, nxla, NXLA, 1);
0275         rz_dmac_ch_writel(channel, channel->chcfg, CHCFG, 1);
0276         rz_dmac_ch_writel(channel, CHCTRL_SWRST, CHCTRL, 1);
0277         rz_dmac_ch_writel(channel, chctrl, CHCTRL, 1);
0278     }
0279 
0280     local_irq_restore(flags);
0281 }
0282 
0283 static void rz_dmac_disable_hw(struct rz_dmac_chan *channel)
0284 {
0285     struct dma_chan *chan = &channel->vc.chan;
0286     struct rz_dmac *dmac = to_rz_dmac(chan->device);
0287     unsigned long flags;
0288 
0289     dev_dbg(dmac->dev, "%s channel %d\n", __func__, channel->index);
0290 
0291     local_irq_save(flags);
0292     rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1);
0293     local_irq_restore(flags);
0294 }
0295 
0296 static void rz_dmac_set_dmars_register(struct rz_dmac *dmac, int nr, u32 dmars)
0297 {
0298     u32 dmars_offset = (nr / 2) * 4;
0299     u32 shift = (nr % 2) * 16;
0300     u32 dmars32;
0301 
0302     dmars32 = rz_dmac_ext_readl(dmac, dmars_offset);
0303     dmars32 &= ~(0xffff << shift);
0304     dmars32 |= dmars << shift;
0305 
0306     rz_dmac_ext_writel(dmac, dmars32, dmars_offset);
0307 }
0308 
0309 static void rz_dmac_prepare_desc_for_memcpy(struct rz_dmac_chan *channel)
0310 {
0311     struct dma_chan *chan = &channel->vc.chan;
0312     struct rz_dmac *dmac = to_rz_dmac(chan->device);
0313     struct rz_lmdesc *lmdesc = channel->lmdesc.tail;
0314     struct rz_dmac_desc *d = channel->desc;
0315     u32 chcfg = CHCFG_MEM_COPY;
0316 
0317     /* prepare descriptor */
0318     lmdesc->sa = d->src;
0319     lmdesc->da = d->dest;
0320     lmdesc->tb = d->len;
0321     lmdesc->chcfg = chcfg;
0322     lmdesc->chitvl = 0;
0323     lmdesc->chext = 0;
0324     lmdesc->header = HEADER_LV;
0325 
0326     rz_dmac_set_dmars_register(dmac, channel->index, 0);
0327 
0328     channel->chcfg = chcfg;
0329     channel->chctrl = CHCTRL_STG | CHCTRL_SETEN;
0330 }
0331 
0332 static void rz_dmac_prepare_descs_for_slave_sg(struct rz_dmac_chan *channel)
0333 {
0334     struct dma_chan *chan = &channel->vc.chan;
0335     struct rz_dmac *dmac = to_rz_dmac(chan->device);
0336     struct rz_dmac_desc *d = channel->desc;
0337     struct scatterlist *sg, *sgl = d->sg;
0338     struct rz_lmdesc *lmdesc;
0339     unsigned int i, sg_len = d->sgcount;
0340 
0341     channel->chcfg |= CHCFG_SEL(channel->index) | CHCFG_DEM | CHCFG_DMS;
0342 
0343     if (d->direction == DMA_DEV_TO_MEM) {
0344         channel->chcfg |= CHCFG_SAD;
0345         channel->chcfg &= ~CHCFG_REQD;
0346     } else {
0347         channel->chcfg |= CHCFG_DAD | CHCFG_REQD;
0348     }
0349 
0350     lmdesc = channel->lmdesc.tail;
0351 
0352     for (i = 0, sg = sgl; i < sg_len; i++, sg = sg_next(sg)) {
0353         if (d->direction == DMA_DEV_TO_MEM) {
0354             lmdesc->sa = channel->src_per_address;
0355             lmdesc->da = sg_dma_address(sg);
0356         } else {
0357             lmdesc->sa = sg_dma_address(sg);
0358             lmdesc->da = channel->dst_per_address;
0359         }
0360 
0361         lmdesc->tb = sg_dma_len(sg);
0362         lmdesc->chitvl = 0;
0363         lmdesc->chext = 0;
0364         if (i == (sg_len - 1)) {
0365             lmdesc->chcfg = (channel->chcfg & ~CHCFG_DEM);
0366             lmdesc->header = HEADER_LV;
0367         } else {
0368             lmdesc->chcfg = channel->chcfg;
0369             lmdesc->header = HEADER_LV;
0370         }
0371         if (++lmdesc >= (channel->lmdesc.base + DMAC_NR_LMDESC))
0372             lmdesc = channel->lmdesc.base;
0373     }
0374 
0375     channel->lmdesc.tail = lmdesc;
0376 
0377     rz_dmac_set_dmars_register(dmac, channel->index, channel->mid_rid);
0378     channel->chctrl = CHCTRL_SETEN;
0379 }
0380 
0381 static int rz_dmac_xfer_desc(struct rz_dmac_chan *chan)
0382 {
0383     struct rz_dmac_desc *d = chan->desc;
0384     struct virt_dma_desc *vd;
0385 
0386     vd = vchan_next_desc(&chan->vc);
0387     if (!vd)
0388         return 0;
0389 
0390     list_del(&vd->node);
0391 
0392     switch (d->type) {
0393     case RZ_DMAC_DESC_MEMCPY:
0394         rz_dmac_prepare_desc_for_memcpy(chan);
0395         break;
0396 
0397     case RZ_DMAC_DESC_SLAVE_SG:
0398         rz_dmac_prepare_descs_for_slave_sg(chan);
0399         break;
0400 
0401     default:
0402         return -EINVAL;
0403     }
0404 
0405     rz_dmac_enable_hw(chan);
0406 
0407     return 0;
0408 }
0409 
0410 /*
0411  * -----------------------------------------------------------------------------
0412  * DMA engine operations
0413  */
0414 
0415 static int rz_dmac_alloc_chan_resources(struct dma_chan *chan)
0416 {
0417     struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
0418 
0419     while (channel->descs_allocated < RZ_DMAC_MAX_CHAN_DESCRIPTORS) {
0420         struct rz_dmac_desc *desc;
0421 
0422         desc = kzalloc(sizeof(*desc), GFP_KERNEL);
0423         if (!desc)
0424             break;
0425 
0426         list_add_tail(&desc->node, &channel->ld_free);
0427         channel->descs_allocated++;
0428     }
0429 
0430     if (!channel->descs_allocated)
0431         return -ENOMEM;
0432 
0433     return channel->descs_allocated;
0434 }
0435 
0436 static void rz_dmac_free_chan_resources(struct dma_chan *chan)
0437 {
0438     struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
0439     struct rz_dmac *dmac = to_rz_dmac(chan->device);
0440     struct rz_lmdesc *lmdesc = channel->lmdesc.base;
0441     struct rz_dmac_desc *desc, *_desc;
0442     unsigned long flags;
0443     unsigned int i;
0444 
0445     spin_lock_irqsave(&channel->vc.lock, flags);
0446 
0447     for (i = 0; i < DMAC_NR_LMDESC; i++)
0448         lmdesc[i].header = 0;
0449 
0450     rz_dmac_disable_hw(channel);
0451     list_splice_tail_init(&channel->ld_active, &channel->ld_free);
0452     list_splice_tail_init(&channel->ld_queue, &channel->ld_free);
0453 
0454     if (channel->mid_rid >= 0) {
0455         clear_bit(channel->mid_rid, dmac->modules);
0456         channel->mid_rid = -EINVAL;
0457     }
0458 
0459     spin_unlock_irqrestore(&channel->vc.lock, flags);
0460 
0461     list_for_each_entry_safe(desc, _desc, &channel->ld_free, node) {
0462         kfree(desc);
0463         channel->descs_allocated--;
0464     }
0465 
0466     INIT_LIST_HEAD(&channel->ld_free);
0467     vchan_free_chan_resources(&channel->vc);
0468 }
0469 
0470 static struct dma_async_tx_descriptor *
0471 rz_dmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
0472             size_t len, unsigned long flags)
0473 {
0474     struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
0475     struct rz_dmac *dmac = to_rz_dmac(chan->device);
0476     struct rz_dmac_desc *desc;
0477 
0478     dev_dbg(dmac->dev, "%s channel: %d src=0x%pad dst=0x%pad len=%zu\n",
0479         __func__, channel->index, &src, &dest, len);
0480 
0481     if (list_empty(&channel->ld_free))
0482         return NULL;
0483 
0484     desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node);
0485 
0486     desc->type = RZ_DMAC_DESC_MEMCPY;
0487     desc->src = src;
0488     desc->dest = dest;
0489     desc->len = len;
0490     desc->direction = DMA_MEM_TO_MEM;
0491 
0492     list_move_tail(channel->ld_free.next, &channel->ld_queue);
0493     return vchan_tx_prep(&channel->vc, &desc->vd, flags);
0494 }
0495 
0496 static struct dma_async_tx_descriptor *
0497 rz_dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
0498               unsigned int sg_len,
0499               enum dma_transfer_direction direction,
0500               unsigned long flags, void *context)
0501 {
0502     struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
0503     struct rz_dmac_desc *desc;
0504     struct scatterlist *sg;
0505     int dma_length = 0;
0506     int i = 0;
0507 
0508     if (list_empty(&channel->ld_free))
0509         return NULL;
0510 
0511     desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node);
0512 
0513     for_each_sg(sgl, sg, sg_len, i) {
0514         dma_length += sg_dma_len(sg);
0515     }
0516 
0517     desc->type = RZ_DMAC_DESC_SLAVE_SG;
0518     desc->sg = sgl;
0519     desc->sgcount = sg_len;
0520     desc->len = dma_length;
0521     desc->direction = direction;
0522 
0523     if (direction == DMA_DEV_TO_MEM)
0524         desc->src = channel->src_per_address;
0525     else
0526         desc->dest = channel->dst_per_address;
0527 
0528     list_move_tail(channel->ld_free.next, &channel->ld_queue);
0529     return vchan_tx_prep(&channel->vc, &desc->vd, flags);
0530 }
0531 
0532 static int rz_dmac_terminate_all(struct dma_chan *chan)
0533 {
0534     struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
0535     unsigned long flags;
0536     LIST_HEAD(head);
0537 
0538     rz_dmac_disable_hw(channel);
0539     spin_lock_irqsave(&channel->vc.lock, flags);
0540     list_splice_tail_init(&channel->ld_active, &channel->ld_free);
0541     list_splice_tail_init(&channel->ld_queue, &channel->ld_free);
0542     spin_unlock_irqrestore(&channel->vc.lock, flags);
0543     vchan_get_all_descriptors(&channel->vc, &head);
0544     vchan_dma_desc_free_list(&channel->vc, &head);
0545 
0546     return 0;
0547 }
0548 
0549 static void rz_dmac_issue_pending(struct dma_chan *chan)
0550 {
0551     struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
0552     struct rz_dmac *dmac = to_rz_dmac(chan->device);
0553     struct rz_dmac_desc *desc;
0554     unsigned long flags;
0555 
0556     spin_lock_irqsave(&channel->vc.lock, flags);
0557 
0558     if (!list_empty(&channel->ld_queue)) {
0559         desc = list_first_entry(&channel->ld_queue,
0560                     struct rz_dmac_desc, node);
0561         channel->desc = desc;
0562         if (vchan_issue_pending(&channel->vc)) {
0563             if (rz_dmac_xfer_desc(channel) < 0)
0564                 dev_warn(dmac->dev, "ch: %d couldn't issue DMA xfer\n",
0565                      channel->index);
0566             else
0567                 list_move_tail(channel->ld_queue.next,
0568                            &channel->ld_active);
0569         }
0570     }
0571 
0572     spin_unlock_irqrestore(&channel->vc.lock, flags);
0573 }
0574 
0575 static u8 rz_dmac_ds_to_val_mapping(enum dma_slave_buswidth ds)
0576 {
0577     u8 i;
0578     static const enum dma_slave_buswidth ds_lut[] = {
0579         DMA_SLAVE_BUSWIDTH_1_BYTE,
0580         DMA_SLAVE_BUSWIDTH_2_BYTES,
0581         DMA_SLAVE_BUSWIDTH_4_BYTES,
0582         DMA_SLAVE_BUSWIDTH_8_BYTES,
0583         DMA_SLAVE_BUSWIDTH_16_BYTES,
0584         DMA_SLAVE_BUSWIDTH_32_BYTES,
0585         DMA_SLAVE_BUSWIDTH_64_BYTES,
0586         DMA_SLAVE_BUSWIDTH_128_BYTES,
0587     };
0588 
0589     for (i = 0; i < ARRAY_SIZE(ds_lut); i++) {
0590         if (ds_lut[i] == ds)
0591             return i;
0592     }
0593 
0594     return CHCFG_DS_INVALID;
0595 }
0596 
0597 static int rz_dmac_config(struct dma_chan *chan,
0598               struct dma_slave_config *config)
0599 {
0600     struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
0601     u32 val;
0602 
0603     channel->src_per_address = config->src_addr;
0604     channel->src_word_size = config->src_addr_width;
0605     channel->dst_per_address = config->dst_addr;
0606     channel->dst_word_size = config->dst_addr_width;
0607 
0608     val = rz_dmac_ds_to_val_mapping(config->dst_addr_width);
0609     if (val == CHCFG_DS_INVALID)
0610         return -EINVAL;
0611 
0612     channel->chcfg |= CHCFG_FILL_DDS(val);
0613 
0614     val = rz_dmac_ds_to_val_mapping(config->src_addr_width);
0615     if (val == CHCFG_DS_INVALID)
0616         return -EINVAL;
0617 
0618     channel->chcfg |= CHCFG_FILL_SDS(val);
0619 
0620     return 0;
0621 }
0622 
0623 static void rz_dmac_virt_desc_free(struct virt_dma_desc *vd)
0624 {
0625     /*
0626      * Place holder
0627      * Descriptor allocation is done during alloc_chan_resources and
0628      * get freed during free_chan_resources.
0629      * list is used to manage the descriptors and avoid any memory
0630      * allocation/free during DMA read/write.
0631      */
0632 }
0633 
0634 static void rz_dmac_device_synchronize(struct dma_chan *chan)
0635 {
0636     struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
0637     struct rz_dmac *dmac = to_rz_dmac(chan->device);
0638     u32 chstat;
0639     int ret;
0640 
0641     ret = read_poll_timeout(rz_dmac_ch_readl, chstat, !(chstat & CHSTAT_EN),
0642                 100, 100000, false, channel, CHSTAT, 1);
0643     if (ret < 0)
0644         dev_warn(dmac->dev, "DMA Timeout");
0645 
0646     rz_dmac_set_dmars_register(dmac, channel->index, 0);
0647 }
0648 
0649 /*
0650  * -----------------------------------------------------------------------------
0651  * IRQ handling
0652  */
0653 
0654 static void rz_dmac_irq_handle_channel(struct rz_dmac_chan *channel)
0655 {
0656     struct dma_chan *chan = &channel->vc.chan;
0657     struct rz_dmac *dmac = to_rz_dmac(chan->device);
0658     u32 chstat, chctrl;
0659 
0660     chstat = rz_dmac_ch_readl(channel, CHSTAT, 1);
0661     if (chstat & CHSTAT_ER) {
0662         dev_err(dmac->dev, "DMAC err CHSTAT_%d = %08X\n",
0663             channel->index, chstat);
0664         rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1);
0665         goto done;
0666     }
0667 
0668     chctrl = rz_dmac_ch_readl(channel, CHCTRL, 1);
0669     rz_dmac_ch_writel(channel, chctrl | CHCTRL_CLREND, CHCTRL, 1);
0670 done:
0671     return;
0672 }
0673 
0674 static irqreturn_t rz_dmac_irq_handler(int irq, void *dev_id)
0675 {
0676     struct rz_dmac_chan *channel = dev_id;
0677 
0678     if (channel) {
0679         rz_dmac_irq_handle_channel(channel);
0680         return IRQ_WAKE_THREAD;
0681     }
0682     /* handle DMAERR irq */
0683     return IRQ_HANDLED;
0684 }
0685 
0686 static irqreturn_t rz_dmac_irq_handler_thread(int irq, void *dev_id)
0687 {
0688     struct rz_dmac_chan *channel = dev_id;
0689     struct rz_dmac_desc *desc = NULL;
0690     unsigned long flags;
0691 
0692     spin_lock_irqsave(&channel->vc.lock, flags);
0693 
0694     if (list_empty(&channel->ld_active)) {
0695         /* Someone might have called terminate all */
0696         goto out;
0697     }
0698 
0699     desc = list_first_entry(&channel->ld_active, struct rz_dmac_desc, node);
0700     vchan_cookie_complete(&desc->vd);
0701     list_move_tail(channel->ld_active.next, &channel->ld_free);
0702     if (!list_empty(&channel->ld_queue)) {
0703         desc = list_first_entry(&channel->ld_queue, struct rz_dmac_desc,
0704                     node);
0705         channel->desc = desc;
0706         if (rz_dmac_xfer_desc(channel) == 0)
0707             list_move_tail(channel->ld_queue.next, &channel->ld_active);
0708     }
0709 out:
0710     spin_unlock_irqrestore(&channel->vc.lock, flags);
0711 
0712     return IRQ_HANDLED;
0713 }
0714 
0715 /*
0716  * -----------------------------------------------------------------------------
0717  * OF xlate and channel filter
0718  */
0719 
0720 static bool rz_dmac_chan_filter(struct dma_chan *chan, void *arg)
0721 {
0722     struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
0723     struct rz_dmac *dmac = to_rz_dmac(chan->device);
0724     struct of_phandle_args *dma_spec = arg;
0725     u32 ch_cfg;
0726 
0727     channel->mid_rid = dma_spec->args[0] & MID_RID_MASK;
0728     ch_cfg = (dma_spec->args[0] & CHCFG_MASK) >> 10;
0729     channel->chcfg = CHCFG_FILL_TM(ch_cfg) | CHCFG_FILL_AM(ch_cfg) |
0730              CHCFG_FILL_LVL(ch_cfg) | CHCFG_FILL_HIEN(ch_cfg);
0731 
0732     return !test_and_set_bit(channel->mid_rid, dmac->modules);
0733 }
0734 
0735 static struct dma_chan *rz_dmac_of_xlate(struct of_phandle_args *dma_spec,
0736                      struct of_dma *ofdma)
0737 {
0738     dma_cap_mask_t mask;
0739 
0740     if (dma_spec->args_count != 1)
0741         return NULL;
0742 
0743     /* Only slave DMA channels can be allocated via DT */
0744     dma_cap_zero(mask);
0745     dma_cap_set(DMA_SLAVE, mask);
0746 
0747     return dma_request_channel(mask, rz_dmac_chan_filter, dma_spec);
0748 }
0749 
0750 /*
0751  * -----------------------------------------------------------------------------
0752  * Probe and remove
0753  */
0754 
0755 static int rz_dmac_chan_probe(struct rz_dmac *dmac,
0756                   struct rz_dmac_chan *channel,
0757                   unsigned int index)
0758 {
0759     struct platform_device *pdev = to_platform_device(dmac->dev);
0760     struct rz_lmdesc *lmdesc;
0761     char pdev_irqname[5];
0762     char *irqname;
0763     int ret;
0764 
0765     channel->index = index;
0766     channel->mid_rid = -EINVAL;
0767 
0768     /* Request the channel interrupt. */
0769     sprintf(pdev_irqname, "ch%u", index);
0770     channel->irq = platform_get_irq_byname(pdev, pdev_irqname);
0771     if (channel->irq < 0)
0772         return channel->irq;
0773 
0774     irqname = devm_kasprintf(dmac->dev, GFP_KERNEL, "%s:%u",
0775                  dev_name(dmac->dev), index);
0776     if (!irqname)
0777         return -ENOMEM;
0778 
0779     ret = devm_request_threaded_irq(dmac->dev, channel->irq,
0780                     rz_dmac_irq_handler,
0781                     rz_dmac_irq_handler_thread, 0,
0782                     irqname, channel);
0783     if (ret) {
0784         dev_err(dmac->dev, "failed to request IRQ %u (%d)\n",
0785             channel->irq, ret);
0786         return ret;
0787     }
0788 
0789     /* Set io base address for each channel */
0790     if (index < 8) {
0791         channel->ch_base = dmac->base + CHANNEL_0_7_OFFSET +
0792             EACH_CHANNEL_OFFSET * index;
0793         channel->ch_cmn_base = dmac->base + CHANNEL_0_7_COMMON_BASE;
0794     } else {
0795         channel->ch_base = dmac->base + CHANNEL_8_15_OFFSET +
0796             EACH_CHANNEL_OFFSET * (index - 8);
0797         channel->ch_cmn_base = dmac->base + CHANNEL_8_15_COMMON_BASE;
0798     }
0799 
0800     /* Allocate descriptors */
0801     lmdesc = dma_alloc_coherent(&pdev->dev,
0802                     sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC,
0803                     &channel->lmdesc.base_dma, GFP_KERNEL);
0804     if (!lmdesc) {
0805         dev_err(&pdev->dev, "Can't allocate memory (lmdesc)\n");
0806         return -ENOMEM;
0807     }
0808     rz_lmdesc_setup(channel, lmdesc);
0809 
0810     /* Initialize register for each channel */
0811     rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1);
0812 
0813     channel->vc.desc_free = rz_dmac_virt_desc_free;
0814     vchan_init(&channel->vc, &dmac->engine);
0815     INIT_LIST_HEAD(&channel->ld_queue);
0816     INIT_LIST_HEAD(&channel->ld_free);
0817     INIT_LIST_HEAD(&channel->ld_active);
0818 
0819     return 0;
0820 }
0821 
0822 static int rz_dmac_parse_of(struct device *dev, struct rz_dmac *dmac)
0823 {
0824     struct device_node *np = dev->of_node;
0825     int ret;
0826 
0827     ret = of_property_read_u32(np, "dma-channels", &dmac->n_channels);
0828     if (ret < 0) {
0829         dev_err(dev, "unable to read dma-channels property\n");
0830         return ret;
0831     }
0832 
0833     if (!dmac->n_channels || dmac->n_channels > RZ_DMAC_MAX_CHANNELS) {
0834         dev_err(dev, "invalid number of channels %u\n", dmac->n_channels);
0835         return -EINVAL;
0836     }
0837 
0838     return 0;
0839 }
0840 
0841 static int rz_dmac_probe(struct platform_device *pdev)
0842 {
0843     const char *irqname = "error";
0844     struct dma_device *engine;
0845     struct rz_dmac *dmac;
0846     int channel_num;
0847     unsigned int i;
0848     int ret;
0849     int irq;
0850 
0851     dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL);
0852     if (!dmac)
0853         return -ENOMEM;
0854 
0855     dmac->dev = &pdev->dev;
0856     platform_set_drvdata(pdev, dmac);
0857 
0858     ret = rz_dmac_parse_of(&pdev->dev, dmac);
0859     if (ret < 0)
0860         return ret;
0861 
0862     dmac->channels = devm_kcalloc(&pdev->dev, dmac->n_channels,
0863                       sizeof(*dmac->channels), GFP_KERNEL);
0864     if (!dmac->channels)
0865         return -ENOMEM;
0866 
0867     /* Request resources */
0868     dmac->base = devm_platform_ioremap_resource(pdev, 0);
0869     if (IS_ERR(dmac->base))
0870         return PTR_ERR(dmac->base);
0871 
0872     dmac->ext_base = devm_platform_ioremap_resource(pdev, 1);
0873     if (IS_ERR(dmac->ext_base))
0874         return PTR_ERR(dmac->ext_base);
0875 
0876     /* Register interrupt handler for error */
0877     irq = platform_get_irq_byname(pdev, irqname);
0878     if (irq < 0)
0879         return irq;
0880 
0881     ret = devm_request_irq(&pdev->dev, irq, rz_dmac_irq_handler, 0,
0882                    irqname, NULL);
0883     if (ret) {
0884         dev_err(&pdev->dev, "failed to request IRQ %u (%d)\n",
0885             irq, ret);
0886         return ret;
0887     }
0888 
0889     /* Initialize the channels. */
0890     INIT_LIST_HEAD(&dmac->engine.channels);
0891 
0892     pm_runtime_enable(&pdev->dev);
0893     ret = pm_runtime_resume_and_get(&pdev->dev);
0894     if (ret < 0) {
0895         dev_err(&pdev->dev, "pm_runtime_resume_and_get failed\n");
0896         goto err_pm_disable;
0897     }
0898 
0899     for (i = 0; i < dmac->n_channels; i++) {
0900         ret = rz_dmac_chan_probe(dmac, &dmac->channels[i], i);
0901         if (ret < 0)
0902             goto err;
0903     }
0904 
0905     /* Register the DMAC as a DMA provider for DT. */
0906     ret = of_dma_controller_register(pdev->dev.of_node, rz_dmac_of_xlate,
0907                      NULL);
0908     if (ret < 0)
0909         goto err;
0910 
0911     /* Register the DMA engine device. */
0912     engine = &dmac->engine;
0913     dma_cap_set(DMA_SLAVE, engine->cap_mask);
0914     dma_cap_set(DMA_MEMCPY, engine->cap_mask);
0915     rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_0_7_COMMON_BASE + DCTRL);
0916     rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_8_15_COMMON_BASE + DCTRL);
0917 
0918     engine->dev = &pdev->dev;
0919 
0920     engine->device_alloc_chan_resources = rz_dmac_alloc_chan_resources;
0921     engine->device_free_chan_resources = rz_dmac_free_chan_resources;
0922     engine->device_tx_status = dma_cookie_status;
0923     engine->device_prep_slave_sg = rz_dmac_prep_slave_sg;
0924     engine->device_prep_dma_memcpy = rz_dmac_prep_dma_memcpy;
0925     engine->device_config = rz_dmac_config;
0926     engine->device_terminate_all = rz_dmac_terminate_all;
0927     engine->device_issue_pending = rz_dmac_issue_pending;
0928     engine->device_synchronize = rz_dmac_device_synchronize;
0929 
0930     engine->copy_align = DMAENGINE_ALIGN_1_BYTE;
0931     dma_set_max_seg_size(engine->dev, U32_MAX);
0932 
0933     ret = dma_async_device_register(engine);
0934     if (ret < 0) {
0935         dev_err(&pdev->dev, "unable to register\n");
0936         goto dma_register_err;
0937     }
0938     return 0;
0939 
0940 dma_register_err:
0941     of_dma_controller_free(pdev->dev.of_node);
0942 err:
0943     channel_num = i ? i - 1 : 0;
0944     for (i = 0; i < channel_num; i++) {
0945         struct rz_dmac_chan *channel = &dmac->channels[i];
0946 
0947         dma_free_coherent(&pdev->dev,
0948                   sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC,
0949                   channel->lmdesc.base,
0950                   channel->lmdesc.base_dma);
0951     }
0952 
0953     pm_runtime_put(&pdev->dev);
0954 err_pm_disable:
0955     pm_runtime_disable(&pdev->dev);
0956 
0957     return ret;
0958 }
0959 
0960 static int rz_dmac_remove(struct platform_device *pdev)
0961 {
0962     struct rz_dmac *dmac = platform_get_drvdata(pdev);
0963     unsigned int i;
0964 
0965     for (i = 0; i < dmac->n_channels; i++) {
0966         struct rz_dmac_chan *channel = &dmac->channels[i];
0967 
0968         dma_free_coherent(&pdev->dev,
0969                   sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC,
0970                   channel->lmdesc.base,
0971                   channel->lmdesc.base_dma);
0972     }
0973     of_dma_controller_free(pdev->dev.of_node);
0974     dma_async_device_unregister(&dmac->engine);
0975     pm_runtime_put(&pdev->dev);
0976     pm_runtime_disable(&pdev->dev);
0977 
0978     return 0;
0979 }
0980 
0981 static const struct of_device_id of_rz_dmac_match[] = {
0982     { .compatible = "renesas,rz-dmac", },
0983     { /* Sentinel */ }
0984 };
0985 MODULE_DEVICE_TABLE(of, of_rz_dmac_match);
0986 
0987 static struct platform_driver rz_dmac_driver = {
0988     .driver     = {
0989         .name   = "rz-dmac",
0990         .of_match_table = of_rz_dmac_match,
0991     },
0992     .probe      = rz_dmac_probe,
0993     .remove     = rz_dmac_remove,
0994 };
0995 
0996 module_platform_driver(rz_dmac_driver);
0997 
0998 MODULE_DESCRIPTION("Renesas RZ/G2L DMA Controller Driver");
0999 MODULE_AUTHOR("Biju Das <biju.das.jz@bp.renesas.com>");
1000 MODULE_LICENSE("GPL v2");
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!