Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * IMG Multi-threaded DMA Controller (MDC)
0004  *
0005  * Copyright (C) 2009,2012,2013 Imagination Technologies Ltd.
0006  * Copyright (C) 2014 Google, Inc.
0007  */
0008 
0009 #include <linux/clk.h>
0010 #include <linux/dma-mapping.h>
0011 #include <linux/dmaengine.h>
0012 #include <linux/dmapool.h>
0013 #include <linux/interrupt.h>
0014 #include <linux/io.h>
0015 #include <linux/irq.h>
0016 #include <linux/kernel.h>
0017 #include <linux/mfd/syscon.h>
0018 #include <linux/module.h>
0019 #include <linux/of.h>
0020 #include <linux/of_device.h>
0021 #include <linux/of_dma.h>
0022 #include <linux/platform_device.h>
0023 #include <linux/pm_runtime.h>
0024 #include <linux/regmap.h>
0025 #include <linux/slab.h>
0026 #include <linux/spinlock.h>
0027 
0028 #include "dmaengine.h"
0029 #include "virt-dma.h"
0030 
0031 #define MDC_MAX_DMA_CHANNELS            32
0032 
0033 #define MDC_GENERAL_CONFIG          0x000
0034 #define MDC_GENERAL_CONFIG_LIST_IEN     BIT(31)
0035 #define MDC_GENERAL_CONFIG_IEN          BIT(29)
0036 #define MDC_GENERAL_CONFIG_LEVEL_INT        BIT(28)
0037 #define MDC_GENERAL_CONFIG_INC_W        BIT(12)
0038 #define MDC_GENERAL_CONFIG_INC_R        BIT(8)
0039 #define MDC_GENERAL_CONFIG_PHYSICAL_W       BIT(7)
0040 #define MDC_GENERAL_CONFIG_WIDTH_W_SHIFT    4
0041 #define MDC_GENERAL_CONFIG_WIDTH_W_MASK     0x7
0042 #define MDC_GENERAL_CONFIG_PHYSICAL_R       BIT(3)
0043 #define MDC_GENERAL_CONFIG_WIDTH_R_SHIFT    0
0044 #define MDC_GENERAL_CONFIG_WIDTH_R_MASK     0x7
0045 
0046 #define MDC_READ_PORT_CONFIG            0x004
0047 #define MDC_READ_PORT_CONFIG_STHREAD_SHIFT  28
0048 #define MDC_READ_PORT_CONFIG_STHREAD_MASK   0xf
0049 #define MDC_READ_PORT_CONFIG_RTHREAD_SHIFT  24
0050 #define MDC_READ_PORT_CONFIG_RTHREAD_MASK   0xf
0051 #define MDC_READ_PORT_CONFIG_WTHREAD_SHIFT  16
0052 #define MDC_READ_PORT_CONFIG_WTHREAD_MASK   0xf
0053 #define MDC_READ_PORT_CONFIG_BURST_SIZE_SHIFT   4
0054 #define MDC_READ_PORT_CONFIG_BURST_SIZE_MASK    0xff
0055 #define MDC_READ_PORT_CONFIG_DREQ_ENABLE    BIT(1)
0056 
0057 #define MDC_READ_ADDRESS            0x008
0058 
0059 #define MDC_WRITE_ADDRESS           0x00c
0060 
0061 #define MDC_TRANSFER_SIZE           0x010
0062 #define MDC_TRANSFER_SIZE_MASK          0xffffff
0063 
0064 #define MDC_LIST_NODE_ADDRESS           0x014
0065 
0066 #define MDC_CMDS_PROCESSED          0x018
0067 #define MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT 16
0068 #define MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK  0x3f
0069 #define MDC_CMDS_PROCESSED_INT_ACTIVE       BIT(8)
0070 #define MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT  0
0071 #define MDC_CMDS_PROCESSED_CMDS_DONE_MASK   0x3f
0072 
0073 #define MDC_CONTROL_AND_STATUS          0x01c
0074 #define MDC_CONTROL_AND_STATUS_CANCEL       BIT(20)
0075 #define MDC_CONTROL_AND_STATUS_LIST_EN      BIT(4)
0076 #define MDC_CONTROL_AND_STATUS_EN       BIT(0)
0077 
0078 #define MDC_ACTIVE_TRANSFER_SIZE        0x030
0079 
0080 #define MDC_GLOBAL_CONFIG_A             0x900
0081 #define MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_SHIFT   16
0082 #define MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_MASK    0xff
0083 #define MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_SHIFT      8
0084 #define MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_MASK       0xff
0085 #define MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_SHIFT     0
0086 #define MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_MASK      0xff
0087 
0088 struct mdc_hw_list_desc {
0089     u32 gen_conf;
0090     u32 readport_conf;
0091     u32 read_addr;
0092     u32 write_addr;
0093     u32 xfer_size;
0094     u32 node_addr;
0095     u32 cmds_done;
0096     u32 ctrl_status;
0097     /*
0098      * Not part of the list descriptor, but instead used by the CPU to
0099      * traverse the list.
0100      */
0101     struct mdc_hw_list_desc *next_desc;
0102 };
0103 
0104 struct mdc_tx_desc {
0105     struct mdc_chan *chan;
0106     struct virt_dma_desc vd;
0107     dma_addr_t list_phys;
0108     struct mdc_hw_list_desc *list;
0109     bool cyclic;
0110     bool cmd_loaded;
0111     unsigned int list_len;
0112     unsigned int list_period_len;
0113     size_t list_xfer_size;
0114     unsigned int list_cmds_done;
0115 };
0116 
0117 struct mdc_chan {
0118     struct mdc_dma *mdma;
0119     struct virt_dma_chan vc;
0120     struct dma_slave_config config;
0121     struct mdc_tx_desc *desc;
0122     int irq;
0123     unsigned int periph;
0124     unsigned int thread;
0125     unsigned int chan_nr;
0126 };
0127 
0128 struct mdc_dma_soc_data {
0129     void (*enable_chan)(struct mdc_chan *mchan);
0130     void (*disable_chan)(struct mdc_chan *mchan);
0131 };
0132 
0133 struct mdc_dma {
0134     struct dma_device dma_dev;
0135     void __iomem *regs;
0136     struct clk *clk;
0137     struct dma_pool *desc_pool;
0138     struct regmap *periph_regs;
0139     spinlock_t lock;
0140     unsigned int nr_threads;
0141     unsigned int nr_channels;
0142     unsigned int bus_width;
0143     unsigned int max_burst_mult;
0144     unsigned int max_xfer_size;
0145     const struct mdc_dma_soc_data *soc;
0146     struct mdc_chan channels[MDC_MAX_DMA_CHANNELS];
0147 };
0148 
0149 static inline u32 mdc_readl(struct mdc_dma *mdma, u32 reg)
0150 {
0151     return readl(mdma->regs + reg);
0152 }
0153 
0154 static inline void mdc_writel(struct mdc_dma *mdma, u32 val, u32 reg)
0155 {
0156     writel(val, mdma->regs + reg);
0157 }
0158 
0159 static inline u32 mdc_chan_readl(struct mdc_chan *mchan, u32 reg)
0160 {
0161     return mdc_readl(mchan->mdma, mchan->chan_nr * 0x040 + reg);
0162 }
0163 
0164 static inline void mdc_chan_writel(struct mdc_chan *mchan, u32 val, u32 reg)
0165 {
0166     mdc_writel(mchan->mdma, val, mchan->chan_nr * 0x040 + reg);
0167 }
0168 
0169 static inline struct mdc_chan *to_mdc_chan(struct dma_chan *c)
0170 {
0171     return container_of(to_virt_chan(c), struct mdc_chan, vc);
0172 }
0173 
0174 static inline struct mdc_tx_desc *to_mdc_desc(struct dma_async_tx_descriptor *t)
0175 {
0176     struct virt_dma_desc *vdesc = container_of(t, struct virt_dma_desc, tx);
0177 
0178     return container_of(vdesc, struct mdc_tx_desc, vd);
0179 }
0180 
0181 static inline struct device *mdma2dev(struct mdc_dma *mdma)
0182 {
0183     return mdma->dma_dev.dev;
0184 }
0185 
0186 static inline unsigned int to_mdc_width(unsigned int bytes)
0187 {
0188     return ffs(bytes) - 1;
0189 }
0190 
0191 static inline void mdc_set_read_width(struct mdc_hw_list_desc *ldesc,
0192                       unsigned int bytes)
0193 {
0194     ldesc->gen_conf |= to_mdc_width(bytes) <<
0195         MDC_GENERAL_CONFIG_WIDTH_R_SHIFT;
0196 }
0197 
0198 static inline void mdc_set_write_width(struct mdc_hw_list_desc *ldesc,
0199                        unsigned int bytes)
0200 {
0201     ldesc->gen_conf |= to_mdc_width(bytes) <<
0202         MDC_GENERAL_CONFIG_WIDTH_W_SHIFT;
0203 }
0204 
0205 static void mdc_list_desc_config(struct mdc_chan *mchan,
0206                  struct mdc_hw_list_desc *ldesc,
0207                  enum dma_transfer_direction dir,
0208                  dma_addr_t src, dma_addr_t dst, size_t len)
0209 {
0210     struct mdc_dma *mdma = mchan->mdma;
0211     unsigned int max_burst, burst_size;
0212 
0213     ldesc->gen_conf = MDC_GENERAL_CONFIG_IEN | MDC_GENERAL_CONFIG_LIST_IEN |
0214         MDC_GENERAL_CONFIG_LEVEL_INT | MDC_GENERAL_CONFIG_PHYSICAL_W |
0215         MDC_GENERAL_CONFIG_PHYSICAL_R;
0216     ldesc->readport_conf =
0217         (mchan->thread << MDC_READ_PORT_CONFIG_STHREAD_SHIFT) |
0218         (mchan->thread << MDC_READ_PORT_CONFIG_RTHREAD_SHIFT) |
0219         (mchan->thread << MDC_READ_PORT_CONFIG_WTHREAD_SHIFT);
0220     ldesc->read_addr = src;
0221     ldesc->write_addr = dst;
0222     ldesc->xfer_size = len - 1;
0223     ldesc->node_addr = 0;
0224     ldesc->cmds_done = 0;
0225     ldesc->ctrl_status = MDC_CONTROL_AND_STATUS_LIST_EN |
0226         MDC_CONTROL_AND_STATUS_EN;
0227     ldesc->next_desc = NULL;
0228 
0229     if (IS_ALIGNED(dst, mdma->bus_width) &&
0230         IS_ALIGNED(src, mdma->bus_width))
0231         max_burst = mdma->bus_width * mdma->max_burst_mult;
0232     else
0233         max_burst = mdma->bus_width * (mdma->max_burst_mult - 1);
0234 
0235     if (dir == DMA_MEM_TO_DEV) {
0236         ldesc->gen_conf |= MDC_GENERAL_CONFIG_INC_R;
0237         ldesc->readport_conf |= MDC_READ_PORT_CONFIG_DREQ_ENABLE;
0238         mdc_set_read_width(ldesc, mdma->bus_width);
0239         mdc_set_write_width(ldesc, mchan->config.dst_addr_width);
0240         burst_size = min(max_burst, mchan->config.dst_maxburst *
0241                  mchan->config.dst_addr_width);
0242     } else if (dir == DMA_DEV_TO_MEM) {
0243         ldesc->gen_conf |= MDC_GENERAL_CONFIG_INC_W;
0244         ldesc->readport_conf |= MDC_READ_PORT_CONFIG_DREQ_ENABLE;
0245         mdc_set_read_width(ldesc, mchan->config.src_addr_width);
0246         mdc_set_write_width(ldesc, mdma->bus_width);
0247         burst_size = min(max_burst, mchan->config.src_maxburst *
0248                  mchan->config.src_addr_width);
0249     } else {
0250         ldesc->gen_conf |= MDC_GENERAL_CONFIG_INC_R |
0251             MDC_GENERAL_CONFIG_INC_W;
0252         mdc_set_read_width(ldesc, mdma->bus_width);
0253         mdc_set_write_width(ldesc, mdma->bus_width);
0254         burst_size = max_burst;
0255     }
0256     ldesc->readport_conf |= (burst_size - 1) <<
0257         MDC_READ_PORT_CONFIG_BURST_SIZE_SHIFT;
0258 }
0259 
0260 static void mdc_list_desc_free(struct mdc_tx_desc *mdesc)
0261 {
0262     struct mdc_dma *mdma = mdesc->chan->mdma;
0263     struct mdc_hw_list_desc *curr, *next;
0264     dma_addr_t curr_phys, next_phys;
0265 
0266     curr = mdesc->list;
0267     curr_phys = mdesc->list_phys;
0268     while (curr) {
0269         next = curr->next_desc;
0270         next_phys = curr->node_addr;
0271         dma_pool_free(mdma->desc_pool, curr, curr_phys);
0272         curr = next;
0273         curr_phys = next_phys;
0274     }
0275 }
0276 
0277 static void mdc_desc_free(struct virt_dma_desc *vd)
0278 {
0279     struct mdc_tx_desc *mdesc = to_mdc_desc(&vd->tx);
0280 
0281     mdc_list_desc_free(mdesc);
0282     kfree(mdesc);
0283 }
0284 
0285 static struct dma_async_tx_descriptor *mdc_prep_dma_memcpy(
0286     struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, size_t len,
0287     unsigned long flags)
0288 {
0289     struct mdc_chan *mchan = to_mdc_chan(chan);
0290     struct mdc_dma *mdma = mchan->mdma;
0291     struct mdc_tx_desc *mdesc;
0292     struct mdc_hw_list_desc *curr, *prev = NULL;
0293     dma_addr_t curr_phys;
0294 
0295     if (!len)
0296         return NULL;
0297 
0298     mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT);
0299     if (!mdesc)
0300         return NULL;
0301     mdesc->chan = mchan;
0302     mdesc->list_xfer_size = len;
0303 
0304     while (len > 0) {
0305         size_t xfer_size;
0306 
0307         curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT, &curr_phys);
0308         if (!curr)
0309             goto free_desc;
0310 
0311         if (prev) {
0312             prev->node_addr = curr_phys;
0313             prev->next_desc = curr;
0314         } else {
0315             mdesc->list_phys = curr_phys;
0316             mdesc->list = curr;
0317         }
0318 
0319         xfer_size = min_t(size_t, mdma->max_xfer_size, len);
0320 
0321         mdc_list_desc_config(mchan, curr, DMA_MEM_TO_MEM, src, dest,
0322                      xfer_size);
0323 
0324         prev = curr;
0325 
0326         mdesc->list_len++;
0327         src += xfer_size;
0328         dest += xfer_size;
0329         len -= xfer_size;
0330     }
0331 
0332     return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags);
0333 
0334 free_desc:
0335     mdc_desc_free(&mdesc->vd);
0336 
0337     return NULL;
0338 }
0339 
0340 static int mdc_check_slave_width(struct mdc_chan *mchan,
0341                  enum dma_transfer_direction dir)
0342 {
0343     enum dma_slave_buswidth width;
0344 
0345     if (dir == DMA_MEM_TO_DEV)
0346         width = mchan->config.dst_addr_width;
0347     else
0348         width = mchan->config.src_addr_width;
0349 
0350     switch (width) {
0351     case DMA_SLAVE_BUSWIDTH_1_BYTE:
0352     case DMA_SLAVE_BUSWIDTH_2_BYTES:
0353     case DMA_SLAVE_BUSWIDTH_4_BYTES:
0354     case DMA_SLAVE_BUSWIDTH_8_BYTES:
0355         break;
0356     default:
0357         return -EINVAL;
0358     }
0359 
0360     if (width > mchan->mdma->bus_width)
0361         return -EINVAL;
0362 
0363     return 0;
0364 }
0365 
0366 static struct dma_async_tx_descriptor *mdc_prep_dma_cyclic(
0367     struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
0368     size_t period_len, enum dma_transfer_direction dir,
0369     unsigned long flags)
0370 {
0371     struct mdc_chan *mchan = to_mdc_chan(chan);
0372     struct mdc_dma *mdma = mchan->mdma;
0373     struct mdc_tx_desc *mdesc;
0374     struct mdc_hw_list_desc *curr, *prev = NULL;
0375     dma_addr_t curr_phys;
0376 
0377     if (!buf_len && !period_len)
0378         return NULL;
0379 
0380     if (!is_slave_direction(dir))
0381         return NULL;
0382 
0383     if (mdc_check_slave_width(mchan, dir) < 0)
0384         return NULL;
0385 
0386     mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT);
0387     if (!mdesc)
0388         return NULL;
0389     mdesc->chan = mchan;
0390     mdesc->cyclic = true;
0391     mdesc->list_xfer_size = buf_len;
0392     mdesc->list_period_len = DIV_ROUND_UP(period_len,
0393                           mdma->max_xfer_size);
0394 
0395     while (buf_len > 0) {
0396         size_t remainder = min(period_len, buf_len);
0397 
0398         while (remainder > 0) {
0399             size_t xfer_size;
0400 
0401             curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT,
0402                           &curr_phys);
0403             if (!curr)
0404                 goto free_desc;
0405 
0406             if (!prev) {
0407                 mdesc->list_phys = curr_phys;
0408                 mdesc->list = curr;
0409             } else {
0410                 prev->node_addr = curr_phys;
0411                 prev->next_desc = curr;
0412             }
0413 
0414             xfer_size = min_t(size_t, mdma->max_xfer_size,
0415                       remainder);
0416 
0417             if (dir == DMA_MEM_TO_DEV) {
0418                 mdc_list_desc_config(mchan, curr, dir,
0419                              buf_addr,
0420                              mchan->config.dst_addr,
0421                              xfer_size);
0422             } else {
0423                 mdc_list_desc_config(mchan, curr, dir,
0424                              mchan->config.src_addr,
0425                              buf_addr,
0426                              xfer_size);
0427             }
0428 
0429             prev = curr;
0430 
0431             mdesc->list_len++;
0432             buf_addr += xfer_size;
0433             buf_len -= xfer_size;
0434             remainder -= xfer_size;
0435         }
0436     }
0437     prev->node_addr = mdesc->list_phys;
0438 
0439     return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags);
0440 
0441 free_desc:
0442     mdc_desc_free(&mdesc->vd);
0443 
0444     return NULL;
0445 }
0446 
0447 static struct dma_async_tx_descriptor *mdc_prep_slave_sg(
0448     struct dma_chan *chan, struct scatterlist *sgl,
0449     unsigned int sg_len, enum dma_transfer_direction dir,
0450     unsigned long flags, void *context)
0451 {
0452     struct mdc_chan *mchan = to_mdc_chan(chan);
0453     struct mdc_dma *mdma = mchan->mdma;
0454     struct mdc_tx_desc *mdesc;
0455     struct scatterlist *sg;
0456     struct mdc_hw_list_desc *curr, *prev = NULL;
0457     dma_addr_t curr_phys;
0458     unsigned int i;
0459 
0460     if (!sgl)
0461         return NULL;
0462 
0463     if (!is_slave_direction(dir))
0464         return NULL;
0465 
0466     if (mdc_check_slave_width(mchan, dir) < 0)
0467         return NULL;
0468 
0469     mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT);
0470     if (!mdesc)
0471         return NULL;
0472     mdesc->chan = mchan;
0473 
0474     for_each_sg(sgl, sg, sg_len, i) {
0475         dma_addr_t buf = sg_dma_address(sg);
0476         size_t buf_len = sg_dma_len(sg);
0477 
0478         while (buf_len > 0) {
0479             size_t xfer_size;
0480 
0481             curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT,
0482                           &curr_phys);
0483             if (!curr)
0484                 goto free_desc;
0485 
0486             if (!prev) {
0487                 mdesc->list_phys = curr_phys;
0488                 mdesc->list = curr;
0489             } else {
0490                 prev->node_addr = curr_phys;
0491                 prev->next_desc = curr;
0492             }
0493 
0494             xfer_size = min_t(size_t, mdma->max_xfer_size,
0495                       buf_len);
0496 
0497             if (dir == DMA_MEM_TO_DEV) {
0498                 mdc_list_desc_config(mchan, curr, dir, buf,
0499                              mchan->config.dst_addr,
0500                              xfer_size);
0501             } else {
0502                 mdc_list_desc_config(mchan, curr, dir,
0503                              mchan->config.src_addr,
0504                              buf, xfer_size);
0505             }
0506 
0507             prev = curr;
0508 
0509             mdesc->list_len++;
0510             mdesc->list_xfer_size += xfer_size;
0511             buf += xfer_size;
0512             buf_len -= xfer_size;
0513         }
0514     }
0515 
0516     return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags);
0517 
0518 free_desc:
0519     mdc_desc_free(&mdesc->vd);
0520 
0521     return NULL;
0522 }
0523 
0524 static void mdc_issue_desc(struct mdc_chan *mchan)
0525 {
0526     struct mdc_dma *mdma = mchan->mdma;
0527     struct virt_dma_desc *vd;
0528     struct mdc_tx_desc *mdesc;
0529     u32 val;
0530 
0531     vd = vchan_next_desc(&mchan->vc);
0532     if (!vd)
0533         return;
0534 
0535     list_del(&vd->node);
0536 
0537     mdesc = to_mdc_desc(&vd->tx);
0538     mchan->desc = mdesc;
0539 
0540     dev_dbg(mdma2dev(mdma), "Issuing descriptor on channel %d\n",
0541         mchan->chan_nr);
0542 
0543     mdma->soc->enable_chan(mchan);
0544 
0545     val = mdc_chan_readl(mchan, MDC_GENERAL_CONFIG);
0546     val |= MDC_GENERAL_CONFIG_LIST_IEN | MDC_GENERAL_CONFIG_IEN |
0547         MDC_GENERAL_CONFIG_LEVEL_INT | MDC_GENERAL_CONFIG_PHYSICAL_W |
0548         MDC_GENERAL_CONFIG_PHYSICAL_R;
0549     mdc_chan_writel(mchan, val, MDC_GENERAL_CONFIG);
0550     val = (mchan->thread << MDC_READ_PORT_CONFIG_STHREAD_SHIFT) |
0551         (mchan->thread << MDC_READ_PORT_CONFIG_RTHREAD_SHIFT) |
0552         (mchan->thread << MDC_READ_PORT_CONFIG_WTHREAD_SHIFT);
0553     mdc_chan_writel(mchan, val, MDC_READ_PORT_CONFIG);
0554     mdc_chan_writel(mchan, mdesc->list_phys, MDC_LIST_NODE_ADDRESS);
0555     val = mdc_chan_readl(mchan, MDC_CONTROL_AND_STATUS);
0556     val |= MDC_CONTROL_AND_STATUS_LIST_EN;
0557     mdc_chan_writel(mchan, val, MDC_CONTROL_AND_STATUS);
0558 }
0559 
0560 static void mdc_issue_pending(struct dma_chan *chan)
0561 {
0562     struct mdc_chan *mchan = to_mdc_chan(chan);
0563     unsigned long flags;
0564 
0565     spin_lock_irqsave(&mchan->vc.lock, flags);
0566     if (vchan_issue_pending(&mchan->vc) && !mchan->desc)
0567         mdc_issue_desc(mchan);
0568     spin_unlock_irqrestore(&mchan->vc.lock, flags);
0569 }
0570 
0571 static enum dma_status mdc_tx_status(struct dma_chan *chan,
0572     dma_cookie_t cookie, struct dma_tx_state *txstate)
0573 {
0574     struct mdc_chan *mchan = to_mdc_chan(chan);
0575     struct mdc_tx_desc *mdesc;
0576     struct virt_dma_desc *vd;
0577     unsigned long flags;
0578     size_t bytes = 0;
0579     int ret;
0580 
0581     ret = dma_cookie_status(chan, cookie, txstate);
0582     if (ret == DMA_COMPLETE)
0583         return ret;
0584 
0585     if (!txstate)
0586         return ret;
0587 
0588     spin_lock_irqsave(&mchan->vc.lock, flags);
0589     vd = vchan_find_desc(&mchan->vc, cookie);
0590     if (vd) {
0591         mdesc = to_mdc_desc(&vd->tx);
0592         bytes = mdesc->list_xfer_size;
0593     } else if (mchan->desc && mchan->desc->vd.tx.cookie == cookie) {
0594         struct mdc_hw_list_desc *ldesc;
0595         u32 val1, val2, done, processed, residue;
0596         int i, cmds;
0597 
0598         mdesc = mchan->desc;
0599 
0600         /*
0601          * Determine the number of commands that haven't been
0602          * processed (handled by the IRQ handler) yet.
0603          */
0604         do {
0605             val1 = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED) &
0606                 ~MDC_CMDS_PROCESSED_INT_ACTIVE;
0607             residue = mdc_chan_readl(mchan,
0608                          MDC_ACTIVE_TRANSFER_SIZE);
0609             val2 = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED) &
0610                 ~MDC_CMDS_PROCESSED_INT_ACTIVE;
0611         } while (val1 != val2);
0612 
0613         done = (val1 >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) &
0614             MDC_CMDS_PROCESSED_CMDS_DONE_MASK;
0615         processed = (val1 >> MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) &
0616             MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK;
0617         cmds = (done - processed) %
0618             (MDC_CMDS_PROCESSED_CMDS_DONE_MASK + 1);
0619 
0620         /*
0621          * If the command loaded event hasn't been processed yet, then
0622          * the difference above includes an extra command.
0623          */
0624         if (!mdesc->cmd_loaded)
0625             cmds--;
0626         else
0627             cmds += mdesc->list_cmds_done;
0628 
0629         bytes = mdesc->list_xfer_size;
0630         ldesc = mdesc->list;
0631         for (i = 0; i < cmds; i++) {
0632             bytes -= ldesc->xfer_size + 1;
0633             ldesc = ldesc->next_desc;
0634         }
0635         if (ldesc) {
0636             if (residue != MDC_TRANSFER_SIZE_MASK)
0637                 bytes -= ldesc->xfer_size - residue;
0638             else
0639                 bytes -= ldesc->xfer_size + 1;
0640         }
0641     }
0642     spin_unlock_irqrestore(&mchan->vc.lock, flags);
0643 
0644     dma_set_residue(txstate, bytes);
0645 
0646     return ret;
0647 }
0648 
0649 static unsigned int mdc_get_new_events(struct mdc_chan *mchan)
0650 {
0651     u32 val, processed, done1, done2;
0652     unsigned int ret;
0653 
0654     val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED);
0655     processed = (val >> MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) &
0656                 MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK;
0657     /*
0658      * CMDS_DONE may have incremented between reading CMDS_PROCESSED
0659      * and clearing INT_ACTIVE.  Re-read CMDS_PROCESSED to ensure we
0660      * didn't miss a command completion.
0661      */
0662     do {
0663         val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED);
0664 
0665         done1 = (val >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) &
0666             MDC_CMDS_PROCESSED_CMDS_DONE_MASK;
0667 
0668         val &= ~((MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK <<
0669               MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) |
0670              MDC_CMDS_PROCESSED_INT_ACTIVE);
0671 
0672         val |= done1 << MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT;
0673 
0674         mdc_chan_writel(mchan, val, MDC_CMDS_PROCESSED);
0675 
0676         val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED);
0677 
0678         done2 = (val >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) &
0679             MDC_CMDS_PROCESSED_CMDS_DONE_MASK;
0680     } while (done1 != done2);
0681 
0682     if (done1 >= processed)
0683         ret = done1 - processed;
0684     else
0685         ret = ((MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK + 1) -
0686             processed) + done1;
0687 
0688     return ret;
0689 }
0690 
0691 static int mdc_terminate_all(struct dma_chan *chan)
0692 {
0693     struct mdc_chan *mchan = to_mdc_chan(chan);
0694     unsigned long flags;
0695     LIST_HEAD(head);
0696 
0697     spin_lock_irqsave(&mchan->vc.lock, flags);
0698 
0699     mdc_chan_writel(mchan, MDC_CONTROL_AND_STATUS_CANCEL,
0700             MDC_CONTROL_AND_STATUS);
0701 
0702     if (mchan->desc) {
0703         vchan_terminate_vdesc(&mchan->desc->vd);
0704         mchan->desc = NULL;
0705     }
0706     vchan_get_all_descriptors(&mchan->vc, &head);
0707 
0708     mdc_get_new_events(mchan);
0709 
0710     spin_unlock_irqrestore(&mchan->vc.lock, flags);
0711 
0712     vchan_dma_desc_free_list(&mchan->vc, &head);
0713 
0714     return 0;
0715 }
0716 
0717 static void mdc_synchronize(struct dma_chan *chan)
0718 {
0719     struct mdc_chan *mchan = to_mdc_chan(chan);
0720 
0721     vchan_synchronize(&mchan->vc);
0722 }
0723 
0724 static int mdc_slave_config(struct dma_chan *chan,
0725                 struct dma_slave_config *config)
0726 {
0727     struct mdc_chan *mchan = to_mdc_chan(chan);
0728     unsigned long flags;
0729 
0730     spin_lock_irqsave(&mchan->vc.lock, flags);
0731     mchan->config = *config;
0732     spin_unlock_irqrestore(&mchan->vc.lock, flags);
0733 
0734     return 0;
0735 }
0736 
0737 static int mdc_alloc_chan_resources(struct dma_chan *chan)
0738 {
0739     struct mdc_chan *mchan = to_mdc_chan(chan);
0740     struct device *dev = mdma2dev(mchan->mdma);
0741 
0742     return pm_runtime_get_sync(dev);
0743 }
0744 
0745 static void mdc_free_chan_resources(struct dma_chan *chan)
0746 {
0747     struct mdc_chan *mchan = to_mdc_chan(chan);
0748     struct mdc_dma *mdma = mchan->mdma;
0749     struct device *dev = mdma2dev(mdma);
0750 
0751     mdc_terminate_all(chan);
0752     mdma->soc->disable_chan(mchan);
0753     pm_runtime_put(dev);
0754 }
0755 
0756 static irqreturn_t mdc_chan_irq(int irq, void *dev_id)
0757 {
0758     struct mdc_chan *mchan = (struct mdc_chan *)dev_id;
0759     struct mdc_tx_desc *mdesc;
0760     unsigned int i, new_events;
0761 
0762     spin_lock(&mchan->vc.lock);
0763 
0764     dev_dbg(mdma2dev(mchan->mdma), "IRQ on channel %d\n", mchan->chan_nr);
0765 
0766     new_events = mdc_get_new_events(mchan);
0767 
0768     if (!new_events)
0769         goto out;
0770 
0771     mdesc = mchan->desc;
0772     if (!mdesc) {
0773         dev_warn(mdma2dev(mchan->mdma),
0774              "IRQ with no active descriptor on channel %d\n",
0775              mchan->chan_nr);
0776         goto out;
0777     }
0778 
0779     for (i = 0; i < new_events; i++) {
0780         /*
0781          * The first interrupt in a transfer indicates that the
0782          * command list has been loaded, not that a command has
0783          * been completed.
0784          */
0785         if (!mdesc->cmd_loaded) {
0786             mdesc->cmd_loaded = true;
0787             continue;
0788         }
0789 
0790         mdesc->list_cmds_done++;
0791         if (mdesc->cyclic) {
0792             mdesc->list_cmds_done %= mdesc->list_len;
0793             if (mdesc->list_cmds_done % mdesc->list_period_len == 0)
0794                 vchan_cyclic_callback(&mdesc->vd);
0795         } else if (mdesc->list_cmds_done == mdesc->list_len) {
0796             mchan->desc = NULL;
0797             vchan_cookie_complete(&mdesc->vd);
0798             mdc_issue_desc(mchan);
0799             break;
0800         }
0801     }
0802 out:
0803     spin_unlock(&mchan->vc.lock);
0804 
0805     return IRQ_HANDLED;
0806 }
0807 
0808 static struct dma_chan *mdc_of_xlate(struct of_phandle_args *dma_spec,
0809                      struct of_dma *ofdma)
0810 {
0811     struct mdc_dma *mdma = ofdma->of_dma_data;
0812     struct dma_chan *chan;
0813 
0814     if (dma_spec->args_count != 3)
0815         return NULL;
0816 
0817     list_for_each_entry(chan, &mdma->dma_dev.channels, device_node) {
0818         struct mdc_chan *mchan = to_mdc_chan(chan);
0819 
0820         if (!(dma_spec->args[1] & BIT(mchan->chan_nr)))
0821             continue;
0822         if (dma_get_slave_channel(chan)) {
0823             mchan->periph = dma_spec->args[0];
0824             mchan->thread = dma_spec->args[2];
0825             return chan;
0826         }
0827     }
0828 
0829     return NULL;
0830 }
0831 
0832 #define PISTACHIO_CR_PERIPH_DMA_ROUTE(ch)   (0x120 + 0x4 * ((ch) / 4))
0833 #define PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(ch) (8 * ((ch) % 4))
0834 #define PISTACHIO_CR_PERIPH_DMA_ROUTE_MASK  0x3f
0835 
0836 static void pistachio_mdc_enable_chan(struct mdc_chan *mchan)
0837 {
0838     struct mdc_dma *mdma = mchan->mdma;
0839 
0840     regmap_update_bits(mdma->periph_regs,
0841                PISTACHIO_CR_PERIPH_DMA_ROUTE(mchan->chan_nr),
0842                PISTACHIO_CR_PERIPH_DMA_ROUTE_MASK <<
0843                PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr),
0844                mchan->periph <<
0845                PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr));
0846 }
0847 
0848 static void pistachio_mdc_disable_chan(struct mdc_chan *mchan)
0849 {
0850     struct mdc_dma *mdma = mchan->mdma;
0851 
0852     regmap_update_bits(mdma->periph_regs,
0853                PISTACHIO_CR_PERIPH_DMA_ROUTE(mchan->chan_nr),
0854                PISTACHIO_CR_PERIPH_DMA_ROUTE_MASK <<
0855                PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr),
0856                0);
0857 }
0858 
0859 static const struct mdc_dma_soc_data pistachio_mdc_data = {
0860     .enable_chan = pistachio_mdc_enable_chan,
0861     .disable_chan = pistachio_mdc_disable_chan,
0862 };
0863 
0864 static const struct of_device_id mdc_dma_of_match[] = {
0865     { .compatible = "img,pistachio-mdc-dma", .data = &pistachio_mdc_data, },
0866     { },
0867 };
0868 MODULE_DEVICE_TABLE(of, mdc_dma_of_match);
0869 
0870 static int img_mdc_runtime_suspend(struct device *dev)
0871 {
0872     struct mdc_dma *mdma = dev_get_drvdata(dev);
0873 
0874     clk_disable_unprepare(mdma->clk);
0875 
0876     return 0;
0877 }
0878 
0879 static int img_mdc_runtime_resume(struct device *dev)
0880 {
0881     struct mdc_dma *mdma = dev_get_drvdata(dev);
0882 
0883     return clk_prepare_enable(mdma->clk);
0884 }
0885 
0886 static int mdc_dma_probe(struct platform_device *pdev)
0887 {
0888     struct mdc_dma *mdma;
0889     struct resource *res;
0890     unsigned int i;
0891     u32 val;
0892     int ret;
0893 
0894     mdma = devm_kzalloc(&pdev->dev, sizeof(*mdma), GFP_KERNEL);
0895     if (!mdma)
0896         return -ENOMEM;
0897     platform_set_drvdata(pdev, mdma);
0898 
0899     mdma->soc = of_device_get_match_data(&pdev->dev);
0900 
0901     res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
0902     mdma->regs = devm_ioremap_resource(&pdev->dev, res);
0903     if (IS_ERR(mdma->regs))
0904         return PTR_ERR(mdma->regs);
0905 
0906     mdma->periph_regs = syscon_regmap_lookup_by_phandle(pdev->dev.of_node,
0907                                 "img,cr-periph");
0908     if (IS_ERR(mdma->periph_regs))
0909         return PTR_ERR(mdma->periph_regs);
0910 
0911     mdma->clk = devm_clk_get(&pdev->dev, "sys");
0912     if (IS_ERR(mdma->clk))
0913         return PTR_ERR(mdma->clk);
0914 
0915     dma_cap_zero(mdma->dma_dev.cap_mask);
0916     dma_cap_set(DMA_SLAVE, mdma->dma_dev.cap_mask);
0917     dma_cap_set(DMA_PRIVATE, mdma->dma_dev.cap_mask);
0918     dma_cap_set(DMA_CYCLIC, mdma->dma_dev.cap_mask);
0919     dma_cap_set(DMA_MEMCPY, mdma->dma_dev.cap_mask);
0920 
0921     val = mdc_readl(mdma, MDC_GLOBAL_CONFIG_A);
0922     mdma->nr_channels = (val >> MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_SHIFT) &
0923         MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_MASK;
0924     mdma->nr_threads =
0925         1 << ((val >> MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_SHIFT) &
0926               MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_MASK);
0927     mdma->bus_width =
0928         (1 << ((val >> MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_SHIFT) &
0929                MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_MASK)) / 8;
0930     /*
0931      * Although transfer sizes of up to MDC_TRANSFER_SIZE_MASK + 1 bytes
0932      * are supported, this makes it possible for the value reported in
0933      * MDC_ACTIVE_TRANSFER_SIZE to be ambiguous - an active transfer size
0934      * of MDC_TRANSFER_SIZE_MASK may indicate either that 0 bytes or
0935      * MDC_TRANSFER_SIZE_MASK + 1 bytes are remaining.  To eliminate this
0936      * ambiguity, restrict transfer sizes to one bus-width less than the
0937      * actual maximum.
0938      */
0939     mdma->max_xfer_size = MDC_TRANSFER_SIZE_MASK + 1 - mdma->bus_width;
0940 
0941     of_property_read_u32(pdev->dev.of_node, "dma-channels",
0942                  &mdma->nr_channels);
0943     ret = of_property_read_u32(pdev->dev.of_node,
0944                    "img,max-burst-multiplier",
0945                    &mdma->max_burst_mult);
0946     if (ret)
0947         return ret;
0948 
0949     mdma->dma_dev.dev = &pdev->dev;
0950     mdma->dma_dev.device_prep_slave_sg = mdc_prep_slave_sg;
0951     mdma->dma_dev.device_prep_dma_cyclic = mdc_prep_dma_cyclic;
0952     mdma->dma_dev.device_prep_dma_memcpy = mdc_prep_dma_memcpy;
0953     mdma->dma_dev.device_alloc_chan_resources = mdc_alloc_chan_resources;
0954     mdma->dma_dev.device_free_chan_resources = mdc_free_chan_resources;
0955     mdma->dma_dev.device_tx_status = mdc_tx_status;
0956     mdma->dma_dev.device_issue_pending = mdc_issue_pending;
0957     mdma->dma_dev.device_terminate_all = mdc_terminate_all;
0958     mdma->dma_dev.device_synchronize = mdc_synchronize;
0959     mdma->dma_dev.device_config = mdc_slave_config;
0960 
0961     mdma->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
0962     mdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
0963     for (i = 1; i <= mdma->bus_width; i <<= 1) {
0964         mdma->dma_dev.src_addr_widths |= BIT(i);
0965         mdma->dma_dev.dst_addr_widths |= BIT(i);
0966     }
0967 
0968     INIT_LIST_HEAD(&mdma->dma_dev.channels);
0969     for (i = 0; i < mdma->nr_channels; i++) {
0970         struct mdc_chan *mchan = &mdma->channels[i];
0971 
0972         mchan->mdma = mdma;
0973         mchan->chan_nr = i;
0974         mchan->irq = platform_get_irq(pdev, i);
0975         if (mchan->irq < 0)
0976             return mchan->irq;
0977 
0978         ret = devm_request_irq(&pdev->dev, mchan->irq, mdc_chan_irq,
0979                        IRQ_TYPE_LEVEL_HIGH,
0980                        dev_name(&pdev->dev), mchan);
0981         if (ret < 0)
0982             return ret;
0983 
0984         mchan->vc.desc_free = mdc_desc_free;
0985         vchan_init(&mchan->vc, &mdma->dma_dev);
0986     }
0987 
0988     mdma->desc_pool = dmam_pool_create(dev_name(&pdev->dev), &pdev->dev,
0989                        sizeof(struct mdc_hw_list_desc),
0990                        4, 0);
0991     if (!mdma->desc_pool)
0992         return -ENOMEM;
0993 
0994     pm_runtime_enable(&pdev->dev);
0995     if (!pm_runtime_enabled(&pdev->dev)) {
0996         ret = img_mdc_runtime_resume(&pdev->dev);
0997         if (ret)
0998             return ret;
0999     }
1000 
1001     ret = dma_async_device_register(&mdma->dma_dev);
1002     if (ret)
1003         goto suspend;
1004 
1005     ret = of_dma_controller_register(pdev->dev.of_node, mdc_of_xlate, mdma);
1006     if (ret)
1007         goto unregister;
1008 
1009     dev_info(&pdev->dev, "MDC with %u channels and %u threads\n",
1010          mdma->nr_channels, mdma->nr_threads);
1011 
1012     return 0;
1013 
1014 unregister:
1015     dma_async_device_unregister(&mdma->dma_dev);
1016 suspend:
1017     if (!pm_runtime_enabled(&pdev->dev))
1018         img_mdc_runtime_suspend(&pdev->dev);
1019     pm_runtime_disable(&pdev->dev);
1020     return ret;
1021 }
1022 
1023 static int mdc_dma_remove(struct platform_device *pdev)
1024 {
1025     struct mdc_dma *mdma = platform_get_drvdata(pdev);
1026     struct mdc_chan *mchan, *next;
1027 
1028     of_dma_controller_free(pdev->dev.of_node);
1029     dma_async_device_unregister(&mdma->dma_dev);
1030 
1031     list_for_each_entry_safe(mchan, next, &mdma->dma_dev.channels,
1032                  vc.chan.device_node) {
1033         list_del(&mchan->vc.chan.device_node);
1034 
1035         devm_free_irq(&pdev->dev, mchan->irq, mchan);
1036 
1037         tasklet_kill(&mchan->vc.task);
1038     }
1039 
1040     pm_runtime_disable(&pdev->dev);
1041     if (!pm_runtime_status_suspended(&pdev->dev))
1042         img_mdc_runtime_suspend(&pdev->dev);
1043 
1044     return 0;
1045 }
1046 
1047 #ifdef CONFIG_PM_SLEEP
1048 static int img_mdc_suspend_late(struct device *dev)
1049 {
1050     struct mdc_dma *mdma = dev_get_drvdata(dev);
1051     int i;
1052 
1053     /* Check that all channels are idle */
1054     for (i = 0; i < mdma->nr_channels; i++) {
1055         struct mdc_chan *mchan = &mdma->channels[i];
1056 
1057         if (unlikely(mchan->desc))
1058             return -EBUSY;
1059     }
1060 
1061     return pm_runtime_force_suspend(dev);
1062 }
1063 
1064 static int img_mdc_resume_early(struct device *dev)
1065 {
1066     return pm_runtime_force_resume(dev);
1067 }
1068 #endif /* CONFIG_PM_SLEEP */
1069 
1070 static const struct dev_pm_ops img_mdc_pm_ops = {
1071     SET_RUNTIME_PM_OPS(img_mdc_runtime_suspend,
1072                img_mdc_runtime_resume, NULL)
1073     SET_LATE_SYSTEM_SLEEP_PM_OPS(img_mdc_suspend_late,
1074                      img_mdc_resume_early)
1075 };
1076 
1077 static struct platform_driver mdc_dma_driver = {
1078     .driver = {
1079         .name = "img-mdc-dma",
1080         .pm = &img_mdc_pm_ops,
1081         .of_match_table = of_match_ptr(mdc_dma_of_match),
1082     },
1083     .probe = mdc_dma_probe,
1084     .remove = mdc_dma_remove,
1085 };
1086 module_platform_driver(mdc_dma_driver);
1087 
1088 MODULE_DESCRIPTION("IMG Multi-threaded DMA Controller (MDC) driver");
1089 MODULE_AUTHOR("Andrew Bresticker <abrestic@chromium.org>");
1090 MODULE_LICENSE("GPL v2");
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!