Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * ADMA driver for Nvidia's Tegra210 ADMA controller.
0004  *
0005  * Copyright (c) 2016, NVIDIA CORPORATION.  All rights reserved.
0006  */
0007 
0008 #include <linux/clk.h>
0009 #include <linux/iopoll.h>
0010 #include <linux/module.h>
0011 #include <linux/of_device.h>
0012 #include <linux/of_dma.h>
0013 #include <linux/of_irq.h>
0014 #include <linux/pm_runtime.h>
0015 #include <linux/slab.h>
0016 
0017 #include "virt-dma.h"
0018 
0019 #define ADMA_CH_CMD                 0x00
0020 #define ADMA_CH_STATUS                  0x0c
0021 #define ADMA_CH_STATUS_XFER_EN              BIT(0)
0022 #define ADMA_CH_STATUS_XFER_PAUSED          BIT(1)
0023 
0024 #define ADMA_CH_INT_STATUS              0x10
0025 #define ADMA_CH_INT_STATUS_XFER_DONE            BIT(0)
0026 
0027 #define ADMA_CH_INT_CLEAR               0x1c
0028 #define ADMA_CH_CTRL                    0x24
0029 #define ADMA_CH_CTRL_DIR(val)               (((val) & 0xf) << 12)
0030 #define ADMA_CH_CTRL_DIR_AHUB2MEM           2
0031 #define ADMA_CH_CTRL_DIR_MEM2AHUB           4
0032 #define ADMA_CH_CTRL_MODE_CONTINUOUS            (2 << 8)
0033 #define ADMA_CH_CTRL_FLOWCTRL_EN            BIT(1)
0034 #define ADMA_CH_CTRL_XFER_PAUSE_SHIFT           0
0035 
0036 #define ADMA_CH_CONFIG                  0x28
0037 #define ADMA_CH_CONFIG_SRC_BUF(val)         (((val) & 0x7) << 28)
0038 #define ADMA_CH_CONFIG_TRG_BUF(val)         (((val) & 0x7) << 24)
0039 #define ADMA_CH_CONFIG_BURST_SIZE_SHIFT         20
0040 #define ADMA_CH_CONFIG_MAX_BURST_SIZE                   16
0041 #define ADMA_CH_CONFIG_WEIGHT_FOR_WRR(val)      ((val) & 0xf)
0042 #define ADMA_CH_CONFIG_MAX_BUFS             8
0043 #define TEGRA186_ADMA_CH_CONFIG_OUTSTANDING_REQS(reqs)  (reqs << 4)
0044 
0045 #define ADMA_CH_FIFO_CTRL               0x2c
0046 #define ADMA_CH_TX_FIFO_SIZE_SHIFT          8
0047 #define ADMA_CH_RX_FIFO_SIZE_SHIFT          0
0048 
0049 #define ADMA_CH_LOWER_SRC_ADDR              0x34
0050 #define ADMA_CH_LOWER_TRG_ADDR              0x3c
0051 #define ADMA_CH_TC                  0x44
0052 #define ADMA_CH_TC_COUNT_MASK               0x3ffffffc
0053 
0054 #define ADMA_CH_XFER_STATUS             0x54
0055 #define ADMA_CH_XFER_STATUS_COUNT_MASK          0xffff
0056 
0057 #define ADMA_GLOBAL_CMD                 0x00
0058 #define ADMA_GLOBAL_SOFT_RESET              0x04
0059 
0060 #define TEGRA_ADMA_BURST_COMPLETE_TIME          20
0061 
0062 #define ADMA_CH_REG_FIELD_VAL(val, mask, shift) (((val) & mask) << shift)
0063 
0064 struct tegra_adma;
0065 
0066 /*
0067  * struct tegra_adma_chip_data - Tegra chip specific data
0068  * @adma_get_burst_config: Function callback used to set DMA burst size.
0069  * @global_reg_offset: Register offset of DMA global register.
0070  * @global_int_clear: Register offset of DMA global interrupt clear.
0071  * @ch_req_tx_shift: Register offset for AHUB transmit channel select.
0072  * @ch_req_rx_shift: Register offset for AHUB receive channel select.
0073  * @ch_base_offset: Register offset of DMA channel registers.
0074  * @ch_fifo_ctrl: Default value for channel FIFO CTRL register.
0075  * @ch_req_mask: Mask for Tx or Rx channel select.
0076  * @ch_req_max: Maximum number of Tx or Rx channels available.
0077  * @ch_reg_size: Size of DMA channel register space.
0078  * @nr_channels: Number of DMA channels available.
0079  * @ch_fifo_size_mask: Mask for FIFO size field.
0080  * @sreq_index_offset: Slave channel index offset.
0081  * @has_outstanding_reqs: If DMA channel can have outstanding requests.
0082  */
0083 struct tegra_adma_chip_data {
0084     unsigned int (*adma_get_burst_config)(unsigned int burst_size);
0085     unsigned int global_reg_offset;
0086     unsigned int global_int_clear;
0087     unsigned int ch_req_tx_shift;
0088     unsigned int ch_req_rx_shift;
0089     unsigned int ch_base_offset;
0090     unsigned int ch_fifo_ctrl;
0091     unsigned int ch_req_mask;
0092     unsigned int ch_req_max;
0093     unsigned int ch_reg_size;
0094     unsigned int nr_channels;
0095     unsigned int ch_fifo_size_mask;
0096     unsigned int sreq_index_offset;
0097     bool has_outstanding_reqs;
0098 };
0099 
0100 /*
0101  * struct tegra_adma_chan_regs - Tegra ADMA channel registers
0102  */
0103 struct tegra_adma_chan_regs {
0104     unsigned int ctrl;
0105     unsigned int config;
0106     unsigned int src_addr;
0107     unsigned int trg_addr;
0108     unsigned int fifo_ctrl;
0109     unsigned int cmd;
0110     unsigned int tc;
0111 };
0112 
0113 /*
0114  * struct tegra_adma_desc - Tegra ADMA descriptor to manage transfer requests.
0115  */
0116 struct tegra_adma_desc {
0117     struct virt_dma_desc        vd;
0118     struct tegra_adma_chan_regs ch_regs;
0119     size_t              buf_len;
0120     size_t              period_len;
0121     size_t              num_periods;
0122 };
0123 
0124 /*
0125  * struct tegra_adma_chan - Tegra ADMA channel information
0126  */
0127 struct tegra_adma_chan {
0128     struct virt_dma_chan        vc;
0129     struct tegra_adma_desc      *desc;
0130     struct tegra_adma       *tdma;
0131     int             irq;
0132     void __iomem            *chan_addr;
0133 
0134     /* Slave channel configuration info */
0135     struct dma_slave_config     sconfig;
0136     enum dma_transfer_direction sreq_dir;
0137     unsigned int            sreq_index;
0138     bool                sreq_reserved;
0139     struct tegra_adma_chan_regs ch_regs;
0140 
0141     /* Transfer count and position info */
0142     unsigned int            tx_buf_count;
0143     unsigned int            tx_buf_pos;
0144 };
0145 
0146 /*
0147  * struct tegra_adma - Tegra ADMA controller information
0148  */
0149 struct tegra_adma {
0150     struct dma_device       dma_dev;
0151     struct device           *dev;
0152     void __iomem            *base_addr;
0153     struct clk          *ahub_clk;
0154     unsigned int            nr_channels;
0155     unsigned long           rx_requests_reserved;
0156     unsigned long           tx_requests_reserved;
0157 
0158     /* Used to store global command register state when suspending */
0159     unsigned int            global_cmd;
0160 
0161     const struct tegra_adma_chip_data *cdata;
0162 
0163     /* Last member of the structure */
0164     struct tegra_adma_chan      channels[];
0165 };
0166 
0167 static inline void tdma_write(struct tegra_adma *tdma, u32 reg, u32 val)
0168 {
0169     writel(val, tdma->base_addr + tdma->cdata->global_reg_offset + reg);
0170 }
0171 
0172 static inline u32 tdma_read(struct tegra_adma *tdma, u32 reg)
0173 {
0174     return readl(tdma->base_addr + tdma->cdata->global_reg_offset + reg);
0175 }
0176 
0177 static inline void tdma_ch_write(struct tegra_adma_chan *tdc, u32 reg, u32 val)
0178 {
0179     writel(val, tdc->chan_addr + reg);
0180 }
0181 
0182 static inline u32 tdma_ch_read(struct tegra_adma_chan *tdc, u32 reg)
0183 {
0184     return readl(tdc->chan_addr + reg);
0185 }
0186 
0187 static inline struct tegra_adma_chan *to_tegra_adma_chan(struct dma_chan *dc)
0188 {
0189     return container_of(dc, struct tegra_adma_chan, vc.chan);
0190 }
0191 
0192 static inline struct tegra_adma_desc *to_tegra_adma_desc(
0193         struct dma_async_tx_descriptor *td)
0194 {
0195     return container_of(td, struct tegra_adma_desc, vd.tx);
0196 }
0197 
0198 static inline struct device *tdc2dev(struct tegra_adma_chan *tdc)
0199 {
0200     return tdc->tdma->dev;
0201 }
0202 
0203 static void tegra_adma_desc_free(struct virt_dma_desc *vd)
0204 {
0205     kfree(container_of(vd, struct tegra_adma_desc, vd));
0206 }
0207 
0208 static int tegra_adma_slave_config(struct dma_chan *dc,
0209                    struct dma_slave_config *sconfig)
0210 {
0211     struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc);
0212 
0213     memcpy(&tdc->sconfig, sconfig, sizeof(*sconfig));
0214 
0215     return 0;
0216 }
0217 
0218 static int tegra_adma_init(struct tegra_adma *tdma)
0219 {
0220     u32 status;
0221     int ret;
0222 
0223     /* Clear any interrupts */
0224     tdma_write(tdma, tdma->cdata->global_int_clear, 0x1);
0225 
0226     /* Assert soft reset */
0227     tdma_write(tdma, ADMA_GLOBAL_SOFT_RESET, 0x1);
0228 
0229     /* Wait for reset to clear */
0230     ret = readx_poll_timeout(readl,
0231                  tdma->base_addr +
0232                  tdma->cdata->global_reg_offset +
0233                  ADMA_GLOBAL_SOFT_RESET,
0234                  status, status == 0, 20, 10000);
0235     if (ret)
0236         return ret;
0237 
0238     /* Enable global ADMA registers */
0239     tdma_write(tdma, ADMA_GLOBAL_CMD, 1);
0240 
0241     return 0;
0242 }
0243 
0244 static int tegra_adma_request_alloc(struct tegra_adma_chan *tdc,
0245                     enum dma_transfer_direction direction)
0246 {
0247     struct tegra_adma *tdma = tdc->tdma;
0248     unsigned int sreq_index = tdc->sreq_index;
0249 
0250     if (tdc->sreq_reserved)
0251         return tdc->sreq_dir == direction ? 0 : -EINVAL;
0252 
0253     if (sreq_index > tdma->cdata->ch_req_max) {
0254         dev_err(tdma->dev, "invalid DMA request\n");
0255         return -EINVAL;
0256     }
0257 
0258     switch (direction) {
0259     case DMA_MEM_TO_DEV:
0260         if (test_and_set_bit(sreq_index, &tdma->tx_requests_reserved)) {
0261             dev_err(tdma->dev, "DMA request reserved\n");
0262             return -EINVAL;
0263         }
0264         break;
0265 
0266     case DMA_DEV_TO_MEM:
0267         if (test_and_set_bit(sreq_index, &tdma->rx_requests_reserved)) {
0268             dev_err(tdma->dev, "DMA request reserved\n");
0269             return -EINVAL;
0270         }
0271         break;
0272 
0273     default:
0274         dev_WARN(tdma->dev, "channel %s has invalid transfer type\n",
0275              dma_chan_name(&tdc->vc.chan));
0276         return -EINVAL;
0277     }
0278 
0279     tdc->sreq_dir = direction;
0280     tdc->sreq_reserved = true;
0281 
0282     return 0;
0283 }
0284 
0285 static void tegra_adma_request_free(struct tegra_adma_chan *tdc)
0286 {
0287     struct tegra_adma *tdma = tdc->tdma;
0288 
0289     if (!tdc->sreq_reserved)
0290         return;
0291 
0292     switch (tdc->sreq_dir) {
0293     case DMA_MEM_TO_DEV:
0294         clear_bit(tdc->sreq_index, &tdma->tx_requests_reserved);
0295         break;
0296 
0297     case DMA_DEV_TO_MEM:
0298         clear_bit(tdc->sreq_index, &tdma->rx_requests_reserved);
0299         break;
0300 
0301     default:
0302         dev_WARN(tdma->dev, "channel %s has invalid transfer type\n",
0303              dma_chan_name(&tdc->vc.chan));
0304         return;
0305     }
0306 
0307     tdc->sreq_reserved = false;
0308 }
0309 
0310 static u32 tegra_adma_irq_status(struct tegra_adma_chan *tdc)
0311 {
0312     u32 status = tdma_ch_read(tdc, ADMA_CH_INT_STATUS);
0313 
0314     return status & ADMA_CH_INT_STATUS_XFER_DONE;
0315 }
0316 
0317 static u32 tegra_adma_irq_clear(struct tegra_adma_chan *tdc)
0318 {
0319     u32 status = tegra_adma_irq_status(tdc);
0320 
0321     if (status)
0322         tdma_ch_write(tdc, ADMA_CH_INT_CLEAR, status);
0323 
0324     return status;
0325 }
0326 
0327 static void tegra_adma_stop(struct tegra_adma_chan *tdc)
0328 {
0329     unsigned int status;
0330 
0331     /* Disable ADMA */
0332     tdma_ch_write(tdc, ADMA_CH_CMD, 0);
0333 
0334     /* Clear interrupt status */
0335     tegra_adma_irq_clear(tdc);
0336 
0337     if (readx_poll_timeout_atomic(readl, tdc->chan_addr + ADMA_CH_STATUS,
0338             status, !(status & ADMA_CH_STATUS_XFER_EN),
0339             20, 10000)) {
0340         dev_err(tdc2dev(tdc), "unable to stop DMA channel\n");
0341         return;
0342     }
0343 
0344     kfree(tdc->desc);
0345     tdc->desc = NULL;
0346 }
0347 
0348 static void tegra_adma_start(struct tegra_adma_chan *tdc)
0349 {
0350     struct virt_dma_desc *vd = vchan_next_desc(&tdc->vc);
0351     struct tegra_adma_chan_regs *ch_regs;
0352     struct tegra_adma_desc *desc;
0353 
0354     if (!vd)
0355         return;
0356 
0357     list_del(&vd->node);
0358 
0359     desc = to_tegra_adma_desc(&vd->tx);
0360 
0361     if (!desc) {
0362         dev_warn(tdc2dev(tdc), "unable to start DMA, no descriptor\n");
0363         return;
0364     }
0365 
0366     ch_regs = &desc->ch_regs;
0367 
0368     tdc->tx_buf_pos = 0;
0369     tdc->tx_buf_count = 0;
0370     tdma_ch_write(tdc, ADMA_CH_TC, ch_regs->tc);
0371     tdma_ch_write(tdc, ADMA_CH_CTRL, ch_regs->ctrl);
0372     tdma_ch_write(tdc, ADMA_CH_LOWER_SRC_ADDR, ch_regs->src_addr);
0373     tdma_ch_write(tdc, ADMA_CH_LOWER_TRG_ADDR, ch_regs->trg_addr);
0374     tdma_ch_write(tdc, ADMA_CH_FIFO_CTRL, ch_regs->fifo_ctrl);
0375     tdma_ch_write(tdc, ADMA_CH_CONFIG, ch_regs->config);
0376 
0377     /* Start ADMA */
0378     tdma_ch_write(tdc, ADMA_CH_CMD, 1);
0379 
0380     tdc->desc = desc;
0381 }
0382 
0383 static unsigned int tegra_adma_get_residue(struct tegra_adma_chan *tdc)
0384 {
0385     struct tegra_adma_desc *desc = tdc->desc;
0386     unsigned int max = ADMA_CH_XFER_STATUS_COUNT_MASK + 1;
0387     unsigned int pos = tdma_ch_read(tdc, ADMA_CH_XFER_STATUS);
0388     unsigned int periods_remaining;
0389 
0390     /*
0391      * Handle wrap around of buffer count register
0392      */
0393     if (pos < tdc->tx_buf_pos)
0394         tdc->tx_buf_count += pos + (max - tdc->tx_buf_pos);
0395     else
0396         tdc->tx_buf_count += pos - tdc->tx_buf_pos;
0397 
0398     periods_remaining = tdc->tx_buf_count % desc->num_periods;
0399     tdc->tx_buf_pos = pos;
0400 
0401     return desc->buf_len - (periods_remaining * desc->period_len);
0402 }
0403 
0404 static irqreturn_t tegra_adma_isr(int irq, void *dev_id)
0405 {
0406     struct tegra_adma_chan *tdc = dev_id;
0407     unsigned long status;
0408 
0409     spin_lock(&tdc->vc.lock);
0410 
0411     status = tegra_adma_irq_clear(tdc);
0412     if (status == 0 || !tdc->desc) {
0413         spin_unlock(&tdc->vc.lock);
0414         return IRQ_NONE;
0415     }
0416 
0417     vchan_cyclic_callback(&tdc->desc->vd);
0418 
0419     spin_unlock(&tdc->vc.lock);
0420 
0421     return IRQ_HANDLED;
0422 }
0423 
0424 static void tegra_adma_issue_pending(struct dma_chan *dc)
0425 {
0426     struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc);
0427     unsigned long flags;
0428 
0429     spin_lock_irqsave(&tdc->vc.lock, flags);
0430 
0431     if (vchan_issue_pending(&tdc->vc)) {
0432         if (!tdc->desc)
0433             tegra_adma_start(tdc);
0434     }
0435 
0436     spin_unlock_irqrestore(&tdc->vc.lock, flags);
0437 }
0438 
0439 static bool tegra_adma_is_paused(struct tegra_adma_chan *tdc)
0440 {
0441     u32 csts;
0442 
0443     csts = tdma_ch_read(tdc, ADMA_CH_STATUS);
0444     csts &= ADMA_CH_STATUS_XFER_PAUSED;
0445 
0446     return csts ? true : false;
0447 }
0448 
0449 static int tegra_adma_pause(struct dma_chan *dc)
0450 {
0451     struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc);
0452     struct tegra_adma_desc *desc = tdc->desc;
0453     struct tegra_adma_chan_regs *ch_regs = &desc->ch_regs;
0454     int dcnt = 10;
0455 
0456     ch_regs->ctrl = tdma_ch_read(tdc, ADMA_CH_CTRL);
0457     ch_regs->ctrl |= (1 << ADMA_CH_CTRL_XFER_PAUSE_SHIFT);
0458     tdma_ch_write(tdc, ADMA_CH_CTRL, ch_regs->ctrl);
0459 
0460     while (dcnt-- && !tegra_adma_is_paused(tdc))
0461         udelay(TEGRA_ADMA_BURST_COMPLETE_TIME);
0462 
0463     if (dcnt < 0) {
0464         dev_err(tdc2dev(tdc), "unable to pause DMA channel\n");
0465         return -EBUSY;
0466     }
0467 
0468     return 0;
0469 }
0470 
0471 static int tegra_adma_resume(struct dma_chan *dc)
0472 {
0473     struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc);
0474     struct tegra_adma_desc *desc = tdc->desc;
0475     struct tegra_adma_chan_regs *ch_regs = &desc->ch_regs;
0476 
0477     ch_regs->ctrl = tdma_ch_read(tdc, ADMA_CH_CTRL);
0478     ch_regs->ctrl &= ~(1 << ADMA_CH_CTRL_XFER_PAUSE_SHIFT);
0479     tdma_ch_write(tdc, ADMA_CH_CTRL, ch_regs->ctrl);
0480 
0481     return 0;
0482 }
0483 
0484 static int tegra_adma_terminate_all(struct dma_chan *dc)
0485 {
0486     struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc);
0487     unsigned long flags;
0488     LIST_HEAD(head);
0489 
0490     spin_lock_irqsave(&tdc->vc.lock, flags);
0491 
0492     if (tdc->desc)
0493         tegra_adma_stop(tdc);
0494 
0495     tegra_adma_request_free(tdc);
0496     vchan_get_all_descriptors(&tdc->vc, &head);
0497     spin_unlock_irqrestore(&tdc->vc.lock, flags);
0498     vchan_dma_desc_free_list(&tdc->vc, &head);
0499 
0500     return 0;
0501 }
0502 
0503 static enum dma_status tegra_adma_tx_status(struct dma_chan *dc,
0504                         dma_cookie_t cookie,
0505                         struct dma_tx_state *txstate)
0506 {
0507     struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc);
0508     struct tegra_adma_desc *desc;
0509     struct virt_dma_desc *vd;
0510     enum dma_status ret;
0511     unsigned long flags;
0512     unsigned int residual;
0513 
0514     ret = dma_cookie_status(dc, cookie, txstate);
0515     if (ret == DMA_COMPLETE || !txstate)
0516         return ret;
0517 
0518     spin_lock_irqsave(&tdc->vc.lock, flags);
0519 
0520     vd = vchan_find_desc(&tdc->vc, cookie);
0521     if (vd) {
0522         desc = to_tegra_adma_desc(&vd->tx);
0523         residual = desc->ch_regs.tc;
0524     } else if (tdc->desc && tdc->desc->vd.tx.cookie == cookie) {
0525         residual = tegra_adma_get_residue(tdc);
0526     } else {
0527         residual = 0;
0528     }
0529 
0530     spin_unlock_irqrestore(&tdc->vc.lock, flags);
0531 
0532     dma_set_residue(txstate, residual);
0533 
0534     return ret;
0535 }
0536 
0537 static unsigned int tegra210_adma_get_burst_config(unsigned int burst_size)
0538 {
0539     if (!burst_size || burst_size > ADMA_CH_CONFIG_MAX_BURST_SIZE)
0540         burst_size = ADMA_CH_CONFIG_MAX_BURST_SIZE;
0541 
0542     return fls(burst_size) << ADMA_CH_CONFIG_BURST_SIZE_SHIFT;
0543 }
0544 
0545 static unsigned int tegra186_adma_get_burst_config(unsigned int burst_size)
0546 {
0547     if (!burst_size || burst_size > ADMA_CH_CONFIG_MAX_BURST_SIZE)
0548         burst_size = ADMA_CH_CONFIG_MAX_BURST_SIZE;
0549 
0550     return (burst_size - 1) << ADMA_CH_CONFIG_BURST_SIZE_SHIFT;
0551 }
0552 
0553 static int tegra_adma_set_xfer_params(struct tegra_adma_chan *tdc,
0554                       struct tegra_adma_desc *desc,
0555                       dma_addr_t buf_addr,
0556                       enum dma_transfer_direction direction)
0557 {
0558     struct tegra_adma_chan_regs *ch_regs = &desc->ch_regs;
0559     const struct tegra_adma_chip_data *cdata = tdc->tdma->cdata;
0560     unsigned int burst_size, adma_dir, fifo_size_shift;
0561 
0562     if (desc->num_periods > ADMA_CH_CONFIG_MAX_BUFS)
0563         return -EINVAL;
0564 
0565     switch (direction) {
0566     case DMA_MEM_TO_DEV:
0567         fifo_size_shift = ADMA_CH_TX_FIFO_SIZE_SHIFT;
0568         adma_dir = ADMA_CH_CTRL_DIR_MEM2AHUB;
0569         burst_size = tdc->sconfig.dst_maxburst;
0570         ch_regs->config = ADMA_CH_CONFIG_SRC_BUF(desc->num_periods - 1);
0571         ch_regs->ctrl = ADMA_CH_REG_FIELD_VAL(tdc->sreq_index,
0572                               cdata->ch_req_mask,
0573                               cdata->ch_req_tx_shift);
0574         ch_regs->src_addr = buf_addr;
0575         break;
0576 
0577     case DMA_DEV_TO_MEM:
0578         fifo_size_shift = ADMA_CH_RX_FIFO_SIZE_SHIFT;
0579         adma_dir = ADMA_CH_CTRL_DIR_AHUB2MEM;
0580         burst_size = tdc->sconfig.src_maxburst;
0581         ch_regs->config = ADMA_CH_CONFIG_TRG_BUF(desc->num_periods - 1);
0582         ch_regs->ctrl = ADMA_CH_REG_FIELD_VAL(tdc->sreq_index,
0583                               cdata->ch_req_mask,
0584                               cdata->ch_req_rx_shift);
0585         ch_regs->trg_addr = buf_addr;
0586         break;
0587 
0588     default:
0589         dev_err(tdc2dev(tdc), "DMA direction is not supported\n");
0590         return -EINVAL;
0591     }
0592 
0593     ch_regs->ctrl |= ADMA_CH_CTRL_DIR(adma_dir) |
0594              ADMA_CH_CTRL_MODE_CONTINUOUS |
0595              ADMA_CH_CTRL_FLOWCTRL_EN;
0596     ch_regs->config |= cdata->adma_get_burst_config(burst_size);
0597     ch_regs->config |= ADMA_CH_CONFIG_WEIGHT_FOR_WRR(1);
0598     if (cdata->has_outstanding_reqs)
0599         ch_regs->config |= TEGRA186_ADMA_CH_CONFIG_OUTSTANDING_REQS(8);
0600 
0601     /*
0602      * 'sreq_index' represents the current ADMAIF channel number and as per
0603      * HW recommendation its FIFO size should match with the corresponding
0604      * ADMA channel.
0605      *
0606      * ADMA FIFO size is set as per below (based on default ADMAIF channel
0607      * FIFO sizes):
0608      *    fifo_size = 0x2 (sreq_index > sreq_index_offset)
0609      *    fifo_size = 0x3 (sreq_index <= sreq_index_offset)
0610      *
0611      */
0612     if (tdc->sreq_index > cdata->sreq_index_offset)
0613         ch_regs->fifo_ctrl =
0614             ADMA_CH_REG_FIELD_VAL(2, cdata->ch_fifo_size_mask,
0615                           fifo_size_shift);
0616     else
0617         ch_regs->fifo_ctrl =
0618             ADMA_CH_REG_FIELD_VAL(3, cdata->ch_fifo_size_mask,
0619                           fifo_size_shift);
0620 
0621     ch_regs->tc = desc->period_len & ADMA_CH_TC_COUNT_MASK;
0622 
0623     return tegra_adma_request_alloc(tdc, direction);
0624 }
0625 
0626 static struct dma_async_tx_descriptor *tegra_adma_prep_dma_cyclic(
0627     struct dma_chan *dc, dma_addr_t buf_addr, size_t buf_len,
0628     size_t period_len, enum dma_transfer_direction direction,
0629     unsigned long flags)
0630 {
0631     struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc);
0632     struct tegra_adma_desc *desc = NULL;
0633 
0634     if (!buf_len || !period_len || period_len > ADMA_CH_TC_COUNT_MASK) {
0635         dev_err(tdc2dev(tdc), "invalid buffer/period len\n");
0636         return NULL;
0637     }
0638 
0639     if (buf_len % period_len) {
0640         dev_err(tdc2dev(tdc), "buf_len not a multiple of period_len\n");
0641         return NULL;
0642     }
0643 
0644     if (!IS_ALIGNED(buf_addr, 4)) {
0645         dev_err(tdc2dev(tdc), "invalid buffer alignment\n");
0646         return NULL;
0647     }
0648 
0649     desc = kzalloc(sizeof(*desc), GFP_NOWAIT);
0650     if (!desc)
0651         return NULL;
0652 
0653     desc->buf_len = buf_len;
0654     desc->period_len = period_len;
0655     desc->num_periods = buf_len / period_len;
0656 
0657     if (tegra_adma_set_xfer_params(tdc, desc, buf_addr, direction)) {
0658         kfree(desc);
0659         return NULL;
0660     }
0661 
0662     return vchan_tx_prep(&tdc->vc, &desc->vd, flags);
0663 }
0664 
0665 static int tegra_adma_alloc_chan_resources(struct dma_chan *dc)
0666 {
0667     struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc);
0668     int ret;
0669 
0670     ret = request_irq(tdc->irq, tegra_adma_isr, 0, dma_chan_name(dc), tdc);
0671     if (ret) {
0672         dev_err(tdc2dev(tdc), "failed to get interrupt for %s\n",
0673             dma_chan_name(dc));
0674         return ret;
0675     }
0676 
0677     ret = pm_runtime_resume_and_get(tdc2dev(tdc));
0678     if (ret < 0) {
0679         free_irq(tdc->irq, tdc);
0680         return ret;
0681     }
0682 
0683     dma_cookie_init(&tdc->vc.chan);
0684 
0685     return 0;
0686 }
0687 
0688 static void tegra_adma_free_chan_resources(struct dma_chan *dc)
0689 {
0690     struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc);
0691 
0692     tegra_adma_terminate_all(dc);
0693     vchan_free_chan_resources(&tdc->vc);
0694     tasklet_kill(&tdc->vc.task);
0695     free_irq(tdc->irq, tdc);
0696     pm_runtime_put(tdc2dev(tdc));
0697 
0698     tdc->sreq_index = 0;
0699     tdc->sreq_dir = DMA_TRANS_NONE;
0700 }
0701 
0702 static struct dma_chan *tegra_dma_of_xlate(struct of_phandle_args *dma_spec,
0703                        struct of_dma *ofdma)
0704 {
0705     struct tegra_adma *tdma = ofdma->of_dma_data;
0706     struct tegra_adma_chan *tdc;
0707     struct dma_chan *chan;
0708     unsigned int sreq_index;
0709 
0710     if (dma_spec->args_count != 1)
0711         return NULL;
0712 
0713     sreq_index = dma_spec->args[0];
0714 
0715     if (sreq_index == 0) {
0716         dev_err(tdma->dev, "DMA request must not be 0\n");
0717         return NULL;
0718     }
0719 
0720     chan = dma_get_any_slave_channel(&tdma->dma_dev);
0721     if (!chan)
0722         return NULL;
0723 
0724     tdc = to_tegra_adma_chan(chan);
0725     tdc->sreq_index = sreq_index;
0726 
0727     return chan;
0728 }
0729 
0730 static int __maybe_unused tegra_adma_runtime_suspend(struct device *dev)
0731 {
0732     struct tegra_adma *tdma = dev_get_drvdata(dev);
0733     struct tegra_adma_chan_regs *ch_reg;
0734     struct tegra_adma_chan *tdc;
0735     int i;
0736 
0737     tdma->global_cmd = tdma_read(tdma, ADMA_GLOBAL_CMD);
0738     if (!tdma->global_cmd)
0739         goto clk_disable;
0740 
0741     for (i = 0; i < tdma->nr_channels; i++) {
0742         tdc = &tdma->channels[i];
0743         ch_reg = &tdc->ch_regs;
0744         ch_reg->cmd = tdma_ch_read(tdc, ADMA_CH_CMD);
0745         /* skip if channel is not active */
0746         if (!ch_reg->cmd)
0747             continue;
0748         ch_reg->tc = tdma_ch_read(tdc, ADMA_CH_TC);
0749         ch_reg->src_addr = tdma_ch_read(tdc, ADMA_CH_LOWER_SRC_ADDR);
0750         ch_reg->trg_addr = tdma_ch_read(tdc, ADMA_CH_LOWER_TRG_ADDR);
0751         ch_reg->ctrl = tdma_ch_read(tdc, ADMA_CH_CTRL);
0752         ch_reg->fifo_ctrl = tdma_ch_read(tdc, ADMA_CH_FIFO_CTRL);
0753         ch_reg->config = tdma_ch_read(tdc, ADMA_CH_CONFIG);
0754     }
0755 
0756 clk_disable:
0757     clk_disable_unprepare(tdma->ahub_clk);
0758 
0759     return 0;
0760 }
0761 
0762 static int __maybe_unused tegra_adma_runtime_resume(struct device *dev)
0763 {
0764     struct tegra_adma *tdma = dev_get_drvdata(dev);
0765     struct tegra_adma_chan_regs *ch_reg;
0766     struct tegra_adma_chan *tdc;
0767     int ret, i;
0768 
0769     ret = clk_prepare_enable(tdma->ahub_clk);
0770     if (ret) {
0771         dev_err(dev, "ahub clk_enable failed: %d\n", ret);
0772         return ret;
0773     }
0774     tdma_write(tdma, ADMA_GLOBAL_CMD, tdma->global_cmd);
0775 
0776     if (!tdma->global_cmd)
0777         return 0;
0778 
0779     for (i = 0; i < tdma->nr_channels; i++) {
0780         tdc = &tdma->channels[i];
0781         ch_reg = &tdc->ch_regs;
0782         /* skip if channel was not active earlier */
0783         if (!ch_reg->cmd)
0784             continue;
0785         tdma_ch_write(tdc, ADMA_CH_TC, ch_reg->tc);
0786         tdma_ch_write(tdc, ADMA_CH_LOWER_SRC_ADDR, ch_reg->src_addr);
0787         tdma_ch_write(tdc, ADMA_CH_LOWER_TRG_ADDR, ch_reg->trg_addr);
0788         tdma_ch_write(tdc, ADMA_CH_CTRL, ch_reg->ctrl);
0789         tdma_ch_write(tdc, ADMA_CH_FIFO_CTRL, ch_reg->fifo_ctrl);
0790         tdma_ch_write(tdc, ADMA_CH_CONFIG, ch_reg->config);
0791         tdma_ch_write(tdc, ADMA_CH_CMD, ch_reg->cmd);
0792     }
0793 
0794     return 0;
0795 }
0796 
0797 static const struct tegra_adma_chip_data tegra210_chip_data = {
0798     .adma_get_burst_config  = tegra210_adma_get_burst_config,
0799     .global_reg_offset  = 0xc00,
0800     .global_int_clear   = 0x20,
0801     .ch_req_tx_shift    = 28,
0802     .ch_req_rx_shift    = 24,
0803     .ch_base_offset     = 0,
0804     .ch_req_mask        = 0xf,
0805     .ch_req_max     = 10,
0806     .ch_reg_size        = 0x80,
0807     .nr_channels        = 22,
0808     .ch_fifo_size_mask  = 0xf,
0809     .sreq_index_offset  = 2,
0810     .has_outstanding_reqs   = false,
0811 };
0812 
0813 static const struct tegra_adma_chip_data tegra186_chip_data = {
0814     .adma_get_burst_config  = tegra186_adma_get_burst_config,
0815     .global_reg_offset  = 0,
0816     .global_int_clear   = 0x402c,
0817     .ch_req_tx_shift    = 27,
0818     .ch_req_rx_shift    = 22,
0819     .ch_base_offset     = 0x10000,
0820     .ch_req_mask        = 0x1f,
0821     .ch_req_max     = 20,
0822     .ch_reg_size        = 0x100,
0823     .nr_channels        = 32,
0824     .ch_fifo_size_mask  = 0x1f,
0825     .sreq_index_offset  = 4,
0826     .has_outstanding_reqs   = true,
0827 };
0828 
0829 static const struct of_device_id tegra_adma_of_match[] = {
0830     { .compatible = "nvidia,tegra210-adma", .data = &tegra210_chip_data },
0831     { .compatible = "nvidia,tegra186-adma", .data = &tegra186_chip_data },
0832     { },
0833 };
0834 MODULE_DEVICE_TABLE(of, tegra_adma_of_match);
0835 
0836 static int tegra_adma_probe(struct platform_device *pdev)
0837 {
0838     const struct tegra_adma_chip_data *cdata;
0839     struct tegra_adma *tdma;
0840     struct resource *res;
0841     int ret, i;
0842 
0843     cdata = of_device_get_match_data(&pdev->dev);
0844     if (!cdata) {
0845         dev_err(&pdev->dev, "device match data not found\n");
0846         return -ENODEV;
0847     }
0848 
0849     tdma = devm_kzalloc(&pdev->dev,
0850                 struct_size(tdma, channels, cdata->nr_channels),
0851                 GFP_KERNEL);
0852     if (!tdma)
0853         return -ENOMEM;
0854 
0855     tdma->dev = &pdev->dev;
0856     tdma->cdata = cdata;
0857     tdma->nr_channels = cdata->nr_channels;
0858     platform_set_drvdata(pdev, tdma);
0859 
0860     res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
0861     tdma->base_addr = devm_ioremap_resource(&pdev->dev, res);
0862     if (IS_ERR(tdma->base_addr))
0863         return PTR_ERR(tdma->base_addr);
0864 
0865     tdma->ahub_clk = devm_clk_get(&pdev->dev, "d_audio");
0866     if (IS_ERR(tdma->ahub_clk)) {
0867         dev_err(&pdev->dev, "Error: Missing ahub controller clock\n");
0868         return PTR_ERR(tdma->ahub_clk);
0869     }
0870 
0871     INIT_LIST_HEAD(&tdma->dma_dev.channels);
0872     for (i = 0; i < tdma->nr_channels; i++) {
0873         struct tegra_adma_chan *tdc = &tdma->channels[i];
0874 
0875         tdc->chan_addr = tdma->base_addr + cdata->ch_base_offset
0876                  + (cdata->ch_reg_size * i);
0877 
0878         tdc->irq = of_irq_get(pdev->dev.of_node, i);
0879         if (tdc->irq <= 0) {
0880             ret = tdc->irq ?: -ENXIO;
0881             goto irq_dispose;
0882         }
0883 
0884         vchan_init(&tdc->vc, &tdma->dma_dev);
0885         tdc->vc.desc_free = tegra_adma_desc_free;
0886         tdc->tdma = tdma;
0887     }
0888 
0889     pm_runtime_enable(&pdev->dev);
0890 
0891     ret = pm_runtime_resume_and_get(&pdev->dev);
0892     if (ret < 0)
0893         goto rpm_disable;
0894 
0895     ret = tegra_adma_init(tdma);
0896     if (ret)
0897         goto rpm_put;
0898 
0899     dma_cap_set(DMA_SLAVE, tdma->dma_dev.cap_mask);
0900     dma_cap_set(DMA_PRIVATE, tdma->dma_dev.cap_mask);
0901     dma_cap_set(DMA_CYCLIC, tdma->dma_dev.cap_mask);
0902 
0903     tdma->dma_dev.dev = &pdev->dev;
0904     tdma->dma_dev.device_alloc_chan_resources =
0905                     tegra_adma_alloc_chan_resources;
0906     tdma->dma_dev.device_free_chan_resources =
0907                     tegra_adma_free_chan_resources;
0908     tdma->dma_dev.device_issue_pending = tegra_adma_issue_pending;
0909     tdma->dma_dev.device_prep_dma_cyclic = tegra_adma_prep_dma_cyclic;
0910     tdma->dma_dev.device_config = tegra_adma_slave_config;
0911     tdma->dma_dev.device_tx_status = tegra_adma_tx_status;
0912     tdma->dma_dev.device_terminate_all = tegra_adma_terminate_all;
0913     tdma->dma_dev.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
0914     tdma->dma_dev.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
0915     tdma->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
0916     tdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
0917     tdma->dma_dev.device_pause = tegra_adma_pause;
0918     tdma->dma_dev.device_resume = tegra_adma_resume;
0919 
0920     ret = dma_async_device_register(&tdma->dma_dev);
0921     if (ret < 0) {
0922         dev_err(&pdev->dev, "ADMA registration failed: %d\n", ret);
0923         goto rpm_put;
0924     }
0925 
0926     ret = of_dma_controller_register(pdev->dev.of_node,
0927                      tegra_dma_of_xlate, tdma);
0928     if (ret < 0) {
0929         dev_err(&pdev->dev, "ADMA OF registration failed %d\n", ret);
0930         goto dma_remove;
0931     }
0932 
0933     pm_runtime_put(&pdev->dev);
0934 
0935     dev_info(&pdev->dev, "Tegra210 ADMA driver registered %d channels\n",
0936          tdma->nr_channels);
0937 
0938     return 0;
0939 
0940 dma_remove:
0941     dma_async_device_unregister(&tdma->dma_dev);
0942 rpm_put:
0943     pm_runtime_put_sync(&pdev->dev);
0944 rpm_disable:
0945     pm_runtime_disable(&pdev->dev);
0946 irq_dispose:
0947     while (--i >= 0)
0948         irq_dispose_mapping(tdma->channels[i].irq);
0949 
0950     return ret;
0951 }
0952 
0953 static int tegra_adma_remove(struct platform_device *pdev)
0954 {
0955     struct tegra_adma *tdma = platform_get_drvdata(pdev);
0956     int i;
0957 
0958     of_dma_controller_free(pdev->dev.of_node);
0959     dma_async_device_unregister(&tdma->dma_dev);
0960 
0961     for (i = 0; i < tdma->nr_channels; ++i)
0962         irq_dispose_mapping(tdma->channels[i].irq);
0963 
0964     pm_runtime_disable(&pdev->dev);
0965 
0966     return 0;
0967 }
0968 
0969 static const struct dev_pm_ops tegra_adma_dev_pm_ops = {
0970     SET_RUNTIME_PM_OPS(tegra_adma_runtime_suspend,
0971                tegra_adma_runtime_resume, NULL)
0972     SET_LATE_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
0973                      pm_runtime_force_resume)
0974 };
0975 
0976 static struct platform_driver tegra_admac_driver = {
0977     .driver = {
0978         .name   = "tegra-adma",
0979         .pm = &tegra_adma_dev_pm_ops,
0980         .of_match_table = tegra_adma_of_match,
0981     },
0982     .probe      = tegra_adma_probe,
0983     .remove     = tegra_adma_remove,
0984 };
0985 
0986 module_platform_driver(tegra_admac_driver);
0987 
0988 MODULE_ALIAS("platform:tegra210-adma");
0989 MODULE_DESCRIPTION("NVIDIA Tegra ADMA driver");
0990 MODULE_AUTHOR("Dara Ramesh <dramesh@nvidia.com>");
0991 MODULE_AUTHOR("Jon Hunter <jonathanh@nvidia.com>");
0992 MODULE_LICENSE("GPL v2");
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!