Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Copyright (c) 2013-2014, The Linux Foundation. All rights reserved.
0004  */
0005 /*
0006  * QCOM BAM DMA engine driver
0007  *
0008  * QCOM BAM DMA blocks are distributed amongst a number of the on-chip
0009  * peripherals on the MSM 8x74.  The configuration of the channels are dependent
0010  * on the way they are hard wired to that specific peripheral.  The peripheral
0011  * device tree entries specify the configuration of each channel.
0012  *
0013  * The DMA controller requires the use of external memory for storage of the
0014  * hardware descriptors for each channel.  The descriptor FIFO is accessed as a
0015  * circular buffer and operations are managed according to the offset within the
0016  * FIFO.  After pipe/channel reset, all of the pipe registers and internal state
0017  * are back to defaults.
0018  *
0019  * During DMA operations, we write descriptors to the FIFO, being careful to
0020  * handle wrapping and then write the last FIFO offset to that channel's
0021  * P_EVNT_REG register to kick off the transaction.  The P_SW_OFSTS register
0022  * indicates the current FIFO offset that is being processed, so there is some
0023  * indication of where the hardware is currently working.
0024  */
0025 
0026 #include <linux/kernel.h>
0027 #include <linux/io.h>
0028 #include <linux/init.h>
0029 #include <linux/slab.h>
0030 #include <linux/module.h>
0031 #include <linux/interrupt.h>
0032 #include <linux/dma-mapping.h>
0033 #include <linux/scatterlist.h>
0034 #include <linux/device.h>
0035 #include <linux/platform_device.h>
0036 #include <linux/of.h>
0037 #include <linux/of_address.h>
0038 #include <linux/of_irq.h>
0039 #include <linux/of_dma.h>
0040 #include <linux/circ_buf.h>
0041 #include <linux/clk.h>
0042 #include <linux/dmaengine.h>
0043 #include <linux/pm_runtime.h>
0044 
0045 #include "../dmaengine.h"
0046 #include "../virt-dma.h"
0047 
0048 struct bam_desc_hw {
0049     __le32 addr;        /* Buffer physical address */
0050     __le16 size;        /* Buffer size in bytes */
0051     __le16 flags;
0052 };
0053 
0054 #define BAM_DMA_AUTOSUSPEND_DELAY 100
0055 
0056 #define DESC_FLAG_INT BIT(15)
0057 #define DESC_FLAG_EOT BIT(14)
0058 #define DESC_FLAG_EOB BIT(13)
0059 #define DESC_FLAG_NWD BIT(12)
0060 #define DESC_FLAG_CMD BIT(11)
0061 
0062 struct bam_async_desc {
0063     struct virt_dma_desc vd;
0064 
0065     u32 num_desc;
0066     u32 xfer_len;
0067 
0068     /* transaction flags, EOT|EOB|NWD */
0069     u16 flags;
0070 
0071     struct bam_desc_hw *curr_desc;
0072 
0073     /* list node for the desc in the bam_chan list of descriptors */
0074     struct list_head desc_node;
0075     enum dma_transfer_direction dir;
0076     size_t length;
0077     struct bam_desc_hw desc[];
0078 };
0079 
0080 enum bam_reg {
0081     BAM_CTRL,
0082     BAM_REVISION,
0083     BAM_NUM_PIPES,
0084     BAM_DESC_CNT_TRSHLD,
0085     BAM_IRQ_SRCS,
0086     BAM_IRQ_SRCS_MSK,
0087     BAM_IRQ_SRCS_UNMASKED,
0088     BAM_IRQ_STTS,
0089     BAM_IRQ_CLR,
0090     BAM_IRQ_EN,
0091     BAM_CNFG_BITS,
0092     BAM_IRQ_SRCS_EE,
0093     BAM_IRQ_SRCS_MSK_EE,
0094     BAM_P_CTRL,
0095     BAM_P_RST,
0096     BAM_P_HALT,
0097     BAM_P_IRQ_STTS,
0098     BAM_P_IRQ_CLR,
0099     BAM_P_IRQ_EN,
0100     BAM_P_EVNT_DEST_ADDR,
0101     BAM_P_EVNT_REG,
0102     BAM_P_SW_OFSTS,
0103     BAM_P_DATA_FIFO_ADDR,
0104     BAM_P_DESC_FIFO_ADDR,
0105     BAM_P_EVNT_GEN_TRSHLD,
0106     BAM_P_FIFO_SIZES,
0107 };
0108 
0109 struct reg_offset_data {
0110     u32 base_offset;
0111     unsigned int pipe_mult, evnt_mult, ee_mult;
0112 };
0113 
0114 static const struct reg_offset_data bam_v1_3_reg_info[] = {
0115     [BAM_CTRL]      = { 0x0F80, 0x00, 0x00, 0x00 },
0116     [BAM_REVISION]      = { 0x0F84, 0x00, 0x00, 0x00 },
0117     [BAM_NUM_PIPES]     = { 0x0FBC, 0x00, 0x00, 0x00 },
0118     [BAM_DESC_CNT_TRSHLD]   = { 0x0F88, 0x00, 0x00, 0x00 },
0119     [BAM_IRQ_SRCS]      = { 0x0F8C, 0x00, 0x00, 0x00 },
0120     [BAM_IRQ_SRCS_MSK]  = { 0x0F90, 0x00, 0x00, 0x00 },
0121     [BAM_IRQ_SRCS_UNMASKED] = { 0x0FB0, 0x00, 0x00, 0x00 },
0122     [BAM_IRQ_STTS]      = { 0x0F94, 0x00, 0x00, 0x00 },
0123     [BAM_IRQ_CLR]       = { 0x0F98, 0x00, 0x00, 0x00 },
0124     [BAM_IRQ_EN]        = { 0x0F9C, 0x00, 0x00, 0x00 },
0125     [BAM_CNFG_BITS]     = { 0x0FFC, 0x00, 0x00, 0x00 },
0126     [BAM_IRQ_SRCS_EE]   = { 0x1800, 0x00, 0x00, 0x80 },
0127     [BAM_IRQ_SRCS_MSK_EE]   = { 0x1804, 0x00, 0x00, 0x80 },
0128     [BAM_P_CTRL]        = { 0x0000, 0x80, 0x00, 0x00 },
0129     [BAM_P_RST]     = { 0x0004, 0x80, 0x00, 0x00 },
0130     [BAM_P_HALT]        = { 0x0008, 0x80, 0x00, 0x00 },
0131     [BAM_P_IRQ_STTS]    = { 0x0010, 0x80, 0x00, 0x00 },
0132     [BAM_P_IRQ_CLR]     = { 0x0014, 0x80, 0x00, 0x00 },
0133     [BAM_P_IRQ_EN]      = { 0x0018, 0x80, 0x00, 0x00 },
0134     [BAM_P_EVNT_DEST_ADDR]  = { 0x102C, 0x00, 0x40, 0x00 },
0135     [BAM_P_EVNT_REG]    = { 0x1018, 0x00, 0x40, 0x00 },
0136     [BAM_P_SW_OFSTS]    = { 0x1000, 0x00, 0x40, 0x00 },
0137     [BAM_P_DATA_FIFO_ADDR]  = { 0x1024, 0x00, 0x40, 0x00 },
0138     [BAM_P_DESC_FIFO_ADDR]  = { 0x101C, 0x00, 0x40, 0x00 },
0139     [BAM_P_EVNT_GEN_TRSHLD] = { 0x1028, 0x00, 0x40, 0x00 },
0140     [BAM_P_FIFO_SIZES]  = { 0x1020, 0x00, 0x40, 0x00 },
0141 };
0142 
0143 static const struct reg_offset_data bam_v1_4_reg_info[] = {
0144     [BAM_CTRL]      = { 0x0000, 0x00, 0x00, 0x00 },
0145     [BAM_REVISION]      = { 0x0004, 0x00, 0x00, 0x00 },
0146     [BAM_NUM_PIPES]     = { 0x003C, 0x00, 0x00, 0x00 },
0147     [BAM_DESC_CNT_TRSHLD]   = { 0x0008, 0x00, 0x00, 0x00 },
0148     [BAM_IRQ_SRCS]      = { 0x000C, 0x00, 0x00, 0x00 },
0149     [BAM_IRQ_SRCS_MSK]  = { 0x0010, 0x00, 0x00, 0x00 },
0150     [BAM_IRQ_SRCS_UNMASKED] = { 0x0030, 0x00, 0x00, 0x00 },
0151     [BAM_IRQ_STTS]      = { 0x0014, 0x00, 0x00, 0x00 },
0152     [BAM_IRQ_CLR]       = { 0x0018, 0x00, 0x00, 0x00 },
0153     [BAM_IRQ_EN]        = { 0x001C, 0x00, 0x00, 0x00 },
0154     [BAM_CNFG_BITS]     = { 0x007C, 0x00, 0x00, 0x00 },
0155     [BAM_IRQ_SRCS_EE]   = { 0x0800, 0x00, 0x00, 0x80 },
0156     [BAM_IRQ_SRCS_MSK_EE]   = { 0x0804, 0x00, 0x00, 0x80 },
0157     [BAM_P_CTRL]        = { 0x1000, 0x1000, 0x00, 0x00 },
0158     [BAM_P_RST]     = { 0x1004, 0x1000, 0x00, 0x00 },
0159     [BAM_P_HALT]        = { 0x1008, 0x1000, 0x00, 0x00 },
0160     [BAM_P_IRQ_STTS]    = { 0x1010, 0x1000, 0x00, 0x00 },
0161     [BAM_P_IRQ_CLR]     = { 0x1014, 0x1000, 0x00, 0x00 },
0162     [BAM_P_IRQ_EN]      = { 0x1018, 0x1000, 0x00, 0x00 },
0163     [BAM_P_EVNT_DEST_ADDR]  = { 0x182C, 0x00, 0x1000, 0x00 },
0164     [BAM_P_EVNT_REG]    = { 0x1818, 0x00, 0x1000, 0x00 },
0165     [BAM_P_SW_OFSTS]    = { 0x1800, 0x00, 0x1000, 0x00 },
0166     [BAM_P_DATA_FIFO_ADDR]  = { 0x1824, 0x00, 0x1000, 0x00 },
0167     [BAM_P_DESC_FIFO_ADDR]  = { 0x181C, 0x00, 0x1000, 0x00 },
0168     [BAM_P_EVNT_GEN_TRSHLD] = { 0x1828, 0x00, 0x1000, 0x00 },
0169     [BAM_P_FIFO_SIZES]  = { 0x1820, 0x00, 0x1000, 0x00 },
0170 };
0171 
0172 static const struct reg_offset_data bam_v1_7_reg_info[] = {
0173     [BAM_CTRL]      = { 0x00000, 0x00, 0x00, 0x00 },
0174     [BAM_REVISION]      = { 0x01000, 0x00, 0x00, 0x00 },
0175     [BAM_NUM_PIPES]     = { 0x01008, 0x00, 0x00, 0x00 },
0176     [BAM_DESC_CNT_TRSHLD]   = { 0x00008, 0x00, 0x00, 0x00 },
0177     [BAM_IRQ_SRCS]      = { 0x03010, 0x00, 0x00, 0x00 },
0178     [BAM_IRQ_SRCS_MSK]  = { 0x03014, 0x00, 0x00, 0x00 },
0179     [BAM_IRQ_SRCS_UNMASKED] = { 0x03018, 0x00, 0x00, 0x00 },
0180     [BAM_IRQ_STTS]      = { 0x00014, 0x00, 0x00, 0x00 },
0181     [BAM_IRQ_CLR]       = { 0x00018, 0x00, 0x00, 0x00 },
0182     [BAM_IRQ_EN]        = { 0x0001C, 0x00, 0x00, 0x00 },
0183     [BAM_CNFG_BITS]     = { 0x0007C, 0x00, 0x00, 0x00 },
0184     [BAM_IRQ_SRCS_EE]   = { 0x03000, 0x00, 0x00, 0x1000 },
0185     [BAM_IRQ_SRCS_MSK_EE]   = { 0x03004, 0x00, 0x00, 0x1000 },
0186     [BAM_P_CTRL]        = { 0x13000, 0x1000, 0x00, 0x00 },
0187     [BAM_P_RST]     = { 0x13004, 0x1000, 0x00, 0x00 },
0188     [BAM_P_HALT]        = { 0x13008, 0x1000, 0x00, 0x00 },
0189     [BAM_P_IRQ_STTS]    = { 0x13010, 0x1000, 0x00, 0x00 },
0190     [BAM_P_IRQ_CLR]     = { 0x13014, 0x1000, 0x00, 0x00 },
0191     [BAM_P_IRQ_EN]      = { 0x13018, 0x1000, 0x00, 0x00 },
0192     [BAM_P_EVNT_DEST_ADDR]  = { 0x1382C, 0x00, 0x1000, 0x00 },
0193     [BAM_P_EVNT_REG]    = { 0x13818, 0x00, 0x1000, 0x00 },
0194     [BAM_P_SW_OFSTS]    = { 0x13800, 0x00, 0x1000, 0x00 },
0195     [BAM_P_DATA_FIFO_ADDR]  = { 0x13824, 0x00, 0x1000, 0x00 },
0196     [BAM_P_DESC_FIFO_ADDR]  = { 0x1381C, 0x00, 0x1000, 0x00 },
0197     [BAM_P_EVNT_GEN_TRSHLD] = { 0x13828, 0x00, 0x1000, 0x00 },
0198     [BAM_P_FIFO_SIZES]  = { 0x13820, 0x00, 0x1000, 0x00 },
0199 };
0200 
0201 /* BAM CTRL */
0202 #define BAM_SW_RST          BIT(0)
0203 #define BAM_EN              BIT(1)
0204 #define BAM_EN_ACCUM            BIT(4)
0205 #define BAM_TESTBUS_SEL_SHIFT       5
0206 #define BAM_TESTBUS_SEL_MASK        0x3F
0207 #define BAM_DESC_CACHE_SEL_SHIFT    13
0208 #define BAM_DESC_CACHE_SEL_MASK     0x3
0209 #define BAM_CACHED_DESC_STORE       BIT(15)
0210 #define IBC_DISABLE         BIT(16)
0211 
0212 /* BAM REVISION */
0213 #define REVISION_SHIFT      0
0214 #define REVISION_MASK       0xFF
0215 #define NUM_EES_SHIFT       8
0216 #define NUM_EES_MASK        0xF
0217 #define CE_BUFFER_SIZE      BIT(13)
0218 #define AXI_ACTIVE      BIT(14)
0219 #define USE_VMIDMT      BIT(15)
0220 #define SECURED         BIT(16)
0221 #define BAM_HAS_NO_BYPASS   BIT(17)
0222 #define HIGH_FREQUENCY_BAM  BIT(18)
0223 #define INACTIV_TMRS_EXST   BIT(19)
0224 #define NUM_INACTIV_TMRS    BIT(20)
0225 #define DESC_CACHE_DEPTH_SHIFT  21
0226 #define DESC_CACHE_DEPTH_1  (0 << DESC_CACHE_DEPTH_SHIFT)
0227 #define DESC_CACHE_DEPTH_2  (1 << DESC_CACHE_DEPTH_SHIFT)
0228 #define DESC_CACHE_DEPTH_3  (2 << DESC_CACHE_DEPTH_SHIFT)
0229 #define DESC_CACHE_DEPTH_4  (3 << DESC_CACHE_DEPTH_SHIFT)
0230 #define CMD_DESC_EN     BIT(23)
0231 #define INACTIV_TMR_BASE_SHIFT  24
0232 #define INACTIV_TMR_BASE_MASK   0xFF
0233 
0234 /* BAM NUM PIPES */
0235 #define BAM_NUM_PIPES_SHIFT     0
0236 #define BAM_NUM_PIPES_MASK      0xFF
0237 #define PERIPH_NON_PIPE_GRP_SHIFT   16
0238 #define PERIPH_NON_PIP_GRP_MASK     0xFF
0239 #define BAM_NON_PIPE_GRP_SHIFT      24
0240 #define BAM_NON_PIPE_GRP_MASK       0xFF
0241 
0242 /* BAM CNFG BITS */
0243 #define BAM_PIPE_CNFG       BIT(2)
0244 #define BAM_FULL_PIPE       BIT(11)
0245 #define BAM_NO_EXT_P_RST    BIT(12)
0246 #define BAM_IBC_DISABLE     BIT(13)
0247 #define BAM_SB_CLK_REQ      BIT(14)
0248 #define BAM_PSM_CSW_REQ     BIT(15)
0249 #define BAM_PSM_P_RES       BIT(16)
0250 #define BAM_AU_P_RES        BIT(17)
0251 #define BAM_SI_P_RES        BIT(18)
0252 #define BAM_WB_P_RES        BIT(19)
0253 #define BAM_WB_BLK_CSW      BIT(20)
0254 #define BAM_WB_CSW_ACK_IDL  BIT(21)
0255 #define BAM_WB_RETR_SVPNT   BIT(22)
0256 #define BAM_WB_DSC_AVL_P_RST    BIT(23)
0257 #define BAM_REG_P_EN        BIT(24)
0258 #define BAM_PSM_P_HD_DATA   BIT(25)
0259 #define BAM_AU_ACCUMED      BIT(26)
0260 #define BAM_CMD_ENABLE      BIT(27)
0261 
0262 #define BAM_CNFG_BITS_DEFAULT   (BAM_PIPE_CNFG |    \
0263                  BAM_NO_EXT_P_RST | \
0264                  BAM_IBC_DISABLE |  \
0265                  BAM_SB_CLK_REQ |   \
0266                  BAM_PSM_CSW_REQ |  \
0267                  BAM_PSM_P_RES |    \
0268                  BAM_AU_P_RES |     \
0269                  BAM_SI_P_RES |     \
0270                  BAM_WB_P_RES |     \
0271                  BAM_WB_BLK_CSW |   \
0272                  BAM_WB_CSW_ACK_IDL |   \
0273                  BAM_WB_RETR_SVPNT |    \
0274                  BAM_WB_DSC_AVL_P_RST | \
0275                  BAM_REG_P_EN |     \
0276                  BAM_PSM_P_HD_DATA |    \
0277                  BAM_AU_ACCUMED |   \
0278                  BAM_CMD_ENABLE)
0279 
0280 /* PIPE CTRL */
0281 #define P_EN            BIT(1)
0282 #define P_DIRECTION     BIT(3)
0283 #define P_SYS_STRM      BIT(4)
0284 #define P_SYS_MODE      BIT(5)
0285 #define P_AUTO_EOB      BIT(6)
0286 #define P_AUTO_EOB_SEL_SHIFT    7
0287 #define P_AUTO_EOB_SEL_512  (0 << P_AUTO_EOB_SEL_SHIFT)
0288 #define P_AUTO_EOB_SEL_256  (1 << P_AUTO_EOB_SEL_SHIFT)
0289 #define P_AUTO_EOB_SEL_128  (2 << P_AUTO_EOB_SEL_SHIFT)
0290 #define P_AUTO_EOB_SEL_64   (3 << P_AUTO_EOB_SEL_SHIFT)
0291 #define P_PREFETCH_LIMIT_SHIFT  9
0292 #define P_PREFETCH_LIMIT_32 (0 << P_PREFETCH_LIMIT_SHIFT)
0293 #define P_PREFETCH_LIMIT_16 (1 << P_PREFETCH_LIMIT_SHIFT)
0294 #define P_PREFETCH_LIMIT_4  (2 << P_PREFETCH_LIMIT_SHIFT)
0295 #define P_WRITE_NWD     BIT(11)
0296 #define P_LOCK_GROUP_SHIFT  16
0297 #define P_LOCK_GROUP_MASK   0x1F
0298 
0299 /* BAM_DESC_CNT_TRSHLD */
0300 #define CNT_TRSHLD      0xffff
0301 #define DEFAULT_CNT_THRSHLD 0x4
0302 
0303 /* BAM_IRQ_SRCS */
0304 #define BAM_IRQ         BIT(31)
0305 #define P_IRQ           0x7fffffff
0306 
0307 /* BAM_IRQ_SRCS_MSK */
0308 #define BAM_IRQ_MSK     BAM_IRQ
0309 #define P_IRQ_MSK       P_IRQ
0310 
0311 /* BAM_IRQ_STTS */
0312 #define BAM_TIMER_IRQ       BIT(4)
0313 #define BAM_EMPTY_IRQ       BIT(3)
0314 #define BAM_ERROR_IRQ       BIT(2)
0315 #define BAM_HRESP_ERR_IRQ   BIT(1)
0316 
0317 /* BAM_IRQ_CLR */
0318 #define BAM_TIMER_CLR       BIT(4)
0319 #define BAM_EMPTY_CLR       BIT(3)
0320 #define BAM_ERROR_CLR       BIT(2)
0321 #define BAM_HRESP_ERR_CLR   BIT(1)
0322 
0323 /* BAM_IRQ_EN */
0324 #define BAM_TIMER_EN        BIT(4)
0325 #define BAM_EMPTY_EN        BIT(3)
0326 #define BAM_ERROR_EN        BIT(2)
0327 #define BAM_HRESP_ERR_EN    BIT(1)
0328 
0329 /* BAM_P_IRQ_EN */
0330 #define P_PRCSD_DESC_EN     BIT(0)
0331 #define P_TIMER_EN      BIT(1)
0332 #define P_WAKE_EN       BIT(2)
0333 #define P_OUT_OF_DESC_EN    BIT(3)
0334 #define P_ERR_EN        BIT(4)
0335 #define P_TRNSFR_END_EN     BIT(5)
0336 #define P_DEFAULT_IRQS_EN   (P_PRCSD_DESC_EN | P_ERR_EN | P_TRNSFR_END_EN)
0337 
0338 /* BAM_P_SW_OFSTS */
0339 #define P_SW_OFSTS_MASK     0xffff
0340 
0341 #define BAM_DESC_FIFO_SIZE  SZ_32K
0342 #define MAX_DESCRIPTORS (BAM_DESC_FIFO_SIZE / sizeof(struct bam_desc_hw) - 1)
0343 #define BAM_FIFO_SIZE   (SZ_32K - 8)
0344 #define IS_BUSY(chan)   (CIRC_SPACE(bchan->tail, bchan->head,\
0345              MAX_DESCRIPTORS + 1) == 0)
0346 
0347 struct bam_chan {
0348     struct virt_dma_chan vc;
0349 
0350     struct bam_device *bdev;
0351 
0352     /* configuration from device tree */
0353     u32 id;
0354 
0355     /* runtime configuration */
0356     struct dma_slave_config slave;
0357 
0358     /* fifo storage */
0359     struct bam_desc_hw *fifo_virt;
0360     dma_addr_t fifo_phys;
0361 
0362     /* fifo markers */
0363     unsigned short head;        /* start of active descriptor entries */
0364     unsigned short tail;        /* end of active descriptor entries */
0365 
0366     unsigned int initialized;   /* is the channel hw initialized? */
0367     unsigned int paused;        /* is the channel paused? */
0368     unsigned int reconfigure;   /* new slave config? */
0369     /* list of descriptors currently processed */
0370     struct list_head desc_list;
0371 
0372     struct list_head node;
0373 };
0374 
0375 static inline struct bam_chan *to_bam_chan(struct dma_chan *common)
0376 {
0377     return container_of(common, struct bam_chan, vc.chan);
0378 }
0379 
0380 struct bam_device {
0381     void __iomem *regs;
0382     struct device *dev;
0383     struct dma_device common;
0384     struct bam_chan *channels;
0385     u32 num_channels;
0386     u32 num_ees;
0387 
0388     /* execution environment ID, from DT */
0389     u32 ee;
0390     bool controlled_remotely;
0391     bool powered_remotely;
0392     u32 active_channels;
0393 
0394     const struct reg_offset_data *layout;
0395 
0396     struct clk *bamclk;
0397     int irq;
0398 
0399     /* dma start transaction tasklet */
0400     struct tasklet_struct task;
0401 };
0402 
0403 /**
0404  * bam_addr - returns BAM register address
0405  * @bdev: bam device
0406  * @pipe: pipe instance (ignored when register doesn't have multiple instances)
0407  * @reg:  register enum
0408  */
0409 static inline void __iomem *bam_addr(struct bam_device *bdev, u32 pipe,
0410         enum bam_reg reg)
0411 {
0412     const struct reg_offset_data r = bdev->layout[reg];
0413 
0414     return bdev->regs + r.base_offset +
0415         r.pipe_mult * pipe +
0416         r.evnt_mult * pipe +
0417         r.ee_mult * bdev->ee;
0418 }
0419 
0420 /**
0421  * bam_reset() - reset and initialize BAM registers
0422  * @bdev: bam device
0423  */
0424 static void bam_reset(struct bam_device *bdev)
0425 {
0426     u32 val;
0427 
0428     /* s/w reset bam */
0429     /* after reset all pipes are disabled and idle */
0430     val = readl_relaxed(bam_addr(bdev, 0, BAM_CTRL));
0431     val |= BAM_SW_RST;
0432     writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL));
0433     val &= ~BAM_SW_RST;
0434     writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL));
0435 
0436     /* make sure previous stores are visible before enabling BAM */
0437     wmb();
0438 
0439     /* enable bam */
0440     val |= BAM_EN;
0441     writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL));
0442 
0443     /* set descriptor threshhold, start with 4 bytes */
0444     writel_relaxed(DEFAULT_CNT_THRSHLD,
0445             bam_addr(bdev, 0, BAM_DESC_CNT_TRSHLD));
0446 
0447     /* Enable default set of h/w workarounds, ie all except BAM_FULL_PIPE */
0448     writel_relaxed(BAM_CNFG_BITS_DEFAULT, bam_addr(bdev, 0, BAM_CNFG_BITS));
0449 
0450     /* enable irqs for errors */
0451     writel_relaxed(BAM_ERROR_EN | BAM_HRESP_ERR_EN,
0452             bam_addr(bdev, 0, BAM_IRQ_EN));
0453 
0454     /* unmask global bam interrupt */
0455     writel_relaxed(BAM_IRQ_MSK, bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE));
0456 }
0457 
0458 /**
0459  * bam_reset_channel - Reset individual BAM DMA channel
0460  * @bchan: bam channel
0461  *
0462  * This function resets a specific BAM channel
0463  */
0464 static void bam_reset_channel(struct bam_chan *bchan)
0465 {
0466     struct bam_device *bdev = bchan->bdev;
0467 
0468     lockdep_assert_held(&bchan->vc.lock);
0469 
0470     /* reset channel */
0471     writel_relaxed(1, bam_addr(bdev, bchan->id, BAM_P_RST));
0472     writel_relaxed(0, bam_addr(bdev, bchan->id, BAM_P_RST));
0473 
0474     /* don't allow cpu to reorder BAM register accesses done after this */
0475     wmb();
0476 
0477     /* make sure hw is initialized when channel is used the first time  */
0478     bchan->initialized = 0;
0479 }
0480 
0481 /**
0482  * bam_chan_init_hw - Initialize channel hardware
0483  * @bchan: bam channel
0484  * @dir: DMA transfer direction
0485  *
0486  * This function resets and initializes the BAM channel
0487  */
0488 static void bam_chan_init_hw(struct bam_chan *bchan,
0489     enum dma_transfer_direction dir)
0490 {
0491     struct bam_device *bdev = bchan->bdev;
0492     u32 val;
0493 
0494     /* Reset the channel to clear internal state of the FIFO */
0495     bam_reset_channel(bchan);
0496 
0497     /*
0498      * write out 8 byte aligned address.  We have enough space for this
0499      * because we allocated 1 more descriptor (8 bytes) than we can use
0500      */
0501     writel_relaxed(ALIGN(bchan->fifo_phys, sizeof(struct bam_desc_hw)),
0502             bam_addr(bdev, bchan->id, BAM_P_DESC_FIFO_ADDR));
0503     writel_relaxed(BAM_FIFO_SIZE,
0504             bam_addr(bdev, bchan->id, BAM_P_FIFO_SIZES));
0505 
0506     /* enable the per pipe interrupts, enable EOT, ERR, and INT irqs */
0507     writel_relaxed(P_DEFAULT_IRQS_EN,
0508             bam_addr(bdev, bchan->id, BAM_P_IRQ_EN));
0509 
0510     /* unmask the specific pipe and EE combo */
0511     val = readl_relaxed(bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE));
0512     val |= BIT(bchan->id);
0513     writel_relaxed(val, bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE));
0514 
0515     /* don't allow cpu to reorder the channel enable done below */
0516     wmb();
0517 
0518     /* set fixed direction and mode, then enable channel */
0519     val = P_EN | P_SYS_MODE;
0520     if (dir == DMA_DEV_TO_MEM)
0521         val |= P_DIRECTION;
0522 
0523     writel_relaxed(val, bam_addr(bdev, bchan->id, BAM_P_CTRL));
0524 
0525     bchan->initialized = 1;
0526 
0527     /* init FIFO pointers */
0528     bchan->head = 0;
0529     bchan->tail = 0;
0530 }
0531 
0532 /**
0533  * bam_alloc_chan - Allocate channel resources for DMA channel.
0534  * @chan: specified channel
0535  *
0536  * This function allocates the FIFO descriptor memory
0537  */
0538 static int bam_alloc_chan(struct dma_chan *chan)
0539 {
0540     struct bam_chan *bchan = to_bam_chan(chan);
0541     struct bam_device *bdev = bchan->bdev;
0542 
0543     if (bchan->fifo_virt)
0544         return 0;
0545 
0546     /* allocate FIFO descriptor space, but only if necessary */
0547     bchan->fifo_virt = dma_alloc_wc(bdev->dev, BAM_DESC_FIFO_SIZE,
0548                     &bchan->fifo_phys, GFP_KERNEL);
0549 
0550     if (!bchan->fifo_virt) {
0551         dev_err(bdev->dev, "Failed to allocate desc fifo\n");
0552         return -ENOMEM;
0553     }
0554 
0555     if (bdev->active_channels++ == 0 && bdev->powered_remotely)
0556         bam_reset(bdev);
0557 
0558     return 0;
0559 }
0560 
0561 /**
0562  * bam_free_chan - Frees dma resources associated with specific channel
0563  * @chan: specified channel
0564  *
0565  * Free the allocated fifo descriptor memory and channel resources
0566  *
0567  */
0568 static void bam_free_chan(struct dma_chan *chan)
0569 {
0570     struct bam_chan *bchan = to_bam_chan(chan);
0571     struct bam_device *bdev = bchan->bdev;
0572     u32 val;
0573     unsigned long flags;
0574     int ret;
0575 
0576     ret = pm_runtime_get_sync(bdev->dev);
0577     if (ret < 0)
0578         return;
0579 
0580     vchan_free_chan_resources(to_virt_chan(chan));
0581 
0582     if (!list_empty(&bchan->desc_list)) {
0583         dev_err(bchan->bdev->dev, "Cannot free busy channel\n");
0584         goto err;
0585     }
0586 
0587     spin_lock_irqsave(&bchan->vc.lock, flags);
0588     bam_reset_channel(bchan);
0589     spin_unlock_irqrestore(&bchan->vc.lock, flags);
0590 
0591     dma_free_wc(bdev->dev, BAM_DESC_FIFO_SIZE, bchan->fifo_virt,
0592             bchan->fifo_phys);
0593     bchan->fifo_virt = NULL;
0594 
0595     /* mask irq for pipe/channel */
0596     val = readl_relaxed(bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE));
0597     val &= ~BIT(bchan->id);
0598     writel_relaxed(val, bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE));
0599 
0600     /* disable irq */
0601     writel_relaxed(0, bam_addr(bdev, bchan->id, BAM_P_IRQ_EN));
0602 
0603     if (--bdev->active_channels == 0 && bdev->powered_remotely) {
0604         /* s/w reset bam */
0605         val = readl_relaxed(bam_addr(bdev, 0, BAM_CTRL));
0606         val |= BAM_SW_RST;
0607         writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL));
0608     }
0609 
0610 err:
0611     pm_runtime_mark_last_busy(bdev->dev);
0612     pm_runtime_put_autosuspend(bdev->dev);
0613 }
0614 
0615 /**
0616  * bam_slave_config - set slave configuration for channel
0617  * @chan: dma channel
0618  * @cfg: slave configuration
0619  *
0620  * Sets slave configuration for channel
0621  *
0622  */
0623 static int bam_slave_config(struct dma_chan *chan,
0624                 struct dma_slave_config *cfg)
0625 {
0626     struct bam_chan *bchan = to_bam_chan(chan);
0627     unsigned long flag;
0628 
0629     spin_lock_irqsave(&bchan->vc.lock, flag);
0630     memcpy(&bchan->slave, cfg, sizeof(*cfg));
0631     bchan->reconfigure = 1;
0632     spin_unlock_irqrestore(&bchan->vc.lock, flag);
0633 
0634     return 0;
0635 }
0636 
0637 /**
0638  * bam_prep_slave_sg - Prep slave sg transaction
0639  *
0640  * @chan: dma channel
0641  * @sgl: scatter gather list
0642  * @sg_len: length of sg
0643  * @direction: DMA transfer direction
0644  * @flags: DMA flags
0645  * @context: transfer context (unused)
0646  */
0647 static struct dma_async_tx_descriptor *bam_prep_slave_sg(struct dma_chan *chan,
0648     struct scatterlist *sgl, unsigned int sg_len,
0649     enum dma_transfer_direction direction, unsigned long flags,
0650     void *context)
0651 {
0652     struct bam_chan *bchan = to_bam_chan(chan);
0653     struct bam_device *bdev = bchan->bdev;
0654     struct bam_async_desc *async_desc;
0655     struct scatterlist *sg;
0656     u32 i;
0657     struct bam_desc_hw *desc;
0658     unsigned int num_alloc = 0;
0659 
0660 
0661     if (!is_slave_direction(direction)) {
0662         dev_err(bdev->dev, "invalid dma direction\n");
0663         return NULL;
0664     }
0665 
0666     /* calculate number of required entries */
0667     for_each_sg(sgl, sg, sg_len, i)
0668         num_alloc += DIV_ROUND_UP(sg_dma_len(sg), BAM_FIFO_SIZE);
0669 
0670     /* allocate enough room to accomodate the number of entries */
0671     async_desc = kzalloc(struct_size(async_desc, desc, num_alloc),
0672                  GFP_NOWAIT);
0673 
0674     if (!async_desc)
0675         return NULL;
0676 
0677     if (flags & DMA_PREP_FENCE)
0678         async_desc->flags |= DESC_FLAG_NWD;
0679 
0680     if (flags & DMA_PREP_INTERRUPT)
0681         async_desc->flags |= DESC_FLAG_EOT;
0682 
0683     async_desc->num_desc = num_alloc;
0684     async_desc->curr_desc = async_desc->desc;
0685     async_desc->dir = direction;
0686 
0687     /* fill in temporary descriptors */
0688     desc = async_desc->desc;
0689     for_each_sg(sgl, sg, sg_len, i) {
0690         unsigned int remainder = sg_dma_len(sg);
0691         unsigned int curr_offset = 0;
0692 
0693         do {
0694             if (flags & DMA_PREP_CMD)
0695                 desc->flags |= cpu_to_le16(DESC_FLAG_CMD);
0696 
0697             desc->addr = cpu_to_le32(sg_dma_address(sg) +
0698                          curr_offset);
0699 
0700             if (remainder > BAM_FIFO_SIZE) {
0701                 desc->size = cpu_to_le16(BAM_FIFO_SIZE);
0702                 remainder -= BAM_FIFO_SIZE;
0703                 curr_offset += BAM_FIFO_SIZE;
0704             } else {
0705                 desc->size = cpu_to_le16(remainder);
0706                 remainder = 0;
0707             }
0708 
0709             async_desc->length += le16_to_cpu(desc->size);
0710             desc++;
0711         } while (remainder > 0);
0712     }
0713 
0714     return vchan_tx_prep(&bchan->vc, &async_desc->vd, flags);
0715 }
0716 
0717 /**
0718  * bam_dma_terminate_all - terminate all transactions on a channel
0719  * @chan: bam dma channel
0720  *
0721  * Dequeues and frees all transactions
0722  * No callbacks are done
0723  *
0724  */
0725 static int bam_dma_terminate_all(struct dma_chan *chan)
0726 {
0727     struct bam_chan *bchan = to_bam_chan(chan);
0728     struct bam_async_desc *async_desc, *tmp;
0729     unsigned long flag;
0730     LIST_HEAD(head);
0731 
0732     /* remove all transactions, including active transaction */
0733     spin_lock_irqsave(&bchan->vc.lock, flag);
0734     /*
0735      * If we have transactions queued, then some might be committed to the
0736      * hardware in the desc fifo.  The only way to reset the desc fifo is
0737      * to do a hardware reset (either by pipe or the entire block).
0738      * bam_chan_init_hw() will trigger a pipe reset, and also reinit the
0739      * pipe.  If the pipe is left disabled (default state after pipe reset)
0740      * and is accessed by a connected hardware engine, a fatal error in
0741      * the BAM will occur.  There is a small window where this could happen
0742      * with bam_chan_init_hw(), but it is assumed that the caller has
0743      * stopped activity on any attached hardware engine.  Make sure to do
0744      * this first so that the BAM hardware doesn't cause memory corruption
0745      * by accessing freed resources.
0746      */
0747     if (!list_empty(&bchan->desc_list)) {
0748         async_desc = list_first_entry(&bchan->desc_list,
0749                           struct bam_async_desc, desc_node);
0750         bam_chan_init_hw(bchan, async_desc->dir);
0751     }
0752 
0753     list_for_each_entry_safe(async_desc, tmp,
0754                  &bchan->desc_list, desc_node) {
0755         list_add(&async_desc->vd.node, &bchan->vc.desc_issued);
0756         list_del(&async_desc->desc_node);
0757     }
0758 
0759     vchan_get_all_descriptors(&bchan->vc, &head);
0760     spin_unlock_irqrestore(&bchan->vc.lock, flag);
0761 
0762     vchan_dma_desc_free_list(&bchan->vc, &head);
0763 
0764     return 0;
0765 }
0766 
0767 /**
0768  * bam_pause - Pause DMA channel
0769  * @chan: dma channel
0770  *
0771  */
0772 static int bam_pause(struct dma_chan *chan)
0773 {
0774     struct bam_chan *bchan = to_bam_chan(chan);
0775     struct bam_device *bdev = bchan->bdev;
0776     unsigned long flag;
0777     int ret;
0778 
0779     ret = pm_runtime_get_sync(bdev->dev);
0780     if (ret < 0)
0781         return ret;
0782 
0783     spin_lock_irqsave(&bchan->vc.lock, flag);
0784     writel_relaxed(1, bam_addr(bdev, bchan->id, BAM_P_HALT));
0785     bchan->paused = 1;
0786     spin_unlock_irqrestore(&bchan->vc.lock, flag);
0787     pm_runtime_mark_last_busy(bdev->dev);
0788     pm_runtime_put_autosuspend(bdev->dev);
0789 
0790     return 0;
0791 }
0792 
0793 /**
0794  * bam_resume - Resume DMA channel operations
0795  * @chan: dma channel
0796  *
0797  */
0798 static int bam_resume(struct dma_chan *chan)
0799 {
0800     struct bam_chan *bchan = to_bam_chan(chan);
0801     struct bam_device *bdev = bchan->bdev;
0802     unsigned long flag;
0803     int ret;
0804 
0805     ret = pm_runtime_get_sync(bdev->dev);
0806     if (ret < 0)
0807         return ret;
0808 
0809     spin_lock_irqsave(&bchan->vc.lock, flag);
0810     writel_relaxed(0, bam_addr(bdev, bchan->id, BAM_P_HALT));
0811     bchan->paused = 0;
0812     spin_unlock_irqrestore(&bchan->vc.lock, flag);
0813     pm_runtime_mark_last_busy(bdev->dev);
0814     pm_runtime_put_autosuspend(bdev->dev);
0815 
0816     return 0;
0817 }
0818 
0819 /**
0820  * process_channel_irqs - processes the channel interrupts
0821  * @bdev: bam controller
0822  *
0823  * This function processes the channel interrupts
0824  *
0825  */
0826 static u32 process_channel_irqs(struct bam_device *bdev)
0827 {
0828     u32 i, srcs, pipe_stts, offset, avail;
0829     unsigned long flags;
0830     struct bam_async_desc *async_desc, *tmp;
0831 
0832     srcs = readl_relaxed(bam_addr(bdev, 0, BAM_IRQ_SRCS_EE));
0833 
0834     /* return early if no pipe/channel interrupts are present */
0835     if (!(srcs & P_IRQ))
0836         return srcs;
0837 
0838     for (i = 0; i < bdev->num_channels; i++) {
0839         struct bam_chan *bchan = &bdev->channels[i];
0840 
0841         if (!(srcs & BIT(i)))
0842             continue;
0843 
0844         /* clear pipe irq */
0845         pipe_stts = readl_relaxed(bam_addr(bdev, i, BAM_P_IRQ_STTS));
0846 
0847         writel_relaxed(pipe_stts, bam_addr(bdev, i, BAM_P_IRQ_CLR));
0848 
0849         spin_lock_irqsave(&bchan->vc.lock, flags);
0850 
0851         offset = readl_relaxed(bam_addr(bdev, i, BAM_P_SW_OFSTS)) &
0852                        P_SW_OFSTS_MASK;
0853         offset /= sizeof(struct bam_desc_hw);
0854 
0855         /* Number of bytes available to read */
0856         avail = CIRC_CNT(offset, bchan->head, MAX_DESCRIPTORS + 1);
0857 
0858         if (offset < bchan->head)
0859             avail--;
0860 
0861         list_for_each_entry_safe(async_desc, tmp,
0862                      &bchan->desc_list, desc_node) {
0863             /* Not enough data to read */
0864             if (avail < async_desc->xfer_len)
0865                 break;
0866 
0867             /* manage FIFO */
0868             bchan->head += async_desc->xfer_len;
0869             bchan->head %= MAX_DESCRIPTORS;
0870 
0871             async_desc->num_desc -= async_desc->xfer_len;
0872             async_desc->curr_desc += async_desc->xfer_len;
0873             avail -= async_desc->xfer_len;
0874 
0875             /*
0876              * if complete, process cookie. Otherwise
0877              * push back to front of desc_issued so that
0878              * it gets restarted by the tasklet
0879              */
0880             if (!async_desc->num_desc) {
0881                 vchan_cookie_complete(&async_desc->vd);
0882             } else {
0883                 list_add(&async_desc->vd.node,
0884                      &bchan->vc.desc_issued);
0885             }
0886             list_del(&async_desc->desc_node);
0887         }
0888 
0889         spin_unlock_irqrestore(&bchan->vc.lock, flags);
0890     }
0891 
0892     return srcs;
0893 }
0894 
0895 /**
0896  * bam_dma_irq - irq handler for bam controller
0897  * @irq: IRQ of interrupt
0898  * @data: callback data
0899  *
0900  * IRQ handler for the bam controller
0901  */
0902 static irqreturn_t bam_dma_irq(int irq, void *data)
0903 {
0904     struct bam_device *bdev = data;
0905     u32 clr_mask = 0, srcs = 0;
0906     int ret;
0907 
0908     srcs |= process_channel_irqs(bdev);
0909 
0910     /* kick off tasklet to start next dma transfer */
0911     if (srcs & P_IRQ)
0912         tasklet_schedule(&bdev->task);
0913 
0914     ret = pm_runtime_get_sync(bdev->dev);
0915     if (ret < 0)
0916         return IRQ_NONE;
0917 
0918     if (srcs & BAM_IRQ) {
0919         clr_mask = readl_relaxed(bam_addr(bdev, 0, BAM_IRQ_STTS));
0920 
0921         /*
0922          * don't allow reorder of the various accesses to the BAM
0923          * registers
0924          */
0925         mb();
0926 
0927         writel_relaxed(clr_mask, bam_addr(bdev, 0, BAM_IRQ_CLR));
0928     }
0929 
0930     pm_runtime_mark_last_busy(bdev->dev);
0931     pm_runtime_put_autosuspend(bdev->dev);
0932 
0933     return IRQ_HANDLED;
0934 }
0935 
0936 /**
0937  * bam_tx_status - returns status of transaction
0938  * @chan: dma channel
0939  * @cookie: transaction cookie
0940  * @txstate: DMA transaction state
0941  *
0942  * Return status of dma transaction
0943  */
0944 static enum dma_status bam_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
0945         struct dma_tx_state *txstate)
0946 {
0947     struct bam_chan *bchan = to_bam_chan(chan);
0948     struct bam_async_desc *async_desc;
0949     struct virt_dma_desc *vd;
0950     int ret;
0951     size_t residue = 0;
0952     unsigned int i;
0953     unsigned long flags;
0954 
0955     ret = dma_cookie_status(chan, cookie, txstate);
0956     if (ret == DMA_COMPLETE)
0957         return ret;
0958 
0959     if (!txstate)
0960         return bchan->paused ? DMA_PAUSED : ret;
0961 
0962     spin_lock_irqsave(&bchan->vc.lock, flags);
0963     vd = vchan_find_desc(&bchan->vc, cookie);
0964     if (vd) {
0965         residue = container_of(vd, struct bam_async_desc, vd)->length;
0966     } else {
0967         list_for_each_entry(async_desc, &bchan->desc_list, desc_node) {
0968             if (async_desc->vd.tx.cookie != cookie)
0969                 continue;
0970 
0971             for (i = 0; i < async_desc->num_desc; i++)
0972                 residue += le16_to_cpu(
0973                         async_desc->curr_desc[i].size);
0974         }
0975     }
0976 
0977     spin_unlock_irqrestore(&bchan->vc.lock, flags);
0978 
0979     dma_set_residue(txstate, residue);
0980 
0981     if (ret == DMA_IN_PROGRESS && bchan->paused)
0982         ret = DMA_PAUSED;
0983 
0984     return ret;
0985 }
0986 
0987 /**
0988  * bam_apply_new_config
0989  * @bchan: bam dma channel
0990  * @dir: DMA direction
0991  */
0992 static void bam_apply_new_config(struct bam_chan *bchan,
0993     enum dma_transfer_direction dir)
0994 {
0995     struct bam_device *bdev = bchan->bdev;
0996     u32 maxburst;
0997 
0998     if (!bdev->controlled_remotely) {
0999         if (dir == DMA_DEV_TO_MEM)
1000             maxburst = bchan->slave.src_maxburst;
1001         else
1002             maxburst = bchan->slave.dst_maxburst;
1003 
1004         writel_relaxed(maxburst,
1005                    bam_addr(bdev, 0, BAM_DESC_CNT_TRSHLD));
1006     }
1007 
1008     bchan->reconfigure = 0;
1009 }
1010 
1011 /**
1012  * bam_start_dma - start next transaction
1013  * @bchan: bam dma channel
1014  */
1015 static void bam_start_dma(struct bam_chan *bchan)
1016 {
1017     struct virt_dma_desc *vd = vchan_next_desc(&bchan->vc);
1018     struct bam_device *bdev = bchan->bdev;
1019     struct bam_async_desc *async_desc = NULL;
1020     struct bam_desc_hw *desc;
1021     struct bam_desc_hw *fifo = PTR_ALIGN(bchan->fifo_virt,
1022                     sizeof(struct bam_desc_hw));
1023     int ret;
1024     unsigned int avail;
1025     struct dmaengine_desc_callback cb;
1026 
1027     lockdep_assert_held(&bchan->vc.lock);
1028 
1029     if (!vd)
1030         return;
1031 
1032     ret = pm_runtime_get_sync(bdev->dev);
1033     if (ret < 0)
1034         return;
1035 
1036     while (vd && !IS_BUSY(bchan)) {
1037         list_del(&vd->node);
1038 
1039         async_desc = container_of(vd, struct bam_async_desc, vd);
1040 
1041         /* on first use, initialize the channel hardware */
1042         if (!bchan->initialized)
1043             bam_chan_init_hw(bchan, async_desc->dir);
1044 
1045         /* apply new slave config changes, if necessary */
1046         if (bchan->reconfigure)
1047             bam_apply_new_config(bchan, async_desc->dir);
1048 
1049         desc = async_desc->curr_desc;
1050         avail = CIRC_SPACE(bchan->tail, bchan->head,
1051                    MAX_DESCRIPTORS + 1);
1052 
1053         if (async_desc->num_desc > avail)
1054             async_desc->xfer_len = avail;
1055         else
1056             async_desc->xfer_len = async_desc->num_desc;
1057 
1058         /* set any special flags on the last descriptor */
1059         if (async_desc->num_desc == async_desc->xfer_len)
1060             desc[async_desc->xfer_len - 1].flags |=
1061                         cpu_to_le16(async_desc->flags);
1062 
1063         vd = vchan_next_desc(&bchan->vc);
1064 
1065         dmaengine_desc_get_callback(&async_desc->vd.tx, &cb);
1066 
1067         /*
1068          * An interrupt is generated at this desc, if
1069          *  - FIFO is FULL.
1070          *  - No more descriptors to add.
1071          *  - If a callback completion was requested for this DESC,
1072          *     In this case, BAM will deliver the completion callback
1073          *     for this desc and continue processing the next desc.
1074          */
1075         if (((avail <= async_desc->xfer_len) || !vd ||
1076              dmaengine_desc_callback_valid(&cb)) &&
1077             !(async_desc->flags & DESC_FLAG_EOT))
1078             desc[async_desc->xfer_len - 1].flags |=
1079                 cpu_to_le16(DESC_FLAG_INT);
1080 
1081         if (bchan->tail + async_desc->xfer_len > MAX_DESCRIPTORS) {
1082             u32 partial = MAX_DESCRIPTORS - bchan->tail;
1083 
1084             memcpy(&fifo[bchan->tail], desc,
1085                    partial * sizeof(struct bam_desc_hw));
1086             memcpy(fifo, &desc[partial],
1087                    (async_desc->xfer_len - partial) *
1088                 sizeof(struct bam_desc_hw));
1089         } else {
1090             memcpy(&fifo[bchan->tail], desc,
1091                    async_desc->xfer_len *
1092                    sizeof(struct bam_desc_hw));
1093         }
1094 
1095         bchan->tail += async_desc->xfer_len;
1096         bchan->tail %= MAX_DESCRIPTORS;
1097         list_add_tail(&async_desc->desc_node, &bchan->desc_list);
1098     }
1099 
1100     /* ensure descriptor writes and dma start not reordered */
1101     wmb();
1102     writel_relaxed(bchan->tail * sizeof(struct bam_desc_hw),
1103             bam_addr(bdev, bchan->id, BAM_P_EVNT_REG));
1104 
1105     pm_runtime_mark_last_busy(bdev->dev);
1106     pm_runtime_put_autosuspend(bdev->dev);
1107 }
1108 
1109 /**
1110  * dma_tasklet - DMA IRQ tasklet
1111  * @t: tasklet argument (bam controller structure)
1112  *
1113  * Sets up next DMA operation and then processes all completed transactions
1114  */
1115 static void dma_tasklet(struct tasklet_struct *t)
1116 {
1117     struct bam_device *bdev = from_tasklet(bdev, t, task);
1118     struct bam_chan *bchan;
1119     unsigned long flags;
1120     unsigned int i;
1121 
1122     /* go through the channels and kick off transactions */
1123     for (i = 0; i < bdev->num_channels; i++) {
1124         bchan = &bdev->channels[i];
1125         spin_lock_irqsave(&bchan->vc.lock, flags);
1126 
1127         if (!list_empty(&bchan->vc.desc_issued) && !IS_BUSY(bchan))
1128             bam_start_dma(bchan);
1129         spin_unlock_irqrestore(&bchan->vc.lock, flags);
1130     }
1131 
1132 }
1133 
1134 /**
1135  * bam_issue_pending - starts pending transactions
1136  * @chan: dma channel
1137  *
1138  * Calls tasklet directly which in turn starts any pending transactions
1139  */
1140 static void bam_issue_pending(struct dma_chan *chan)
1141 {
1142     struct bam_chan *bchan = to_bam_chan(chan);
1143     unsigned long flags;
1144 
1145     spin_lock_irqsave(&bchan->vc.lock, flags);
1146 
1147     /* if work pending and idle, start a transaction */
1148     if (vchan_issue_pending(&bchan->vc) && !IS_BUSY(bchan))
1149         bam_start_dma(bchan);
1150 
1151     spin_unlock_irqrestore(&bchan->vc.lock, flags);
1152 }
1153 
1154 /**
1155  * bam_dma_free_desc - free descriptor memory
1156  * @vd: virtual descriptor
1157  *
1158  */
1159 static void bam_dma_free_desc(struct virt_dma_desc *vd)
1160 {
1161     struct bam_async_desc *async_desc = container_of(vd,
1162             struct bam_async_desc, vd);
1163 
1164     kfree(async_desc);
1165 }
1166 
1167 static struct dma_chan *bam_dma_xlate(struct of_phandle_args *dma_spec,
1168         struct of_dma *of)
1169 {
1170     struct bam_device *bdev = container_of(of->of_dma_data,
1171                     struct bam_device, common);
1172     unsigned int request;
1173 
1174     if (dma_spec->args_count != 1)
1175         return NULL;
1176 
1177     request = dma_spec->args[0];
1178     if (request >= bdev->num_channels)
1179         return NULL;
1180 
1181     return dma_get_slave_channel(&(bdev->channels[request].vc.chan));
1182 }
1183 
1184 /**
1185  * bam_init
1186  * @bdev: bam device
1187  *
1188  * Initialization helper for global bam registers
1189  */
1190 static int bam_init(struct bam_device *bdev)
1191 {
1192     u32 val;
1193 
1194     /* read revision and configuration information */
1195     if (!bdev->num_ees) {
1196         val = readl_relaxed(bam_addr(bdev, 0, BAM_REVISION));
1197         bdev->num_ees = (val >> NUM_EES_SHIFT) & NUM_EES_MASK;
1198     }
1199 
1200     /* check that configured EE is within range */
1201     if (bdev->ee >= bdev->num_ees)
1202         return -EINVAL;
1203 
1204     if (!bdev->num_channels) {
1205         val = readl_relaxed(bam_addr(bdev, 0, BAM_NUM_PIPES));
1206         bdev->num_channels = val & BAM_NUM_PIPES_MASK;
1207     }
1208 
1209     /* Reset BAM now if fully controlled locally */
1210     if (!bdev->controlled_remotely && !bdev->powered_remotely)
1211         bam_reset(bdev);
1212 
1213     return 0;
1214 }
1215 
1216 static void bam_channel_init(struct bam_device *bdev, struct bam_chan *bchan,
1217     u32 index)
1218 {
1219     bchan->id = index;
1220     bchan->bdev = bdev;
1221 
1222     vchan_init(&bchan->vc, &bdev->common);
1223     bchan->vc.desc_free = bam_dma_free_desc;
1224     INIT_LIST_HEAD(&bchan->desc_list);
1225 }
1226 
1227 static const struct of_device_id bam_of_match[] = {
1228     { .compatible = "qcom,bam-v1.3.0", .data = &bam_v1_3_reg_info },
1229     { .compatible = "qcom,bam-v1.4.0", .data = &bam_v1_4_reg_info },
1230     { .compatible = "qcom,bam-v1.7.0", .data = &bam_v1_7_reg_info },
1231     {}
1232 };
1233 
1234 MODULE_DEVICE_TABLE(of, bam_of_match);
1235 
1236 static int bam_dma_probe(struct platform_device *pdev)
1237 {
1238     struct bam_device *bdev;
1239     const struct of_device_id *match;
1240     struct resource *iores;
1241     int ret, i;
1242 
1243     bdev = devm_kzalloc(&pdev->dev, sizeof(*bdev), GFP_KERNEL);
1244     if (!bdev)
1245         return -ENOMEM;
1246 
1247     bdev->dev = &pdev->dev;
1248 
1249     match = of_match_node(bam_of_match, pdev->dev.of_node);
1250     if (!match) {
1251         dev_err(&pdev->dev, "Unsupported BAM module\n");
1252         return -ENODEV;
1253     }
1254 
1255     bdev->layout = match->data;
1256 
1257     iores = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1258     bdev->regs = devm_ioremap_resource(&pdev->dev, iores);
1259     if (IS_ERR(bdev->regs))
1260         return PTR_ERR(bdev->regs);
1261 
1262     bdev->irq = platform_get_irq(pdev, 0);
1263     if (bdev->irq < 0)
1264         return bdev->irq;
1265 
1266     ret = of_property_read_u32(pdev->dev.of_node, "qcom,ee", &bdev->ee);
1267     if (ret) {
1268         dev_err(bdev->dev, "Execution environment unspecified\n");
1269         return ret;
1270     }
1271 
1272     bdev->controlled_remotely = of_property_read_bool(pdev->dev.of_node,
1273                         "qcom,controlled-remotely");
1274     bdev->powered_remotely = of_property_read_bool(pdev->dev.of_node,
1275                         "qcom,powered-remotely");
1276 
1277     if (bdev->controlled_remotely || bdev->powered_remotely) {
1278         ret = of_property_read_u32(pdev->dev.of_node, "num-channels",
1279                        &bdev->num_channels);
1280         if (ret)
1281             dev_err(bdev->dev, "num-channels unspecified in dt\n");
1282 
1283         ret = of_property_read_u32(pdev->dev.of_node, "qcom,num-ees",
1284                        &bdev->num_ees);
1285         if (ret)
1286             dev_err(bdev->dev, "num-ees unspecified in dt\n");
1287     }
1288 
1289     if (bdev->controlled_remotely || bdev->powered_remotely)
1290         bdev->bamclk = devm_clk_get_optional(bdev->dev, "bam_clk");
1291     else
1292         bdev->bamclk = devm_clk_get(bdev->dev, "bam_clk");
1293 
1294     if (IS_ERR(bdev->bamclk))
1295         return PTR_ERR(bdev->bamclk);
1296 
1297     ret = clk_prepare_enable(bdev->bamclk);
1298     if (ret) {
1299         dev_err(bdev->dev, "failed to prepare/enable clock\n");
1300         return ret;
1301     }
1302 
1303     ret = bam_init(bdev);
1304     if (ret)
1305         goto err_disable_clk;
1306 
1307     tasklet_setup(&bdev->task, dma_tasklet);
1308 
1309     bdev->channels = devm_kcalloc(bdev->dev, bdev->num_channels,
1310                 sizeof(*bdev->channels), GFP_KERNEL);
1311 
1312     if (!bdev->channels) {
1313         ret = -ENOMEM;
1314         goto err_tasklet_kill;
1315     }
1316 
1317     /* allocate and initialize channels */
1318     INIT_LIST_HEAD(&bdev->common.channels);
1319 
1320     for (i = 0; i < bdev->num_channels; i++)
1321         bam_channel_init(bdev, &bdev->channels[i], i);
1322 
1323     ret = devm_request_irq(bdev->dev, bdev->irq, bam_dma_irq,
1324             IRQF_TRIGGER_HIGH, "bam_dma", bdev);
1325     if (ret)
1326         goto err_bam_channel_exit;
1327 
1328     /* set max dma segment size */
1329     bdev->common.dev = bdev->dev;
1330     ret = dma_set_max_seg_size(bdev->common.dev, BAM_FIFO_SIZE);
1331     if (ret) {
1332         dev_err(bdev->dev, "cannot set maximum segment size\n");
1333         goto err_bam_channel_exit;
1334     }
1335 
1336     platform_set_drvdata(pdev, bdev);
1337 
1338     /* set capabilities */
1339     dma_cap_zero(bdev->common.cap_mask);
1340     dma_cap_set(DMA_SLAVE, bdev->common.cap_mask);
1341 
1342     /* initialize dmaengine apis */
1343     bdev->common.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
1344     bdev->common.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
1345     bdev->common.src_addr_widths = DMA_SLAVE_BUSWIDTH_4_BYTES;
1346     bdev->common.dst_addr_widths = DMA_SLAVE_BUSWIDTH_4_BYTES;
1347     bdev->common.device_alloc_chan_resources = bam_alloc_chan;
1348     bdev->common.device_free_chan_resources = bam_free_chan;
1349     bdev->common.device_prep_slave_sg = bam_prep_slave_sg;
1350     bdev->common.device_config = bam_slave_config;
1351     bdev->common.device_pause = bam_pause;
1352     bdev->common.device_resume = bam_resume;
1353     bdev->common.device_terminate_all = bam_dma_terminate_all;
1354     bdev->common.device_issue_pending = bam_issue_pending;
1355     bdev->common.device_tx_status = bam_tx_status;
1356     bdev->common.dev = bdev->dev;
1357 
1358     ret = dma_async_device_register(&bdev->common);
1359     if (ret) {
1360         dev_err(bdev->dev, "failed to register dma async device\n");
1361         goto err_bam_channel_exit;
1362     }
1363 
1364     ret = of_dma_controller_register(pdev->dev.of_node, bam_dma_xlate,
1365                     &bdev->common);
1366     if (ret)
1367         goto err_unregister_dma;
1368 
1369     pm_runtime_irq_safe(&pdev->dev);
1370     pm_runtime_set_autosuspend_delay(&pdev->dev, BAM_DMA_AUTOSUSPEND_DELAY);
1371     pm_runtime_use_autosuspend(&pdev->dev);
1372     pm_runtime_mark_last_busy(&pdev->dev);
1373     pm_runtime_set_active(&pdev->dev);
1374     pm_runtime_enable(&pdev->dev);
1375 
1376     return 0;
1377 
1378 err_unregister_dma:
1379     dma_async_device_unregister(&bdev->common);
1380 err_bam_channel_exit:
1381     for (i = 0; i < bdev->num_channels; i++)
1382         tasklet_kill(&bdev->channels[i].vc.task);
1383 err_tasklet_kill:
1384     tasklet_kill(&bdev->task);
1385 err_disable_clk:
1386     clk_disable_unprepare(bdev->bamclk);
1387 
1388     return ret;
1389 }
1390 
1391 static int bam_dma_remove(struct platform_device *pdev)
1392 {
1393     struct bam_device *bdev = platform_get_drvdata(pdev);
1394     u32 i;
1395 
1396     pm_runtime_force_suspend(&pdev->dev);
1397 
1398     of_dma_controller_free(pdev->dev.of_node);
1399     dma_async_device_unregister(&bdev->common);
1400 
1401     /* mask all interrupts for this execution environment */
1402     writel_relaxed(0, bam_addr(bdev, 0,  BAM_IRQ_SRCS_MSK_EE));
1403 
1404     devm_free_irq(bdev->dev, bdev->irq, bdev);
1405 
1406     for (i = 0; i < bdev->num_channels; i++) {
1407         bam_dma_terminate_all(&bdev->channels[i].vc.chan);
1408         tasklet_kill(&bdev->channels[i].vc.task);
1409 
1410         if (!bdev->channels[i].fifo_virt)
1411             continue;
1412 
1413         dma_free_wc(bdev->dev, BAM_DESC_FIFO_SIZE,
1414                 bdev->channels[i].fifo_virt,
1415                 bdev->channels[i].fifo_phys);
1416     }
1417 
1418     tasklet_kill(&bdev->task);
1419 
1420     clk_disable_unprepare(bdev->bamclk);
1421 
1422     return 0;
1423 }
1424 
1425 static int __maybe_unused bam_dma_runtime_suspend(struct device *dev)
1426 {
1427     struct bam_device *bdev = dev_get_drvdata(dev);
1428 
1429     clk_disable(bdev->bamclk);
1430 
1431     return 0;
1432 }
1433 
1434 static int __maybe_unused bam_dma_runtime_resume(struct device *dev)
1435 {
1436     struct bam_device *bdev = dev_get_drvdata(dev);
1437     int ret;
1438 
1439     ret = clk_enable(bdev->bamclk);
1440     if (ret < 0) {
1441         dev_err(dev, "clk_enable failed: %d\n", ret);
1442         return ret;
1443     }
1444 
1445     return 0;
1446 }
1447 
1448 static int __maybe_unused bam_dma_suspend(struct device *dev)
1449 {
1450     struct bam_device *bdev = dev_get_drvdata(dev);
1451 
1452     pm_runtime_force_suspend(dev);
1453     clk_unprepare(bdev->bamclk);
1454 
1455     return 0;
1456 }
1457 
1458 static int __maybe_unused bam_dma_resume(struct device *dev)
1459 {
1460     struct bam_device *bdev = dev_get_drvdata(dev);
1461     int ret;
1462 
1463     ret = clk_prepare(bdev->bamclk);
1464     if (ret)
1465         return ret;
1466 
1467     pm_runtime_force_resume(dev);
1468 
1469     return 0;
1470 }
1471 
1472 static const struct dev_pm_ops bam_dma_pm_ops = {
1473     SET_LATE_SYSTEM_SLEEP_PM_OPS(bam_dma_suspend, bam_dma_resume)
1474     SET_RUNTIME_PM_OPS(bam_dma_runtime_suspend, bam_dma_runtime_resume,
1475                 NULL)
1476 };
1477 
1478 static struct platform_driver bam_dma_driver = {
1479     .probe = bam_dma_probe,
1480     .remove = bam_dma_remove,
1481     .driver = {
1482         .name = "bam-dma-engine",
1483         .pm = &bam_dma_pm_ops,
1484         .of_match_table = bam_of_match,
1485     },
1486 };
1487 
1488 module_platform_driver(bam_dma_driver);
1489 
1490 MODULE_AUTHOR("Andy Gross <agross@codeaurora.org>");
1491 MODULE_DESCRIPTION("QCOM BAM DMA engine driver");
1492 MODULE_LICENSE("GPL v2");
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!