dma/dw/regs.h

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 /*
0003  * Driver for the Synopsys DesignWare AHB DMA Controller
0004  *
0005  * Copyright (C) 2005-2007 Atmel Corporation
0006  * Copyright (C) 2010-2011 ST Microelectronics
0007  * Copyright (C) 2016 Intel Corporation
0008  */
0009
0010 #include <linux/bitops.h>
0011 #include <linux/interrupt.h>
0012 #include <linux/dmaengine.h>
0013
0014 #include <linux/io-64-nonatomic-hi-lo.h>
0015
0016 #include "internal.h"
0017
0018 #define DW_DMA_MAX_NR_REQUESTS  16
0019
0020 /* flow controller */
0021 enum dw_dma_fc {
0022     DW_DMA_FC_D_M2M,
0023     DW_DMA_FC_D_M2P,
0024     DW_DMA_FC_D_P2M,
0025     DW_DMA_FC_D_P2P,
0026     DW_DMA_FC_P_P2M,
0027     DW_DMA_FC_SP_P2P,
0028     DW_DMA_FC_P_M2P,
0029     DW_DMA_FC_DP_P2P,
0030 };
0031
0032 /*
0033  * Redefine this macro to handle differences between 32- and 64-bit
0034  * addressing, big vs. little endian, etc.
0035  */
0036 #define DW_REG(name)        u32 name; u32 __pad_##name
0037
0038 /* Hardware register definitions. */
0039 struct dw_dma_chan_regs {
0040     DW_REG(SAR);        /* Source Address Register */
0041     DW_REG(DAR);        /* Destination Address Register */
0042     DW_REG(LLP);        /* Linked List Pointer */
0043     u32 CTL_LO;     /* Control Register Low */
0044     u32 CTL_HI;     /* Control Register High */
0045     DW_REG(SSTAT);
0046     DW_REG(DSTAT);
0047     DW_REG(SSTATAR);
0048     DW_REG(DSTATAR);
0049     u32 CFG_LO;     /* Configuration Register Low */
0050     u32 CFG_HI;     /* Configuration Register High */
0051     DW_REG(SGR);
0052     DW_REG(DSR);
0053 };
0054
0055 struct dw_dma_irq_regs {
0056     DW_REG(XFER);
0057     DW_REG(BLOCK);
0058     DW_REG(SRC_TRAN);
0059     DW_REG(DST_TRAN);
0060     DW_REG(ERROR);
0061 };
0062
0063 struct dw_dma_regs {
0064     /* per-channel registers */
0065     struct dw_dma_chan_regs CHAN[DW_DMA_MAX_NR_CHANNELS];
0066
0067     /* irq handling */
0068     struct dw_dma_irq_regs  RAW;        /* r */
0069     struct dw_dma_irq_regs  STATUS;     /* r (raw & mask) */
0070     struct dw_dma_irq_regs  MASK;       /* rw (set = irq enabled) */
0071     struct dw_dma_irq_regs  CLEAR;      /* w (ack, affects "raw") */
0072
0073     DW_REG(STATUS_INT);         /* r */
0074
0075     /* software handshaking */
0076     DW_REG(REQ_SRC);
0077     DW_REG(REQ_DST);
0078     DW_REG(SGL_REQ_SRC);
0079     DW_REG(SGL_REQ_DST);
0080     DW_REG(LAST_SRC);
0081     DW_REG(LAST_DST);
0082
0083     /* miscellaneous */
0084     DW_REG(CFG);
0085     DW_REG(CH_EN);
0086     DW_REG(ID);
0087     DW_REG(TEST);
0088
0089     /* iDMA 32-bit support */
0090     DW_REG(CLASS_PRIORITY0);
0091     DW_REG(CLASS_PRIORITY1);
0092
0093     /* optional encoded params, 0x3c8..0x3f7 */
0094     u32 __reserved;
0095
0096     /* per-channel configuration registers */
0097     u32 DWC_PARAMS[DW_DMA_MAX_NR_CHANNELS];
0098     u32 MULTI_BLK_TYPE;
0099     u32 MAX_BLK_SIZE;
0100
0101     /* top-level parameters */
0102     u32 DW_PARAMS;
0103
0104     /* component ID */
0105     u32 COMP_TYPE;
0106     u32 COMP_VERSION;
0107
0108     /* iDMA 32-bit support */
0109     DW_REG(FIFO_PARTITION0);
0110     DW_REG(FIFO_PARTITION1);
0111
0112     DW_REG(SAI_ERR);
0113     DW_REG(GLOBAL_CFG);
0114 };
0115
0116 /* Bitfields in DW_PARAMS */
0117 #define DW_PARAMS_NR_CHAN   8       /* number of channels */
0118 #define DW_PARAMS_NR_MASTER 11      /* number of AHB masters */
0119 #define DW_PARAMS_DATA_WIDTH(n) (15 + 2 * (n))
0120 #define DW_PARAMS_DATA_WIDTH1   15      /* master 1 data width */
0121 #define DW_PARAMS_DATA_WIDTH2   17      /* master 2 data width */
0122 #define DW_PARAMS_DATA_WIDTH3   19      /* master 3 data width */
0123 #define DW_PARAMS_DATA_WIDTH4   21      /* master 4 data width */
0124 #define DW_PARAMS_EN        28      /* encoded parameters */
0125
0126 /* Bitfields in DWC_PARAMS */
0127 #define DWC_PARAMS_MBLK_EN  11      /* multi block transfer */
0128 #define DWC_PARAMS_HC_LLP   13      /* set LLP register to zero */
0129 #define DWC_PARAMS_MSIZE    16      /* max group transaction size */
0130
0131 /* bursts size */
0132 enum dw_dma_msize {
0133     DW_DMA_MSIZE_1,
0134     DW_DMA_MSIZE_4,
0135     DW_DMA_MSIZE_8,
0136     DW_DMA_MSIZE_16,
0137     DW_DMA_MSIZE_32,
0138     DW_DMA_MSIZE_64,
0139     DW_DMA_MSIZE_128,
0140     DW_DMA_MSIZE_256,
0141 };
0142
0143 /* Bitfields in LLP */
0144 #define DWC_LLP_LMS(x)      ((x) & 3)   /* list master select */
0145 #define DWC_LLP_LOC(x)      ((x) & ~3)  /* next lli */
0146
0147 /* Bitfields in CTL_LO */
0148 #define DWC_CTLL_INT_EN     (1 << 0)    /* irqs enabled? */
0149 #define DWC_CTLL_DST_WIDTH(n)   ((n)<<1)    /* bytes per element */
0150 #define DWC_CTLL_SRC_WIDTH(n)   ((n)<<4)
0151 #define DWC_CTLL_DST_INC    (0<<7)      /* DAR update/not */
0152 #define DWC_CTLL_DST_DEC    (1<<7)
0153 #define DWC_CTLL_DST_FIX    (2<<7)
0154 #define DWC_CTLL_SRC_INC    (0<<9)      /* SAR update/not */
0155 #define DWC_CTLL_SRC_DEC    (1<<9)
0156 #define DWC_CTLL_SRC_FIX    (2<<9)
0157 #define DWC_CTLL_DST_MSIZE(n)   ((n)<<11)   /* burst, #elements */
0158 #define DWC_CTLL_SRC_MSIZE(n)   ((n)<<14)
0159 #define DWC_CTLL_S_GATH_EN  (1 << 17)   /* src gather, !FIX */
0160 #define DWC_CTLL_D_SCAT_EN  (1 << 18)   /* dst scatter, !FIX */
0161 #define DWC_CTLL_FC(n)      ((n) << 20)
0162 #define DWC_CTLL_FC_M2M     (0 << 20)   /* mem-to-mem */
0163 #define DWC_CTLL_FC_M2P     (1 << 20)   /* mem-to-periph */
0164 #define DWC_CTLL_FC_P2M     (2 << 20)   /* periph-to-mem */
0165 #define DWC_CTLL_FC_P2P     (3 << 20)   /* periph-to-periph */
0166 /* plus 4 transfer types for peripheral-as-flow-controller */
0167 #define DWC_CTLL_DMS(n)     ((n)<<23)   /* dst master select */
0168 #define DWC_CTLL_SMS(n)     ((n)<<25)   /* src master select */
0169 #define DWC_CTLL_LLP_D_EN   (1 << 27)   /* dest block chain */
0170 #define DWC_CTLL_LLP_S_EN   (1 << 28)   /* src block chain */
0171
0172 /* Bitfields in CTL_HI */
0173 #define DWC_CTLH_BLOCK_TS_MASK  GENMASK(11, 0)
0174 #define DWC_CTLH_BLOCK_TS(x)    ((x) & DWC_CTLH_BLOCK_TS_MASK)
0175 #define DWC_CTLH_DONE       (1 << 12)
0176
0177 /* Bitfields in CFG_LO */
0178 #define DWC_CFGL_CH_PRIOR_MASK  (0x7 << 5)  /* priority mask */
0179 #define DWC_CFGL_CH_PRIOR(x)    ((x) << 5)  /* priority */
0180 #define DWC_CFGL_CH_SUSP    (1 << 8)    /* pause xfer */
0181 #define DWC_CFGL_FIFO_EMPTY (1 << 9)    /* pause xfer */
0182 #define DWC_CFGL_HS_DST     (1 << 10)   /* handshake w/dst */
0183 #define DWC_CFGL_HS_SRC     (1 << 11)   /* handshake w/src */
0184 #define DWC_CFGL_LOCK_CH_XFER   (0 << 12)   /* scope of LOCK_CH */
0185 #define DWC_CFGL_LOCK_CH_BLOCK  (1 << 12)
0186 #define DWC_CFGL_LOCK_CH_XACT   (2 << 12)
0187 #define DWC_CFGL_LOCK_BUS_XFER  (0 << 14)   /* scope of LOCK_BUS */
0188 #define DWC_CFGL_LOCK_BUS_BLOCK (1 << 14)
0189 #define DWC_CFGL_LOCK_BUS_XACT  (2 << 14)
0190 #define DWC_CFGL_LOCK_CH    (1 << 15)   /* channel lockout */
0191 #define DWC_CFGL_LOCK_BUS   (1 << 16)   /* busmaster lockout */
0192 #define DWC_CFGL_HS_DST_POL (1 << 18)   /* dst handshake active low */
0193 #define DWC_CFGL_HS_SRC_POL (1 << 19)   /* src handshake active low */
0194 #define DWC_CFGL_MAX_BURST(x)   ((x) << 20)
0195 #define DWC_CFGL_RELOAD_SAR (1 << 30)
0196 #define DWC_CFGL_RELOAD_DAR (1 << 31)
0197
0198 /* Bitfields in CFG_HI */
0199 #define DWC_CFGH_FCMODE     (1 << 0)
0200 #define DWC_CFGH_FIFO_MODE  (1 << 1)
0201 #define DWC_CFGH_PROTCTL(x) ((x) << 2)
0202 #define DWC_CFGH_PROTCTL_DATA   (0 << 2)    /* data access - always set */
0203 #define DWC_CFGH_PROTCTL_PRIV   (1 << 2)    /* privileged -> AHB HPROT[1] */
0204 #define DWC_CFGH_PROTCTL_BUFFER (2 << 2)    /* bufferable -> AHB HPROT[2] */
0205 #define DWC_CFGH_PROTCTL_CACHE  (4 << 2)    /* cacheable  -> AHB HPROT[3] */
0206 #define DWC_CFGH_DS_UPD_EN  (1 << 5)
0207 #define DWC_CFGH_SS_UPD_EN  (1 << 6)
0208 #define DWC_CFGH_SRC_PER(x) ((x) << 7)
0209 #define DWC_CFGH_DST_PER(x) ((x) << 11)
0210
0211 /* Bitfields in SGR */
0212 #define DWC_SGR_SGI(x)      ((x) << 0)
0213 #define DWC_SGR_SGC(x)      ((x) << 20)
0214
0215 /* Bitfields in DSR */
0216 #define DWC_DSR_DSI(x)      ((x) << 0)
0217 #define DWC_DSR_DSC(x)      ((x) << 20)
0218
0219 /* Bitfields in CFG */
0220 #define DW_CFG_DMA_EN       (1 << 0)
0221
0222 /* iDMA 32-bit support */
0223
0224 /* bursts size */
0225 enum idma32_msize {
0226     IDMA32_MSIZE_1,
0227     IDMA32_MSIZE_2,
0228     IDMA32_MSIZE_4,
0229     IDMA32_MSIZE_8,
0230     IDMA32_MSIZE_16,
0231     IDMA32_MSIZE_32,
0232 };
0233
0234 /* Bitfields in CTL_HI */
0235 #define IDMA32C_CTLH_BLOCK_TS_MASK  GENMASK(16, 0)
0236 #define IDMA32C_CTLH_BLOCK_TS(x)    ((x) & IDMA32C_CTLH_BLOCK_TS_MASK)
0237 #define IDMA32C_CTLH_DONE       (1 << 17)
0238
0239 /* Bitfields in CFG_LO */
0240 #define IDMA32C_CFGL_DST_BURST_ALIGN    (1 << 0)    /* dst burst align */
0241 #define IDMA32C_CFGL_SRC_BURST_ALIGN    (1 << 1)    /* src burst align */
0242 #define IDMA32C_CFGL_CH_DRAIN       (1 << 10)   /* drain FIFO */
0243 #define IDMA32C_CFGL_DST_OPT_BL     (1 << 20)   /* optimize dst burst length */
0244 #define IDMA32C_CFGL_SRC_OPT_BL     (1 << 21)   /* optimize src burst length */
0245
0246 /* Bitfields in CFG_HI */
0247 #define IDMA32C_CFGH_SRC_PER(x)     ((x) << 0)
0248 #define IDMA32C_CFGH_DST_PER(x)     ((x) << 4)
0249 #define IDMA32C_CFGH_RD_ISSUE_THD(x)    ((x) << 8)
0250 #define IDMA32C_CFGH_RW_ISSUE_THD(x)    ((x) << 18)
0251 #define IDMA32C_CFGH_SRC_PER_EXT(x) ((x) << 28) /* src peripheral extension */
0252 #define IDMA32C_CFGH_DST_PER_EXT(x) ((x) << 30) /* dst peripheral extension */
0253
0254 /* Bitfields in FIFO_PARTITION */
0255 #define IDMA32C_FP_PSIZE_CH0(x)     ((x) << 0)
0256 #define IDMA32C_FP_PSIZE_CH1(x)     ((x) << 13)
0257 #define IDMA32C_FP_UPDATE       (1 << 26)
0258
0259 enum dw_dmac_flags {
0260     DW_DMA_IS_CYCLIC = 0,
0261     DW_DMA_IS_SOFT_LLP = 1,
0262     DW_DMA_IS_PAUSED = 2,
0263     DW_DMA_IS_INITIALIZED = 3,
0264 };
0265
0266 struct dw_dma_chan {
0267     struct dma_chan         chan;
0268     void __iomem            *ch_regs;
0269     u8              mask;
0270     u8              priority;
0271     enum dma_transfer_direction direction;
0272
0273     /* software emulation of the LLP transfers */
0274     struct list_head    *tx_node_active;
0275
0276     spinlock_t      lock;
0277
0278     /* these other elements are all protected by lock */
0279     unsigned long       flags;
0280     struct list_head    active_list;
0281     struct list_head    queue;
0282
0283     unsigned int        descs_allocated;
0284
0285     /* hardware configuration */
0286     unsigned int        block_size;
0287     bool            nollp;
0288     u32         max_burst;
0289
0290     /* custom slave configuration */
0291     struct dw_dma_slave dws;
0292
0293     /* configuration passed via .device_config */
0294     struct dma_slave_config dma_sconfig;
0295 };
0296
0297 static inline struct dw_dma_chan_regs __iomem *
0298 __dwc_regs(struct dw_dma_chan *dwc)
0299 {
0300     return dwc->ch_regs;
0301 }
0302
0303 #define channel_readl(dwc, name) \
0304     readl(&(__dwc_regs(dwc)->name))
0305 #define channel_writel(dwc, name, val) \
0306     writel((val), &(__dwc_regs(dwc)->name))
0307
0308 static inline struct dw_dma_chan *to_dw_dma_chan(struct dma_chan *chan)
0309 {
0310     return container_of(chan, struct dw_dma_chan, chan);
0311 }
0312
0313 struct dw_dma {
0314     struct dma_device   dma;
0315     char            name[20];
0316     void __iomem        *regs;
0317     struct dma_pool     *desc_pool;
0318     struct tasklet_struct   tasklet;
0319
0320     /* channels */
0321     struct dw_dma_chan  *chan;
0322     u8          all_chan_mask;
0323     u8          in_use;
0324
0325     /* Channel operations */
0326     void    (*initialize_chan)(struct dw_dma_chan *dwc);
0327     void    (*suspend_chan)(struct dw_dma_chan *dwc, bool drain);
0328     void    (*resume_chan)(struct dw_dma_chan *dwc, bool drain);
0329     u32 (*prepare_ctllo)(struct dw_dma_chan *dwc);
0330     void    (*encode_maxburst)(struct dw_dma_chan *dwc, u32 *maxburst);
0331     u32 (*bytes2block)(struct dw_dma_chan *dwc, size_t bytes,
0332                    unsigned int width, size_t *len);
0333     size_t  (*block2bytes)(struct dw_dma_chan *dwc, u32 block, u32 width);
0334
0335     /* Device operations */
0336     void (*set_device_name)(struct dw_dma *dw, int id);
0337     void (*disable)(struct dw_dma *dw);
0338     void (*enable)(struct dw_dma *dw);
0339
0340     /* platform data */
0341     struct dw_dma_platform_data *pdata;
0342 };
0343
0344 static inline struct dw_dma_regs __iomem *__dw_regs(struct dw_dma *dw)
0345 {
0346     return dw->regs;
0347 }
0348
0349 #define dma_readl(dw, name) \
0350     readl(&(__dw_regs(dw)->name))
0351 #define dma_writel(dw, name, val) \
0352     writel((val), &(__dw_regs(dw)->name))
0353
0354 #define idma32_readq(dw, name)              \
0355     hi_lo_readq(&(__dw_regs(dw)->name))
0356 #define idma32_writeq(dw, name, val)            \
0357     hi_lo_writeq((val), &(__dw_regs(dw)->name))
0358
0359 #define channel_set_bit(dw, reg, mask) \
0360     dma_writel(dw, reg, ((mask) << 8) | (mask))
0361 #define channel_clear_bit(dw, reg, mask) \
0362     dma_writel(dw, reg, ((mask) << 8) | 0)
0363
0364 static inline struct dw_dma *to_dw_dma(struct dma_device *ddev)
0365 {
0366     return container_of(ddev, struct dw_dma, dma);
0367 }
0368
0369 /* LLI == Linked List Item; a.k.a. DMA block descriptor */
0370 struct dw_lli {
0371     /* values that are not changed by hardware */
0372     __le32      sar;
0373     __le32      dar;
0374     __le32      llp;        /* chain to next lli */
0375     __le32      ctllo;
0376     /* values that may get written back: */
0377     __le32      ctlhi;
0378     /* sstat and dstat can snapshot peripheral register state.
0379      * silicon config may discard either or both...
0380      */
0381     __le32      sstat;
0382     __le32      dstat;
0383 };
0384
0385 struct dw_desc {
0386     /* FIRST values the hardware uses */
0387     struct dw_lli           lli;
0388
0389 #define lli_set(d, reg, v)      ((d)->lli.reg |= cpu_to_le32(v))
0390 #define lli_clear(d, reg, v)        ((d)->lli.reg &= ~cpu_to_le32(v))
0391 #define lli_read(d, reg)        le32_to_cpu((d)->lli.reg)
0392 #define lli_write(d, reg, v)        ((d)->lli.reg = cpu_to_le32(v))
0393
0394     /* THEN values for driver housekeeping */
0395     struct list_head        desc_node;
0396     struct list_head        tx_list;
0397     struct dma_async_tx_descriptor  txd;
0398     size_t              len;
0399     size_t              total_len;
0400     u32             residue;
0401 };
0402
0403 #define to_dw_desc(h)   list_entry(h, struct dw_desc, desc_node)
0404
0405 static inline struct dw_desc *
0406 txd_to_dw_desc(struct dma_async_tx_descriptor *txd)
0407 {
0408     return container_of(txd, struct dw_desc, txd);
0409 }