Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  * Copyright (C) 2015-2016 Marvell International Ltd.
0004 
0005  */
0006 
0007 #include <linux/clk.h>
0008 #include <linux/dma-mapping.h>
0009 #include <linux/interrupt.h>
0010 #include <linux/io.h>
0011 #include <linux/module.h>
0012 #include <linux/msi.h>
0013 #include <linux/of.h>
0014 #include <linux/of_irq.h>
0015 #include <linux/platform_device.h>
0016 #include <linux/spinlock.h>
0017 
0018 #include "dmaengine.h"
0019 
0020 /* DMA Engine Registers */
0021 #define MV_XOR_V2_DMA_DESQ_BALR_OFF         0x000
0022 #define MV_XOR_V2_DMA_DESQ_BAHR_OFF         0x004
0023 #define MV_XOR_V2_DMA_DESQ_SIZE_OFF         0x008
0024 #define MV_XOR_V2_DMA_DESQ_DONE_OFF         0x00C
0025 #define   MV_XOR_V2_DMA_DESQ_DONE_PENDING_MASK      0x7FFF
0026 #define   MV_XOR_V2_DMA_DESQ_DONE_PENDING_SHIFT     0
0027 #define   MV_XOR_V2_DMA_DESQ_DONE_READ_PTR_MASK     0x1FFF
0028 #define   MV_XOR_V2_DMA_DESQ_DONE_READ_PTR_SHIFT    16
0029 #define MV_XOR_V2_DMA_DESQ_ARATTR_OFF           0x010
0030 #define   MV_XOR_V2_DMA_DESQ_ATTR_CACHE_MASK        0x3F3F
0031 #define   MV_XOR_V2_DMA_DESQ_ATTR_OUTER_SHAREABLE   0x202
0032 #define   MV_XOR_V2_DMA_DESQ_ATTR_CACHEABLE     0x3C3C
0033 #define MV_XOR_V2_DMA_IMSG_CDAT_OFF         0x014
0034 #define MV_XOR_V2_DMA_IMSG_THRD_OFF         0x018
0035 #define   MV_XOR_V2_DMA_IMSG_THRD_MASK          0x7FFF
0036 #define   MV_XOR_V2_DMA_IMSG_TIMER_EN           BIT(18)
0037 #define MV_XOR_V2_DMA_DESQ_AWATTR_OFF           0x01C
0038   /* Same flags as MV_XOR_V2_DMA_DESQ_ARATTR_OFF */
0039 #define MV_XOR_V2_DMA_DESQ_ALLOC_OFF            0x04C
0040 #define   MV_XOR_V2_DMA_DESQ_ALLOC_WRPTR_MASK       0xFFFF
0041 #define   MV_XOR_V2_DMA_DESQ_ALLOC_WRPTR_SHIFT      16
0042 #define MV_XOR_V2_DMA_IMSG_BALR_OFF         0x050
0043 #define MV_XOR_V2_DMA_IMSG_BAHR_OFF         0x054
0044 #define MV_XOR_V2_DMA_DESQ_CTRL_OFF         0x100
0045 #define   MV_XOR_V2_DMA_DESQ_CTRL_32B           1
0046 #define   MV_XOR_V2_DMA_DESQ_CTRL_128B          7
0047 #define MV_XOR_V2_DMA_DESQ_STOP_OFF         0x800
0048 #define MV_XOR_V2_DMA_DESQ_DEALLOC_OFF          0x804
0049 #define MV_XOR_V2_DMA_DESQ_ADD_OFF          0x808
0050 #define MV_XOR_V2_DMA_IMSG_TMOT             0x810
0051 #define   MV_XOR_V2_DMA_IMSG_TIMER_THRD_MASK        0x1FFF
0052 
0053 /* XOR Global registers */
0054 #define MV_XOR_V2_GLOB_BW_CTRL              0x4
0055 #define   MV_XOR_V2_GLOB_BW_CTRL_NUM_OSTD_RD_SHIFT  0
0056 #define   MV_XOR_V2_GLOB_BW_CTRL_NUM_OSTD_RD_VAL    64
0057 #define   MV_XOR_V2_GLOB_BW_CTRL_NUM_OSTD_WR_SHIFT  8
0058 #define   MV_XOR_V2_GLOB_BW_CTRL_NUM_OSTD_WR_VAL    8
0059 #define   MV_XOR_V2_GLOB_BW_CTRL_RD_BURST_LEN_SHIFT 12
0060 #define   MV_XOR_V2_GLOB_BW_CTRL_RD_BURST_LEN_VAL   4
0061 #define   MV_XOR_V2_GLOB_BW_CTRL_WR_BURST_LEN_SHIFT 16
0062 #define   MV_XOR_V2_GLOB_BW_CTRL_WR_BURST_LEN_VAL   4
0063 #define MV_XOR_V2_GLOB_PAUSE                0x014
0064 #define   MV_XOR_V2_GLOB_PAUSE_AXI_TIME_DIS_VAL     0x8
0065 #define MV_XOR_V2_GLOB_SYS_INT_CAUSE            0x200
0066 #define MV_XOR_V2_GLOB_SYS_INT_MASK         0x204
0067 #define MV_XOR_V2_GLOB_MEM_INT_CAUSE            0x220
0068 #define MV_XOR_V2_GLOB_MEM_INT_MASK         0x224
0069 
0070 #define MV_XOR_V2_MIN_DESC_SIZE             32
0071 #define MV_XOR_V2_EXT_DESC_SIZE             128
0072 
0073 #define MV_XOR_V2_DESC_RESERVED_SIZE            12
0074 #define MV_XOR_V2_DESC_BUFF_D_ADDR_SIZE         12
0075 
0076 #define MV_XOR_V2_CMD_LINE_NUM_MAX_D_BUF        8
0077 
0078 /*
0079  * Descriptors queue size. With 32 bytes descriptors, up to 2^14
0080  * descriptors are allowed, with 128 bytes descriptors, up to 2^12
0081  * descriptors are allowed. This driver uses 128 bytes descriptors,
0082  * but experimentation has shown that a set of 1024 descriptors is
0083  * sufficient to reach a good level of performance.
0084  */
0085 #define MV_XOR_V2_DESC_NUM              1024
0086 
0087 /*
0088  * Threshold values for descriptors and timeout, determined by
0089  * experimentation as giving a good level of performance.
0090  */
0091 #define MV_XOR_V2_DONE_IMSG_THRD  0x14
0092 #define MV_XOR_V2_TIMER_THRD      0xB0
0093 
0094 /**
0095  * struct mv_xor_v2_descriptor - DMA HW descriptor
0096  * @desc_id: used by S/W and is not affected by H/W.
0097  * @flags: error and status flags
0098  * @crc32_result: CRC32 calculation result
0099  * @desc_ctrl: operation mode and control flags
0100  * @buff_size: amount of bytes to be processed
0101  * @fill_pattern_src_addr: Fill-Pattern or Source-Address and
0102  * AW-Attributes
0103  * @data_buff_addr: Source (and might be RAID6 destination)
0104  * addresses of data buffers in RAID5 and RAID6
0105  * @reserved: reserved
0106  */
0107 struct mv_xor_v2_descriptor {
0108     u16 desc_id;
0109     u16 flags;
0110     u32 crc32_result;
0111     u32 desc_ctrl;
0112 
0113     /* Definitions for desc_ctrl */
0114 #define DESC_NUM_ACTIVE_D_BUF_SHIFT 22
0115 #define DESC_OP_MODE_SHIFT      28
0116 #define DESC_OP_MODE_NOP        0   /* Idle operation */
0117 #define DESC_OP_MODE_MEMCPY     1   /* Pure-DMA operation */
0118 #define DESC_OP_MODE_MEMSET     2   /* Mem-Fill operation */
0119 #define DESC_OP_MODE_MEMINIT        3   /* Mem-Init operation */
0120 #define DESC_OP_MODE_MEM_COMPARE    4   /* Mem-Compare operation */
0121 #define DESC_OP_MODE_CRC32      5   /* CRC32 calculation */
0122 #define DESC_OP_MODE_XOR        6   /* RAID5 (XOR) operation */
0123 #define DESC_OP_MODE_RAID6      7   /* RAID6 P&Q-generation */
0124 #define DESC_OP_MODE_RAID6_REC      8   /* RAID6 Recovery */
0125 #define DESC_Q_BUFFER_ENABLE        BIT(16)
0126 #define DESC_P_BUFFER_ENABLE        BIT(17)
0127 #define DESC_IOD            BIT(27)
0128 
0129     u32 buff_size;
0130     u32 fill_pattern_src_addr[4];
0131     u32 data_buff_addr[MV_XOR_V2_DESC_BUFF_D_ADDR_SIZE];
0132     u32 reserved[MV_XOR_V2_DESC_RESERVED_SIZE];
0133 };
0134 
0135 /**
0136  * struct mv_xor_v2_device - implements a xor device
0137  * @lock: lock for the engine
0138  * @clk: reference to the 'core' clock
0139  * @reg_clk: reference to the 'reg' clock
0140  * @dma_base: memory mapped DMA register base
0141  * @glob_base: memory mapped global register base
0142  * @irq_tasklet: tasklet used for IRQ handling call-backs
0143  * @free_sw_desc: linked list of free SW descriptors
0144  * @dmadev: dma device
0145  * @dmachan: dma channel
0146  * @hw_desq: HW descriptors queue
0147  * @hw_desq_virt: virtual address of DESCQ
0148  * @sw_desq: SW descriptors queue
0149  * @desc_size: HW descriptor size
0150  * @npendings: number of pending descriptors (for which tx_submit has
0151  * @hw_queue_idx: HW queue index
0152  * @irq: The Linux interrupt number
0153  * been called, but not yet issue_pending)
0154  */
0155 struct mv_xor_v2_device {
0156     spinlock_t lock;
0157     void __iomem *dma_base;
0158     void __iomem *glob_base;
0159     struct clk *clk;
0160     struct clk *reg_clk;
0161     struct tasklet_struct irq_tasklet;
0162     struct list_head free_sw_desc;
0163     struct dma_device dmadev;
0164     struct dma_chan dmachan;
0165     dma_addr_t hw_desq;
0166     struct mv_xor_v2_descriptor *hw_desq_virt;
0167     struct mv_xor_v2_sw_desc *sw_desq;
0168     int desc_size;
0169     unsigned int npendings;
0170     unsigned int hw_queue_idx;
0171     unsigned int irq;
0172 };
0173 
0174 /**
0175  * struct mv_xor_v2_sw_desc - implements a xor SW descriptor
0176  * @idx: descriptor index
0177  * @async_tx: support for the async_tx api
0178  * @hw_desc: assosiated HW descriptor
0179  * @free_list: node of the free SW descriprots list
0180 */
0181 struct mv_xor_v2_sw_desc {
0182     int idx;
0183     struct dma_async_tx_descriptor async_tx;
0184     struct mv_xor_v2_descriptor hw_desc;
0185     struct list_head free_list;
0186 };
0187 
0188 /*
0189  * Fill the data buffers to a HW descriptor
0190  */
0191 static void mv_xor_v2_set_data_buffers(struct mv_xor_v2_device *xor_dev,
0192                     struct mv_xor_v2_descriptor *desc,
0193                     dma_addr_t src, int index)
0194 {
0195     int arr_index = ((index >> 1) * 3);
0196 
0197     /*
0198      * Fill the buffer's addresses to the descriptor.
0199      *
0200      * The format of the buffers address for 2 sequential buffers
0201      * X and X + 1:
0202      *
0203      *  First word:  Buffer-DX-Address-Low[31:0]
0204      *  Second word: Buffer-DX+1-Address-Low[31:0]
0205      *  Third word:  DX+1-Buffer-Address-High[47:32] [31:16]
0206      *       DX-Buffer-Address-High[47:32] [15:0]
0207      */
0208     if ((index & 0x1) == 0) {
0209         desc->data_buff_addr[arr_index] = lower_32_bits(src);
0210 
0211         desc->data_buff_addr[arr_index + 2] &= ~0xFFFF;
0212         desc->data_buff_addr[arr_index + 2] |=
0213             upper_32_bits(src) & 0xFFFF;
0214     } else {
0215         desc->data_buff_addr[arr_index + 1] =
0216             lower_32_bits(src);
0217 
0218         desc->data_buff_addr[arr_index + 2] &= ~0xFFFF0000;
0219         desc->data_buff_addr[arr_index + 2] |=
0220             (upper_32_bits(src) & 0xFFFF) << 16;
0221     }
0222 }
0223 
0224 /*
0225  * notify the engine of new descriptors, and update the available index.
0226  */
0227 static void mv_xor_v2_add_desc_to_desq(struct mv_xor_v2_device *xor_dev,
0228                        int num_of_desc)
0229 {
0230     /* write the number of new descriptors in the DESQ. */
0231     writel(num_of_desc, xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_ADD_OFF);
0232 }
0233 
0234 /*
0235  * free HW descriptors
0236  */
0237 static void mv_xor_v2_free_desc_from_desq(struct mv_xor_v2_device *xor_dev,
0238                       int num_of_desc)
0239 {
0240     /* write the number of new descriptors in the DESQ. */
0241     writel(num_of_desc, xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_DEALLOC_OFF);
0242 }
0243 
0244 /*
0245  * Set descriptor size
0246  * Return the HW descriptor size in bytes
0247  */
0248 static int mv_xor_v2_set_desc_size(struct mv_xor_v2_device *xor_dev)
0249 {
0250     writel(MV_XOR_V2_DMA_DESQ_CTRL_128B,
0251            xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_CTRL_OFF);
0252 
0253     return MV_XOR_V2_EXT_DESC_SIZE;
0254 }
0255 
0256 /*
0257  * Set the IMSG threshold
0258  */
0259 static inline
0260 void mv_xor_v2_enable_imsg_thrd(struct mv_xor_v2_device *xor_dev)
0261 {
0262     u32 reg;
0263 
0264     /* Configure threshold of number of descriptors, and enable timer */
0265     reg = readl(xor_dev->dma_base + MV_XOR_V2_DMA_IMSG_THRD_OFF);
0266     reg &= ~MV_XOR_V2_DMA_IMSG_THRD_MASK;
0267     reg |= MV_XOR_V2_DONE_IMSG_THRD;
0268     reg |= MV_XOR_V2_DMA_IMSG_TIMER_EN;
0269     writel(reg, xor_dev->dma_base + MV_XOR_V2_DMA_IMSG_THRD_OFF);
0270 
0271     /* Configure Timer Threshold */
0272     reg = readl(xor_dev->dma_base + MV_XOR_V2_DMA_IMSG_TMOT);
0273     reg &= ~MV_XOR_V2_DMA_IMSG_TIMER_THRD_MASK;
0274     reg |= MV_XOR_V2_TIMER_THRD;
0275     writel(reg, xor_dev->dma_base + MV_XOR_V2_DMA_IMSG_TMOT);
0276 }
0277 
0278 static irqreturn_t mv_xor_v2_interrupt_handler(int irq, void *data)
0279 {
0280     struct mv_xor_v2_device *xor_dev = data;
0281     unsigned int ndescs;
0282     u32 reg;
0283 
0284     reg = readl(xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_DONE_OFF);
0285 
0286     ndescs = ((reg >> MV_XOR_V2_DMA_DESQ_DONE_PENDING_SHIFT) &
0287           MV_XOR_V2_DMA_DESQ_DONE_PENDING_MASK);
0288 
0289     /* No descriptors to process */
0290     if (!ndescs)
0291         return IRQ_NONE;
0292 
0293     /* schedule a tasklet to handle descriptors callbacks */
0294     tasklet_schedule(&xor_dev->irq_tasklet);
0295 
0296     return IRQ_HANDLED;
0297 }
0298 
0299 /*
0300  * submit a descriptor to the DMA engine
0301  */
0302 static dma_cookie_t
0303 mv_xor_v2_tx_submit(struct dma_async_tx_descriptor *tx)
0304 {
0305     void *dest_hw_desc;
0306     dma_cookie_t cookie;
0307     struct mv_xor_v2_sw_desc *sw_desc =
0308         container_of(tx, struct mv_xor_v2_sw_desc, async_tx);
0309     struct mv_xor_v2_device *xor_dev =
0310         container_of(tx->chan, struct mv_xor_v2_device, dmachan);
0311 
0312     dev_dbg(xor_dev->dmadev.dev,
0313         "%s sw_desc %p: async_tx %p\n",
0314         __func__, sw_desc, &sw_desc->async_tx);
0315 
0316     /* assign cookie */
0317     spin_lock_bh(&xor_dev->lock);
0318     cookie = dma_cookie_assign(tx);
0319 
0320     /* copy the HW descriptor from the SW descriptor to the DESQ */
0321     dest_hw_desc = xor_dev->hw_desq_virt + xor_dev->hw_queue_idx;
0322 
0323     memcpy(dest_hw_desc, &sw_desc->hw_desc, xor_dev->desc_size);
0324 
0325     xor_dev->npendings++;
0326     xor_dev->hw_queue_idx++;
0327     if (xor_dev->hw_queue_idx >= MV_XOR_V2_DESC_NUM)
0328         xor_dev->hw_queue_idx = 0;
0329 
0330     spin_unlock_bh(&xor_dev->lock);
0331 
0332     return cookie;
0333 }
0334 
0335 /*
0336  * Prepare a SW descriptor
0337  */
0338 static struct mv_xor_v2_sw_desc *
0339 mv_xor_v2_prep_sw_desc(struct mv_xor_v2_device *xor_dev)
0340 {
0341     struct mv_xor_v2_sw_desc *sw_desc;
0342     bool found = false;
0343 
0344     /* Lock the channel */
0345     spin_lock_bh(&xor_dev->lock);
0346 
0347     if (list_empty(&xor_dev->free_sw_desc)) {
0348         spin_unlock_bh(&xor_dev->lock);
0349         /* schedule tasklet to free some descriptors */
0350         tasklet_schedule(&xor_dev->irq_tasklet);
0351         return NULL;
0352     }
0353 
0354     list_for_each_entry(sw_desc, &xor_dev->free_sw_desc, free_list) {
0355         if (async_tx_test_ack(&sw_desc->async_tx)) {
0356             found = true;
0357             break;
0358         }
0359     }
0360 
0361     if (!found) {
0362         spin_unlock_bh(&xor_dev->lock);
0363         return NULL;
0364     }
0365 
0366     list_del(&sw_desc->free_list);
0367 
0368     /* Release the channel */
0369     spin_unlock_bh(&xor_dev->lock);
0370 
0371     return sw_desc;
0372 }
0373 
0374 /*
0375  * Prepare a HW descriptor for a memcpy operation
0376  */
0377 static struct dma_async_tx_descriptor *
0378 mv_xor_v2_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest,
0379               dma_addr_t src, size_t len, unsigned long flags)
0380 {
0381     struct mv_xor_v2_sw_desc *sw_desc;
0382     struct mv_xor_v2_descriptor *hw_descriptor;
0383     struct mv_xor_v2_device *xor_dev;
0384 
0385     xor_dev = container_of(chan, struct mv_xor_v2_device, dmachan);
0386 
0387     dev_dbg(xor_dev->dmadev.dev,
0388         "%s len: %zu src %pad dest %pad flags: %ld\n",
0389         __func__, len, &src, &dest, flags);
0390 
0391     sw_desc = mv_xor_v2_prep_sw_desc(xor_dev);
0392     if (!sw_desc)
0393         return NULL;
0394 
0395     sw_desc->async_tx.flags = flags;
0396 
0397     /* set the HW descriptor */
0398     hw_descriptor = &sw_desc->hw_desc;
0399 
0400     /* save the SW descriptor ID to restore when operation is done */
0401     hw_descriptor->desc_id = sw_desc->idx;
0402 
0403     /* Set the MEMCPY control word */
0404     hw_descriptor->desc_ctrl =
0405         DESC_OP_MODE_MEMCPY << DESC_OP_MODE_SHIFT;
0406 
0407     if (flags & DMA_PREP_INTERRUPT)
0408         hw_descriptor->desc_ctrl |= DESC_IOD;
0409 
0410     /* Set source address */
0411     hw_descriptor->fill_pattern_src_addr[0] = lower_32_bits(src);
0412     hw_descriptor->fill_pattern_src_addr[1] =
0413         upper_32_bits(src) & 0xFFFF;
0414 
0415     /* Set Destination address */
0416     hw_descriptor->fill_pattern_src_addr[2] = lower_32_bits(dest);
0417     hw_descriptor->fill_pattern_src_addr[3] =
0418         upper_32_bits(dest) & 0xFFFF;
0419 
0420     /* Set buffers size */
0421     hw_descriptor->buff_size = len;
0422 
0423     /* return the async tx descriptor */
0424     return &sw_desc->async_tx;
0425 }
0426 
0427 /*
0428  * Prepare a HW descriptor for a XOR operation
0429  */
0430 static struct dma_async_tx_descriptor *
0431 mv_xor_v2_prep_dma_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src,
0432                unsigned int src_cnt, size_t len, unsigned long flags)
0433 {
0434     struct mv_xor_v2_sw_desc *sw_desc;
0435     struct mv_xor_v2_descriptor *hw_descriptor;
0436     struct mv_xor_v2_device *xor_dev =
0437         container_of(chan, struct mv_xor_v2_device, dmachan);
0438     int i;
0439 
0440     if (src_cnt > MV_XOR_V2_CMD_LINE_NUM_MAX_D_BUF || src_cnt < 1)
0441         return NULL;
0442 
0443     dev_dbg(xor_dev->dmadev.dev,
0444         "%s src_cnt: %d len: %zu dest %pad flags: %ld\n",
0445         __func__, src_cnt, len, &dest, flags);
0446 
0447     sw_desc = mv_xor_v2_prep_sw_desc(xor_dev);
0448     if (!sw_desc)
0449         return NULL;
0450 
0451     sw_desc->async_tx.flags = flags;
0452 
0453     /* set the HW descriptor */
0454     hw_descriptor = &sw_desc->hw_desc;
0455 
0456     /* save the SW descriptor ID to restore when operation is done */
0457     hw_descriptor->desc_id = sw_desc->idx;
0458 
0459     /* Set the XOR control word */
0460     hw_descriptor->desc_ctrl =
0461         DESC_OP_MODE_XOR << DESC_OP_MODE_SHIFT;
0462     hw_descriptor->desc_ctrl |= DESC_P_BUFFER_ENABLE;
0463 
0464     if (flags & DMA_PREP_INTERRUPT)
0465         hw_descriptor->desc_ctrl |= DESC_IOD;
0466 
0467     /* Set the data buffers */
0468     for (i = 0; i < src_cnt; i++)
0469         mv_xor_v2_set_data_buffers(xor_dev, hw_descriptor, src[i], i);
0470 
0471     hw_descriptor->desc_ctrl |=
0472         src_cnt << DESC_NUM_ACTIVE_D_BUF_SHIFT;
0473 
0474     /* Set Destination address */
0475     hw_descriptor->fill_pattern_src_addr[2] = lower_32_bits(dest);
0476     hw_descriptor->fill_pattern_src_addr[3] =
0477         upper_32_bits(dest) & 0xFFFF;
0478 
0479     /* Set buffers size */
0480     hw_descriptor->buff_size = len;
0481 
0482     /* return the async tx descriptor */
0483     return &sw_desc->async_tx;
0484 }
0485 
0486 /*
0487  * Prepare a HW descriptor for interrupt operation.
0488  */
0489 static struct dma_async_tx_descriptor *
0490 mv_xor_v2_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags)
0491 {
0492     struct mv_xor_v2_sw_desc *sw_desc;
0493     struct mv_xor_v2_descriptor *hw_descriptor;
0494     struct mv_xor_v2_device *xor_dev =
0495         container_of(chan, struct mv_xor_v2_device, dmachan);
0496 
0497     sw_desc = mv_xor_v2_prep_sw_desc(xor_dev);
0498     if (!sw_desc)
0499         return NULL;
0500 
0501     /* set the HW descriptor */
0502     hw_descriptor = &sw_desc->hw_desc;
0503 
0504     /* save the SW descriptor ID to restore when operation is done */
0505     hw_descriptor->desc_id = sw_desc->idx;
0506 
0507     /* Set the INTERRUPT control word */
0508     hw_descriptor->desc_ctrl =
0509         DESC_OP_MODE_NOP << DESC_OP_MODE_SHIFT;
0510     hw_descriptor->desc_ctrl |= DESC_IOD;
0511 
0512     /* return the async tx descriptor */
0513     return &sw_desc->async_tx;
0514 }
0515 
0516 /*
0517  * push pending transactions to hardware
0518  */
0519 static void mv_xor_v2_issue_pending(struct dma_chan *chan)
0520 {
0521     struct mv_xor_v2_device *xor_dev =
0522         container_of(chan, struct mv_xor_v2_device, dmachan);
0523 
0524     spin_lock_bh(&xor_dev->lock);
0525 
0526     /*
0527      * update the engine with the number of descriptors to
0528      * process
0529      */
0530     mv_xor_v2_add_desc_to_desq(xor_dev, xor_dev->npendings);
0531     xor_dev->npendings = 0;
0532 
0533     spin_unlock_bh(&xor_dev->lock);
0534 }
0535 
0536 static inline
0537 int mv_xor_v2_get_pending_params(struct mv_xor_v2_device *xor_dev,
0538                  int *pending_ptr)
0539 {
0540     u32 reg;
0541 
0542     reg = readl(xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_DONE_OFF);
0543 
0544     /* get the next pending descriptor index */
0545     *pending_ptr = ((reg >> MV_XOR_V2_DMA_DESQ_DONE_READ_PTR_SHIFT) &
0546             MV_XOR_V2_DMA_DESQ_DONE_READ_PTR_MASK);
0547 
0548     /* get the number of descriptors pending handle */
0549     return ((reg >> MV_XOR_V2_DMA_DESQ_DONE_PENDING_SHIFT) &
0550         MV_XOR_V2_DMA_DESQ_DONE_PENDING_MASK);
0551 }
0552 
0553 /*
0554  * handle the descriptors after HW process
0555  */
0556 static void mv_xor_v2_tasklet(struct tasklet_struct *t)
0557 {
0558     struct mv_xor_v2_device *xor_dev = from_tasklet(xor_dev, t,
0559                             irq_tasklet);
0560     int pending_ptr, num_of_pending, i;
0561     struct mv_xor_v2_sw_desc *next_pending_sw_desc = NULL;
0562 
0563     dev_dbg(xor_dev->dmadev.dev, "%s %d\n", __func__, __LINE__);
0564 
0565     /* get the pending descriptors parameters */
0566     num_of_pending = mv_xor_v2_get_pending_params(xor_dev, &pending_ptr);
0567 
0568     /* loop over free descriptors */
0569     for (i = 0; i < num_of_pending; i++) {
0570         struct mv_xor_v2_descriptor *next_pending_hw_desc =
0571             xor_dev->hw_desq_virt + pending_ptr;
0572 
0573         /* get the SW descriptor related to the HW descriptor */
0574         next_pending_sw_desc =
0575             &xor_dev->sw_desq[next_pending_hw_desc->desc_id];
0576 
0577         /* call the callback */
0578         if (next_pending_sw_desc->async_tx.cookie > 0) {
0579             /*
0580              * update the channel's completed cookie - no
0581              * lock is required the IMSG threshold provide
0582              * the locking
0583              */
0584             dma_cookie_complete(&next_pending_sw_desc->async_tx);
0585 
0586             dma_descriptor_unmap(&next_pending_sw_desc->async_tx);
0587             dmaengine_desc_get_callback_invoke(
0588                     &next_pending_sw_desc->async_tx, NULL);
0589         }
0590 
0591         dma_run_dependencies(&next_pending_sw_desc->async_tx);
0592 
0593         /* Lock the channel */
0594         spin_lock(&xor_dev->lock);
0595 
0596         /* add the SW descriptor to the free descriptors list */
0597         list_add(&next_pending_sw_desc->free_list,
0598              &xor_dev->free_sw_desc);
0599 
0600         /* Release the channel */
0601         spin_unlock(&xor_dev->lock);
0602 
0603         /* increment the next descriptor */
0604         pending_ptr++;
0605         if (pending_ptr >= MV_XOR_V2_DESC_NUM)
0606             pending_ptr = 0;
0607     }
0608 
0609     if (num_of_pending != 0) {
0610         /* free the descriptores */
0611         mv_xor_v2_free_desc_from_desq(xor_dev, num_of_pending);
0612     }
0613 }
0614 
0615 /*
0616  *  Set DMA Interrupt-message (IMSG) parameters
0617  */
0618 static void mv_xor_v2_set_msi_msg(struct msi_desc *desc, struct msi_msg *msg)
0619 {
0620     struct mv_xor_v2_device *xor_dev = dev_get_drvdata(desc->dev);
0621 
0622     writel(msg->address_lo,
0623            xor_dev->dma_base + MV_XOR_V2_DMA_IMSG_BALR_OFF);
0624     writel(msg->address_hi & 0xFFFF,
0625            xor_dev->dma_base + MV_XOR_V2_DMA_IMSG_BAHR_OFF);
0626     writel(msg->data,
0627            xor_dev->dma_base + MV_XOR_V2_DMA_IMSG_CDAT_OFF);
0628 }
0629 
0630 static int mv_xor_v2_descq_init(struct mv_xor_v2_device *xor_dev)
0631 {
0632     u32 reg;
0633 
0634     /* write the DESQ size to the DMA engine */
0635     writel(MV_XOR_V2_DESC_NUM,
0636            xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_SIZE_OFF);
0637 
0638     /* write the DESQ address to the DMA enngine*/
0639     writel(lower_32_bits(xor_dev->hw_desq),
0640            xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_BALR_OFF);
0641     writel(upper_32_bits(xor_dev->hw_desq),
0642            xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_BAHR_OFF);
0643 
0644     /*
0645      * This is a temporary solution, until we activate the
0646      * SMMU. Set the attributes for reading & writing data buffers
0647      * & descriptors to:
0648      *
0649      *  - OuterShareable - Snoops will be performed on CPU caches
0650      *  - Enable cacheable - Bufferable, Modifiable, Other Allocate
0651      *    and Allocate
0652      */
0653     reg = readl(xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_ARATTR_OFF);
0654     reg &= ~MV_XOR_V2_DMA_DESQ_ATTR_CACHE_MASK;
0655     reg |= MV_XOR_V2_DMA_DESQ_ATTR_OUTER_SHAREABLE |
0656         MV_XOR_V2_DMA_DESQ_ATTR_CACHEABLE;
0657     writel(reg, xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_ARATTR_OFF);
0658 
0659     reg = readl(xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_AWATTR_OFF);
0660     reg &= ~MV_XOR_V2_DMA_DESQ_ATTR_CACHE_MASK;
0661     reg |= MV_XOR_V2_DMA_DESQ_ATTR_OUTER_SHAREABLE |
0662         MV_XOR_V2_DMA_DESQ_ATTR_CACHEABLE;
0663     writel(reg, xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_AWATTR_OFF);
0664 
0665     /* BW CTRL - set values to optimize the XOR performance:
0666      *
0667      *  - Set WrBurstLen & RdBurstLen - the unit will issue
0668      *    maximum of 256B write/read transactions.
0669      * -  Limit the number of outstanding write & read data
0670      *    (OBB/IBB) requests to the maximal value.
0671     */
0672     reg = ((MV_XOR_V2_GLOB_BW_CTRL_NUM_OSTD_RD_VAL <<
0673         MV_XOR_V2_GLOB_BW_CTRL_NUM_OSTD_RD_SHIFT) |
0674            (MV_XOR_V2_GLOB_BW_CTRL_NUM_OSTD_WR_VAL  <<
0675         MV_XOR_V2_GLOB_BW_CTRL_NUM_OSTD_WR_SHIFT) |
0676            (MV_XOR_V2_GLOB_BW_CTRL_RD_BURST_LEN_VAL <<
0677         MV_XOR_V2_GLOB_BW_CTRL_RD_BURST_LEN_SHIFT) |
0678            (MV_XOR_V2_GLOB_BW_CTRL_WR_BURST_LEN_VAL <<
0679         MV_XOR_V2_GLOB_BW_CTRL_WR_BURST_LEN_SHIFT));
0680     writel(reg, xor_dev->glob_base + MV_XOR_V2_GLOB_BW_CTRL);
0681 
0682     /* Disable the AXI timer feature */
0683     reg = readl(xor_dev->glob_base + MV_XOR_V2_GLOB_PAUSE);
0684     reg |= MV_XOR_V2_GLOB_PAUSE_AXI_TIME_DIS_VAL;
0685     writel(reg, xor_dev->glob_base + MV_XOR_V2_GLOB_PAUSE);
0686 
0687     /* enable the DMA engine */
0688     writel(0, xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_STOP_OFF);
0689 
0690     return 0;
0691 }
0692 
0693 static int mv_xor_v2_suspend(struct platform_device *dev, pm_message_t state)
0694 {
0695     struct mv_xor_v2_device *xor_dev = platform_get_drvdata(dev);
0696 
0697     /* Set this bit to disable to stop the XOR unit. */
0698     writel(0x1, xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_STOP_OFF);
0699 
0700     return 0;
0701 }
0702 
0703 static int mv_xor_v2_resume(struct platform_device *dev)
0704 {
0705     struct mv_xor_v2_device *xor_dev = platform_get_drvdata(dev);
0706 
0707     mv_xor_v2_set_desc_size(xor_dev);
0708     mv_xor_v2_enable_imsg_thrd(xor_dev);
0709     mv_xor_v2_descq_init(xor_dev);
0710 
0711     return 0;
0712 }
0713 
0714 static int mv_xor_v2_probe(struct platform_device *pdev)
0715 {
0716     struct mv_xor_v2_device *xor_dev;
0717     struct resource *res;
0718     int i, ret = 0;
0719     struct dma_device *dma_dev;
0720     struct mv_xor_v2_sw_desc *sw_desc;
0721 
0722     BUILD_BUG_ON(sizeof(struct mv_xor_v2_descriptor) !=
0723              MV_XOR_V2_EXT_DESC_SIZE);
0724 
0725     xor_dev = devm_kzalloc(&pdev->dev, sizeof(*xor_dev), GFP_KERNEL);
0726     if (!xor_dev)
0727         return -ENOMEM;
0728 
0729     res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
0730     xor_dev->dma_base = devm_ioremap_resource(&pdev->dev, res);
0731     if (IS_ERR(xor_dev->dma_base))
0732         return PTR_ERR(xor_dev->dma_base);
0733 
0734     res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
0735     xor_dev->glob_base = devm_ioremap_resource(&pdev->dev, res);
0736     if (IS_ERR(xor_dev->glob_base))
0737         return PTR_ERR(xor_dev->glob_base);
0738 
0739     platform_set_drvdata(pdev, xor_dev);
0740 
0741     ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40));
0742     if (ret)
0743         return ret;
0744 
0745     xor_dev->reg_clk = devm_clk_get(&pdev->dev, "reg");
0746     if (PTR_ERR(xor_dev->reg_clk) != -ENOENT) {
0747         if (!IS_ERR(xor_dev->reg_clk)) {
0748             ret = clk_prepare_enable(xor_dev->reg_clk);
0749             if (ret)
0750                 return ret;
0751         } else {
0752             return PTR_ERR(xor_dev->reg_clk);
0753         }
0754     }
0755 
0756     xor_dev->clk = devm_clk_get(&pdev->dev, NULL);
0757     if (PTR_ERR(xor_dev->clk) == -EPROBE_DEFER) {
0758         ret = EPROBE_DEFER;
0759         goto disable_reg_clk;
0760     }
0761     if (!IS_ERR(xor_dev->clk)) {
0762         ret = clk_prepare_enable(xor_dev->clk);
0763         if (ret)
0764             goto disable_reg_clk;
0765     }
0766 
0767     ret = platform_msi_domain_alloc_irqs(&pdev->dev, 1,
0768                          mv_xor_v2_set_msi_msg);
0769     if (ret)
0770         goto disable_clk;
0771 
0772     xor_dev->irq = msi_get_virq(&pdev->dev, 0);
0773 
0774     ret = devm_request_irq(&pdev->dev, xor_dev->irq,
0775                    mv_xor_v2_interrupt_handler, 0,
0776                    dev_name(&pdev->dev), xor_dev);
0777     if (ret)
0778         goto free_msi_irqs;
0779 
0780     tasklet_setup(&xor_dev->irq_tasklet, mv_xor_v2_tasklet);
0781 
0782     xor_dev->desc_size = mv_xor_v2_set_desc_size(xor_dev);
0783 
0784     dma_cookie_init(&xor_dev->dmachan);
0785 
0786     /*
0787      * allocate coherent memory for hardware descriptors
0788      * note: writecombine gives slightly better performance, but
0789      * requires that we explicitly flush the writes
0790      */
0791     xor_dev->hw_desq_virt =
0792         dma_alloc_coherent(&pdev->dev,
0793                    xor_dev->desc_size * MV_XOR_V2_DESC_NUM,
0794                    &xor_dev->hw_desq, GFP_KERNEL);
0795     if (!xor_dev->hw_desq_virt) {
0796         ret = -ENOMEM;
0797         goto free_msi_irqs;
0798     }
0799 
0800     /* alloc memory for the SW descriptors */
0801     xor_dev->sw_desq = devm_kcalloc(&pdev->dev,
0802                     MV_XOR_V2_DESC_NUM, sizeof(*sw_desc),
0803                     GFP_KERNEL);
0804     if (!xor_dev->sw_desq) {
0805         ret = -ENOMEM;
0806         goto free_hw_desq;
0807     }
0808 
0809     spin_lock_init(&xor_dev->lock);
0810 
0811     /* init the free SW descriptors list */
0812     INIT_LIST_HEAD(&xor_dev->free_sw_desc);
0813 
0814     /* add all SW descriptors to the free list */
0815     for (i = 0; i < MV_XOR_V2_DESC_NUM; i++) {
0816         struct mv_xor_v2_sw_desc *sw_desc =
0817             xor_dev->sw_desq + i;
0818         sw_desc->idx = i;
0819         dma_async_tx_descriptor_init(&sw_desc->async_tx,
0820                          &xor_dev->dmachan);
0821         sw_desc->async_tx.tx_submit = mv_xor_v2_tx_submit;
0822         async_tx_ack(&sw_desc->async_tx);
0823 
0824         list_add(&sw_desc->free_list,
0825              &xor_dev->free_sw_desc);
0826     }
0827 
0828     dma_dev = &xor_dev->dmadev;
0829 
0830     /* set DMA capabilities */
0831     dma_cap_zero(dma_dev->cap_mask);
0832     dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
0833     dma_cap_set(DMA_XOR, dma_dev->cap_mask);
0834     dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask);
0835 
0836     /* init dma link list */
0837     INIT_LIST_HEAD(&dma_dev->channels);
0838 
0839     /* set base routines */
0840     dma_dev->device_tx_status = dma_cookie_status;
0841     dma_dev->device_issue_pending = mv_xor_v2_issue_pending;
0842     dma_dev->dev = &pdev->dev;
0843 
0844     dma_dev->device_prep_dma_memcpy = mv_xor_v2_prep_dma_memcpy;
0845     dma_dev->device_prep_dma_interrupt = mv_xor_v2_prep_dma_interrupt;
0846     dma_dev->max_xor = 8;
0847     dma_dev->device_prep_dma_xor = mv_xor_v2_prep_dma_xor;
0848 
0849     xor_dev->dmachan.device = dma_dev;
0850 
0851     list_add_tail(&xor_dev->dmachan.device_node,
0852               &dma_dev->channels);
0853 
0854     mv_xor_v2_enable_imsg_thrd(xor_dev);
0855 
0856     mv_xor_v2_descq_init(xor_dev);
0857 
0858     ret = dma_async_device_register(dma_dev);
0859     if (ret)
0860         goto free_hw_desq;
0861 
0862     dev_notice(&pdev->dev, "Marvell Version 2 XOR driver\n");
0863 
0864     return 0;
0865 
0866 free_hw_desq:
0867     dma_free_coherent(&pdev->dev,
0868               xor_dev->desc_size * MV_XOR_V2_DESC_NUM,
0869               xor_dev->hw_desq_virt, xor_dev->hw_desq);
0870 free_msi_irqs:
0871     platform_msi_domain_free_irqs(&pdev->dev);
0872 disable_clk:
0873     clk_disable_unprepare(xor_dev->clk);
0874 disable_reg_clk:
0875     clk_disable_unprepare(xor_dev->reg_clk);
0876     return ret;
0877 }
0878 
0879 static int mv_xor_v2_remove(struct platform_device *pdev)
0880 {
0881     struct mv_xor_v2_device *xor_dev = platform_get_drvdata(pdev);
0882 
0883     dma_async_device_unregister(&xor_dev->dmadev);
0884 
0885     dma_free_coherent(&pdev->dev,
0886               xor_dev->desc_size * MV_XOR_V2_DESC_NUM,
0887               xor_dev->hw_desq_virt, xor_dev->hw_desq);
0888 
0889     devm_free_irq(&pdev->dev, xor_dev->irq, xor_dev);
0890 
0891     platform_msi_domain_free_irqs(&pdev->dev);
0892 
0893     tasklet_kill(&xor_dev->irq_tasklet);
0894 
0895     clk_disable_unprepare(xor_dev->clk);
0896 
0897     return 0;
0898 }
0899 
0900 #ifdef CONFIG_OF
0901 static const struct of_device_id mv_xor_v2_dt_ids[] = {
0902     { .compatible = "marvell,xor-v2", },
0903     {},
0904 };
0905 MODULE_DEVICE_TABLE(of, mv_xor_v2_dt_ids);
0906 #endif
0907 
0908 static struct platform_driver mv_xor_v2_driver = {
0909     .probe      = mv_xor_v2_probe,
0910     .suspend    = mv_xor_v2_suspend,
0911     .resume     = mv_xor_v2_resume,
0912     .remove     = mv_xor_v2_remove,
0913     .driver     = {
0914         .name   = "mv_xor_v2",
0915         .of_match_table = of_match_ptr(mv_xor_v2_dt_ids),
0916     },
0917 };
0918 
0919 module_platform_driver(mv_xor_v2_driver);
0920 
0921 MODULE_DESCRIPTION("DMA engine driver for Marvell's Version 2 of XOR engine");
0922 MODULE_LICENSE("GPL");
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!