drivers/dma/ste_dma40_ll.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Copyright (C) ST-Ericsson SA 2007-2010
0004  * Author: Per Forlin <per.forlin@stericsson.com> for ST-Ericsson
0005  * Author: Jonas Aaberg <jonas.aberg@stericsson.com> for ST-Ericsson
0006  */
0007
0008 #include <linux/kernel.h>
0009 #include <linux/platform_data/dma-ste-dma40.h>
0010
0011 #include "ste_dma40_ll.h"
0012
0013 static u8 d40_width_to_bits(enum dma_slave_buswidth width)
0014 {
0015     if (width == DMA_SLAVE_BUSWIDTH_1_BYTE)
0016         return STEDMA40_ESIZE_8_BIT;
0017     else if (width == DMA_SLAVE_BUSWIDTH_2_BYTES)
0018         return STEDMA40_ESIZE_16_BIT;
0019     else if (width == DMA_SLAVE_BUSWIDTH_8_BYTES)
0020         return STEDMA40_ESIZE_64_BIT;
0021     else
0022         return STEDMA40_ESIZE_32_BIT;
0023 }
0024
0025 /* Sets up proper LCSP1 and LCSP3 register for a logical channel */
0026 void d40_log_cfg(struct stedma40_chan_cfg *cfg,
0027          u32 *lcsp1, u32 *lcsp3)
0028 {
0029     u32 l3 = 0; /* dst */
0030     u32 l1 = 0; /* src */
0031
0032     /* src is mem? -> increase address pos */
0033     if (cfg->dir ==  DMA_MEM_TO_DEV ||
0034         cfg->dir ==  DMA_MEM_TO_MEM)
0035         l1 |= BIT(D40_MEM_LCSP1_SCFG_INCR_POS);
0036
0037     /* dst is mem? -> increase address pos */
0038     if (cfg->dir ==  DMA_DEV_TO_MEM ||
0039         cfg->dir ==  DMA_MEM_TO_MEM)
0040         l3 |= BIT(D40_MEM_LCSP3_DCFG_INCR_POS);
0041
0042     /* src is hw? -> master port 1 */
0043     if (cfg->dir ==  DMA_DEV_TO_MEM ||
0044         cfg->dir ==  DMA_DEV_TO_DEV)
0045         l1 |= BIT(D40_MEM_LCSP1_SCFG_MST_POS);
0046
0047     /* dst is hw? -> master port 1 */
0048     if (cfg->dir ==  DMA_MEM_TO_DEV ||
0049         cfg->dir ==  DMA_DEV_TO_DEV)
0050         l3 |= BIT(D40_MEM_LCSP3_DCFG_MST_POS);
0051
0052     l3 |= BIT(D40_MEM_LCSP3_DCFG_EIM_POS);
0053     l3 |= cfg->dst_info.psize << D40_MEM_LCSP3_DCFG_PSIZE_POS;
0054     l3 |= d40_width_to_bits(cfg->dst_info.data_width)
0055         << D40_MEM_LCSP3_DCFG_ESIZE_POS;
0056
0057     l1 |= BIT(D40_MEM_LCSP1_SCFG_EIM_POS);
0058     l1 |= cfg->src_info.psize << D40_MEM_LCSP1_SCFG_PSIZE_POS;
0059     l1 |= d40_width_to_bits(cfg->src_info.data_width)
0060         << D40_MEM_LCSP1_SCFG_ESIZE_POS;
0061
0062     *lcsp1 = l1;
0063     *lcsp3 = l3;
0064
0065 }
0066
0067 void d40_phy_cfg(struct stedma40_chan_cfg *cfg, u32 *src_cfg, u32 *dst_cfg)
0068 {
0069     u32 src = 0;
0070     u32 dst = 0;
0071
0072     if ((cfg->dir == DMA_DEV_TO_MEM) ||
0073         (cfg->dir == DMA_DEV_TO_DEV)) {
0074         /* Set master port to 1 */
0075         src |= BIT(D40_SREG_CFG_MST_POS);
0076         src |= D40_TYPE_TO_EVENT(cfg->dev_type);
0077
0078         if (cfg->src_info.flow_ctrl == STEDMA40_NO_FLOW_CTRL)
0079             src |= BIT(D40_SREG_CFG_PHY_TM_POS);
0080         else
0081             src |= 3 << D40_SREG_CFG_PHY_TM_POS;
0082     }
0083     if ((cfg->dir == DMA_MEM_TO_DEV) ||
0084         (cfg->dir == DMA_DEV_TO_DEV)) {
0085         /* Set master port to 1 */
0086         dst |= BIT(D40_SREG_CFG_MST_POS);
0087         dst |= D40_TYPE_TO_EVENT(cfg->dev_type);
0088
0089         if (cfg->dst_info.flow_ctrl == STEDMA40_NO_FLOW_CTRL)
0090             dst |= BIT(D40_SREG_CFG_PHY_TM_POS);
0091         else
0092             dst |= 3 << D40_SREG_CFG_PHY_TM_POS;
0093     }
0094     /* Interrupt on end of transfer for destination */
0095     dst |= BIT(D40_SREG_CFG_TIM_POS);
0096
0097     /* Generate interrupt on error */
0098     src |= BIT(D40_SREG_CFG_EIM_POS);
0099     dst |= BIT(D40_SREG_CFG_EIM_POS);
0100
0101     /* PSIZE */
0102     if (cfg->src_info.psize != STEDMA40_PSIZE_PHY_1) {
0103         src |= BIT(D40_SREG_CFG_PHY_PEN_POS);
0104         src |= cfg->src_info.psize << D40_SREG_CFG_PSIZE_POS;
0105     }
0106     if (cfg->dst_info.psize != STEDMA40_PSIZE_PHY_1) {
0107         dst |= BIT(D40_SREG_CFG_PHY_PEN_POS);
0108         dst |= cfg->dst_info.psize << D40_SREG_CFG_PSIZE_POS;
0109     }
0110
0111     /* Element size */
0112     src |= d40_width_to_bits(cfg->src_info.data_width)
0113         << D40_SREG_CFG_ESIZE_POS;
0114     dst |= d40_width_to_bits(cfg->dst_info.data_width)
0115         << D40_SREG_CFG_ESIZE_POS;
0116
0117     /* Set the priority bit to high for the physical channel */
0118     if (cfg->high_priority) {
0119         src |= BIT(D40_SREG_CFG_PRI_POS);
0120         dst |= BIT(D40_SREG_CFG_PRI_POS);
0121     }
0122
0123     if (cfg->src_info.big_endian)
0124         src |= BIT(D40_SREG_CFG_LBE_POS);
0125     if (cfg->dst_info.big_endian)
0126         dst |= BIT(D40_SREG_CFG_LBE_POS);
0127
0128     *src_cfg = src;
0129     *dst_cfg = dst;
0130 }
0131
0132 static int d40_phy_fill_lli(struct d40_phy_lli *lli,
0133                 dma_addr_t data,
0134                 u32 data_size,
0135                 dma_addr_t next_lli,
0136                 u32 reg_cfg,
0137                 struct stedma40_half_channel_info *info,
0138                 unsigned int flags)
0139 {
0140     bool addr_inc = flags & LLI_ADDR_INC;
0141     bool term_int = flags & LLI_TERM_INT;
0142     unsigned int data_width = info->data_width;
0143     int psize = info->psize;
0144     int num_elems;
0145
0146     if (psize == STEDMA40_PSIZE_PHY_1)
0147         num_elems = 1;
0148     else
0149         num_elems = 2 << psize;
0150
0151     /* Must be aligned */
0152     if (!IS_ALIGNED(data, data_width))
0153         return -EINVAL;
0154
0155     /* Transfer size can't be smaller than (num_elms * elem_size) */
0156     if (data_size < num_elems * data_width)
0157         return -EINVAL;
0158
0159     /* The number of elements. IE now many chunks */
0160     lli->reg_elt = (data_size / data_width) << D40_SREG_ELEM_PHY_ECNT_POS;
0161
0162     /*
0163      * Distance to next element sized entry.
0164      * Usually the size of the element unless you want gaps.
0165      */
0166     if (addr_inc)
0167         lli->reg_elt |= data_width << D40_SREG_ELEM_PHY_EIDX_POS;
0168
0169     /* Where the data is */
0170     lli->reg_ptr = data;
0171     lli->reg_cfg = reg_cfg;
0172
0173     /* If this scatter list entry is the last one, no next link */
0174     if (next_lli == 0)
0175         lli->reg_lnk = BIT(D40_SREG_LNK_PHY_TCP_POS);
0176     else
0177         lli->reg_lnk = next_lli;
0178
0179     /* Set/clear interrupt generation on this link item.*/
0180     if (term_int)
0181         lli->reg_cfg |= BIT(D40_SREG_CFG_TIM_POS);
0182     else
0183         lli->reg_cfg &= ~BIT(D40_SREG_CFG_TIM_POS);
0184
0185     /*
0186      * Post link - D40_SREG_LNK_PHY_PRE_POS = 0
0187      * Relink happens after transfer completion.
0188      */
0189
0190     return 0;
0191 }
0192
0193 static int d40_seg_size(int size, int data_width1, int data_width2)
0194 {
0195     u32 max_w = max(data_width1, data_width2);
0196     u32 min_w = min(data_width1, data_width2);
0197     u32 seg_max = ALIGN(STEDMA40_MAX_SEG_SIZE * min_w, max_w);
0198
0199     if (seg_max > STEDMA40_MAX_SEG_SIZE)
0200         seg_max -= max_w;
0201
0202     if (size <= seg_max)
0203         return size;
0204
0205     if (size <= 2 * seg_max)
0206         return ALIGN(size / 2, max_w);
0207
0208     return seg_max;
0209 }
0210
0211 static struct d40_phy_lli *
0212 d40_phy_buf_to_lli(struct d40_phy_lli *lli, dma_addr_t addr, u32 size,
0213            dma_addr_t lli_phys, dma_addr_t first_phys, u32 reg_cfg,
0214            struct stedma40_half_channel_info *info,
0215            struct stedma40_half_channel_info *otherinfo,
0216            unsigned long flags)
0217 {
0218     bool lastlink = flags & LLI_LAST_LINK;
0219     bool addr_inc = flags & LLI_ADDR_INC;
0220     bool term_int = flags & LLI_TERM_INT;
0221     bool cyclic = flags & LLI_CYCLIC;
0222     int err;
0223     dma_addr_t next = lli_phys;
0224     int size_rest = size;
0225     int size_seg = 0;
0226
0227     /*
0228      * This piece may be split up based on d40_seg_size(); we only want the
0229      * term int on the last part.
0230      */
0231     if (term_int)
0232         flags &= ~LLI_TERM_INT;
0233
0234     do {
0235         size_seg = d40_seg_size(size_rest, info->data_width,
0236                     otherinfo->data_width);
0237         size_rest -= size_seg;
0238
0239         if (size_rest == 0 && term_int)
0240             flags |= LLI_TERM_INT;
0241
0242         if (size_rest == 0 && lastlink)
0243             next = cyclic ? first_phys : 0;
0244         else
0245             next = ALIGN(next + sizeof(struct d40_phy_lli),
0246                      D40_LLI_ALIGN);
0247
0248         err = d40_phy_fill_lli(lli, addr, size_seg, next,
0249                        reg_cfg, info, flags);
0250
0251         if (err)
0252             goto err;
0253
0254         lli++;
0255         if (addr_inc)
0256             addr += size_seg;
0257     } while (size_rest);
0258
0259     return lli;
0260
0261 err:
0262     return NULL;
0263 }
0264
0265 int d40_phy_sg_to_lli(struct scatterlist *sg,
0266               int sg_len,
0267               dma_addr_t target,
0268               struct d40_phy_lli *lli_sg,
0269               dma_addr_t lli_phys,
0270               u32 reg_cfg,
0271               struct stedma40_half_channel_info *info,
0272               struct stedma40_half_channel_info *otherinfo,
0273               unsigned long flags)
0274 {
0275     int total_size = 0;
0276     int i;
0277     struct scatterlist *current_sg = sg;
0278     struct d40_phy_lli *lli = lli_sg;
0279     dma_addr_t l_phys = lli_phys;
0280
0281     if (!target)
0282         flags |= LLI_ADDR_INC;
0283
0284     for_each_sg(sg, current_sg, sg_len, i) {
0285         dma_addr_t sg_addr = sg_dma_address(current_sg);
0286         unsigned int len = sg_dma_len(current_sg);
0287         dma_addr_t dst = target ?: sg_addr;
0288
0289         total_size += sg_dma_len(current_sg);
0290
0291         if (i == sg_len - 1)
0292             flags |= LLI_TERM_INT | LLI_LAST_LINK;
0293
0294         l_phys = ALIGN(lli_phys + (lli - lli_sg) *
0295                    sizeof(struct d40_phy_lli), D40_LLI_ALIGN);
0296
0297         lli = d40_phy_buf_to_lli(lli, dst, len, l_phys, lli_phys,
0298                      reg_cfg, info, otherinfo, flags);
0299
0300         if (lli == NULL)
0301             return -EINVAL;
0302     }
0303
0304     return total_size;
0305 }
0306
0307
0308 /* DMA logical lli operations */
0309
0310 static void d40_log_lli_link(struct d40_log_lli *lli_dst,
0311                  struct d40_log_lli *lli_src,
0312                  int next, unsigned int flags)
0313 {
0314     bool interrupt = flags & LLI_TERM_INT;
0315     u32 slos = 0;
0316     u32 dlos = 0;
0317
0318     if (next != -EINVAL) {
0319         slos = next * 2;
0320         dlos = next * 2 + 1;
0321     }
0322
0323     if (interrupt) {
0324         lli_dst->lcsp13 |= D40_MEM_LCSP1_SCFG_TIM_MASK;
0325         lli_dst->lcsp13 |= D40_MEM_LCSP3_DTCP_MASK;
0326     }
0327
0328     lli_src->lcsp13 = (lli_src->lcsp13 & ~D40_MEM_LCSP1_SLOS_MASK) |
0329         (slos << D40_MEM_LCSP1_SLOS_POS);
0330
0331     lli_dst->lcsp13 = (lli_dst->lcsp13 & ~D40_MEM_LCSP1_SLOS_MASK) |
0332         (dlos << D40_MEM_LCSP1_SLOS_POS);
0333 }
0334
0335 void d40_log_lli_lcpa_write(struct d40_log_lli_full *lcpa,
0336                struct d40_log_lli *lli_dst,
0337                struct d40_log_lli *lli_src,
0338                int next, unsigned int flags)
0339 {
0340     d40_log_lli_link(lli_dst, lli_src, next, flags);
0341
0342     writel_relaxed(lli_src->lcsp02, &lcpa[0].lcsp0);
0343     writel_relaxed(lli_src->lcsp13, &lcpa[0].lcsp1);
0344     writel_relaxed(lli_dst->lcsp02, &lcpa[0].lcsp2);
0345     writel_relaxed(lli_dst->lcsp13, &lcpa[0].lcsp3);
0346 }
0347
0348 void d40_log_lli_lcla_write(struct d40_log_lli *lcla,
0349                struct d40_log_lli *lli_dst,
0350                struct d40_log_lli *lli_src,
0351                int next, unsigned int flags)
0352 {
0353     d40_log_lli_link(lli_dst, lli_src, next, flags);
0354
0355     writel_relaxed(lli_src->lcsp02, &lcla[0].lcsp02);
0356     writel_relaxed(lli_src->lcsp13, &lcla[0].lcsp13);
0357     writel_relaxed(lli_dst->lcsp02, &lcla[1].lcsp02);
0358     writel_relaxed(lli_dst->lcsp13, &lcla[1].lcsp13);
0359 }
0360
0361 static void d40_log_fill_lli(struct d40_log_lli *lli,
0362                  dma_addr_t data, u32 data_size,
0363                  u32 reg_cfg,
0364                  u32 data_width,
0365                  unsigned int flags)
0366 {
0367     bool addr_inc = flags & LLI_ADDR_INC;
0368
0369     lli->lcsp13 = reg_cfg;
0370
0371     /* The number of elements to transfer */
0372     lli->lcsp02 = ((data_size / data_width) <<
0373                D40_MEM_LCSP0_ECNT_POS) & D40_MEM_LCSP0_ECNT_MASK;
0374
0375     BUG_ON((data_size / data_width) > STEDMA40_MAX_SEG_SIZE);
0376
0377     /* 16 LSBs address of the current element */
0378     lli->lcsp02 |= data & D40_MEM_LCSP0_SPTR_MASK;
0379     /* 16 MSBs address of the current element */
0380     lli->lcsp13 |= data & D40_MEM_LCSP1_SPTR_MASK;
0381
0382     if (addr_inc)
0383         lli->lcsp13 |= D40_MEM_LCSP1_SCFG_INCR_MASK;
0384
0385 }
0386
0387 static struct d40_log_lli *d40_log_buf_to_lli(struct d40_log_lli *lli_sg,
0388                        dma_addr_t addr,
0389                        int size,
0390                        u32 lcsp13, /* src or dst*/
0391                        u32 data_width1,
0392                        u32 data_width2,
0393                        unsigned int flags)
0394 {
0395     bool addr_inc = flags & LLI_ADDR_INC;
0396     struct d40_log_lli *lli = lli_sg;
0397     int size_rest = size;
0398     int size_seg = 0;
0399
0400     do {
0401         size_seg = d40_seg_size(size_rest, data_width1, data_width2);
0402         size_rest -= size_seg;
0403
0404         d40_log_fill_lli(lli,
0405                  addr,
0406                  size_seg,
0407                  lcsp13, data_width1,
0408                  flags);
0409         if (addr_inc)
0410             addr += size_seg;
0411         lli++;
0412     } while (size_rest);
0413
0414     return lli;
0415 }
0416
0417 int d40_log_sg_to_lli(struct scatterlist *sg,
0418               int sg_len,
0419               dma_addr_t dev_addr,
0420               struct d40_log_lli *lli_sg,
0421               u32 lcsp13, /* src or dst*/
0422               u32 data_width1, u32 data_width2)
0423 {
0424     int total_size = 0;
0425     struct scatterlist *current_sg = sg;
0426     int i;
0427     struct d40_log_lli *lli = lli_sg;
0428     unsigned long flags = 0;
0429
0430     if (!dev_addr)
0431         flags |= LLI_ADDR_INC;
0432
0433     for_each_sg(sg, current_sg, sg_len, i) {
0434         dma_addr_t sg_addr = sg_dma_address(current_sg);
0435         unsigned int len = sg_dma_len(current_sg);
0436         dma_addr_t addr = dev_addr ?: sg_addr;
0437
0438         total_size += sg_dma_len(current_sg);
0439
0440         lli = d40_log_buf_to_lli(lli, addr, len,
0441                      lcsp13,
0442                      data_width1,
0443                      data_width2,
0444                      flags);
0445     }
0446
0447     return total_size;
0448 }