dma/hsu/hsu.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Core driver for the High Speed UART DMA
0004  *
0005  * Copyright (C) 2015 Intel Corporation
0006  * Author: Andy Shevchenko <andriy.shevchenko@linux.intel.com>
0007  *
0008  * Partially based on the bits found in drivers/tty/serial/mfd.c.
0009  */
0010
0011 /*
0012  * DMA channel allocation:
0013  * 1. Even number chans are used for DMA Read (UART TX), odd chans for DMA
0014  *    Write (UART RX).
0015  * 2. 0/1 channel are assigned to port 0, 2/3 chan to port 1, 4/5 chan to
0016  *    port 3, and so on.
0017  */
0018
0019 #include <linux/delay.h>
0020 #include <linux/dmaengine.h>
0021 #include <linux/dma-mapping.h>
0022 #include <linux/init.h>
0023 #include <linux/module.h>
0024 #include <linux/slab.h>
0025
0026 #include "hsu.h"
0027
0028 #define HSU_DMA_BUSWIDTHS               \
0029     BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED)   |   \
0030     BIT(DMA_SLAVE_BUSWIDTH_1_BYTE)      |   \
0031     BIT(DMA_SLAVE_BUSWIDTH_2_BYTES)     |   \
0032     BIT(DMA_SLAVE_BUSWIDTH_3_BYTES)     |   \
0033     BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)     |   \
0034     BIT(DMA_SLAVE_BUSWIDTH_8_BYTES)     |   \
0035     BIT(DMA_SLAVE_BUSWIDTH_16_BYTES)
0036
0037 static inline void hsu_chan_disable(struct hsu_dma_chan *hsuc)
0038 {
0039     hsu_chan_writel(hsuc, HSU_CH_CR, 0);
0040 }
0041
0042 static inline void hsu_chan_enable(struct hsu_dma_chan *hsuc)
0043 {
0044     u32 cr = HSU_CH_CR_CHA;
0045
0046     if (hsuc->direction == DMA_MEM_TO_DEV)
0047         cr &= ~HSU_CH_CR_CHD;
0048     else if (hsuc->direction == DMA_DEV_TO_MEM)
0049         cr |= HSU_CH_CR_CHD;
0050
0051     hsu_chan_writel(hsuc, HSU_CH_CR, cr);
0052 }
0053
0054 static void hsu_dma_chan_start(struct hsu_dma_chan *hsuc)
0055 {
0056     struct dma_slave_config *config = &hsuc->config;
0057     struct hsu_dma_desc *desc = hsuc->desc;
0058     u32 bsr = 0, mtsr = 0;  /* to shut the compiler up */
0059     u32 dcr = HSU_CH_DCR_CHSOE | HSU_CH_DCR_CHEI;
0060     unsigned int i, count;
0061
0062     if (hsuc->direction == DMA_MEM_TO_DEV) {
0063         bsr = config->dst_maxburst;
0064         mtsr = config->dst_addr_width;
0065     } else if (hsuc->direction == DMA_DEV_TO_MEM) {
0066         bsr = config->src_maxburst;
0067         mtsr = config->src_addr_width;
0068     }
0069
0070     hsu_chan_disable(hsuc);
0071
0072     hsu_chan_writel(hsuc, HSU_CH_DCR, 0);
0073     hsu_chan_writel(hsuc, HSU_CH_BSR, bsr);
0074     hsu_chan_writel(hsuc, HSU_CH_MTSR, mtsr);
0075
0076     /* Set descriptors */
0077     count = desc->nents - desc->active;
0078     for (i = 0; i < count && i < HSU_DMA_CHAN_NR_DESC; i++) {
0079         hsu_chan_writel(hsuc, HSU_CH_DxSAR(i), desc->sg[i].addr);
0080         hsu_chan_writel(hsuc, HSU_CH_DxTSR(i), desc->sg[i].len);
0081
0082         /* Prepare value for DCR */
0083         dcr |= HSU_CH_DCR_DESCA(i);
0084         dcr |= HSU_CH_DCR_CHTOI(i); /* timeout bit, see HSU Errata 1 */
0085
0086         desc->active++;
0087     }
0088     /* Only for the last descriptor in the chain */
0089     dcr |= HSU_CH_DCR_CHSOD(count - 1);
0090     dcr |= HSU_CH_DCR_CHDI(count - 1);
0091
0092     hsu_chan_writel(hsuc, HSU_CH_DCR, dcr);
0093
0094     hsu_chan_enable(hsuc);
0095 }
0096
0097 static void hsu_dma_stop_channel(struct hsu_dma_chan *hsuc)
0098 {
0099     hsu_chan_disable(hsuc);
0100     hsu_chan_writel(hsuc, HSU_CH_DCR, 0);
0101 }
0102
0103 static void hsu_dma_start_channel(struct hsu_dma_chan *hsuc)
0104 {
0105     hsu_dma_chan_start(hsuc);
0106 }
0107
0108 static void hsu_dma_start_transfer(struct hsu_dma_chan *hsuc)
0109 {
0110     struct virt_dma_desc *vdesc;
0111
0112     /* Get the next descriptor */
0113     vdesc = vchan_next_desc(&hsuc->vchan);
0114     if (!vdesc) {
0115         hsuc->desc = NULL;
0116         return;
0117     }
0118
0119     list_del(&vdesc->node);
0120     hsuc->desc = to_hsu_dma_desc(vdesc);
0121
0122     /* Start the channel with a new descriptor */
0123     hsu_dma_start_channel(hsuc);
0124 }
0125
0126 /*
0127  *      hsu_dma_get_status() - get DMA channel status
0128  *      @chip: HSUART DMA chip
0129  *      @nr: DMA channel number
0130  *      @status: pointer for DMA Channel Status Register value
0131  *
0132  *      Description:
0133  *      The function reads and clears the DMA Channel Status Register, checks
0134  *      if it was a timeout interrupt and returns a corresponding value.
0135  *
0136  *      Caller should provide a valid pointer for the DMA Channel Status
0137  *      Register value that will be returned in @status.
0138  *
0139  *      Return:
0140  *      1 for DMA timeout status, 0 for other DMA status, or error code for
0141  *      invalid parameters or no interrupt pending.
0142  */
0143 int hsu_dma_get_status(struct hsu_dma_chip *chip, unsigned short nr,
0144                u32 *status)
0145 {
0146     struct hsu_dma_chan *hsuc;
0147     unsigned long flags;
0148     u32 sr;
0149
0150     /* Sanity check */
0151     if (nr >= chip->hsu->nr_channels)
0152         return -EINVAL;
0153
0154     hsuc = &chip->hsu->chan[nr];
0155
0156     /*
0157      * No matter what situation, need read clear the IRQ status
0158      * There is a bug, see Errata 5, HSD 2900918
0159      */
0160     spin_lock_irqsave(&hsuc->vchan.lock, flags);
0161     sr = hsu_chan_readl(hsuc, HSU_CH_SR);
0162     spin_unlock_irqrestore(&hsuc->vchan.lock, flags);
0163
0164     /* Check if any interrupt is pending */
0165     sr &= ~(HSU_CH_SR_DESCE_ANY | HSU_CH_SR_CDESC_ANY);
0166     if (!sr)
0167         return -EIO;
0168
0169     /* Timeout IRQ, need wait some time, see Errata 2 */
0170     if (sr & HSU_CH_SR_DESCTO_ANY)
0171         udelay(2);
0172
0173     /*
0174      * At this point, at least one of Descriptor Time Out, Channel Error
0175      * or Descriptor Done bits must be set. Clear the Descriptor Time Out
0176      * bits and if sr is still non-zero, it must be channel error or
0177      * descriptor done which are higher priority than timeout and handled
0178      * in hsu_dma_do_irq(). Else, it must be a timeout.
0179      */
0180     sr &= ~HSU_CH_SR_DESCTO_ANY;
0181
0182     *status = sr;
0183
0184     return sr ? 0 : 1;
0185 }
0186 EXPORT_SYMBOL_GPL(hsu_dma_get_status);
0187
0188 /*
0189  *      hsu_dma_do_irq() - DMA interrupt handler
0190  *      @chip: HSUART DMA chip
0191  *      @nr: DMA channel number
0192  *      @status: Channel Status Register value
0193  *
0194  *      Description:
0195  *      This function handles Channel Error and Descriptor Done interrupts.
0196  *      This function should be called after determining that the DMA interrupt
0197  *      is not a normal timeout interrupt, ie. hsu_dma_get_status() returned 0.
0198  *
0199  *      Return:
0200  *      0 for invalid channel number, 1 otherwise.
0201  */
0202 int hsu_dma_do_irq(struct hsu_dma_chip *chip, unsigned short nr, u32 status)
0203 {
0204     struct dma_chan_percpu *stat;
0205     struct hsu_dma_chan *hsuc;
0206     struct hsu_dma_desc *desc;
0207     unsigned long flags;
0208
0209     /* Sanity check */
0210     if (nr >= chip->hsu->nr_channels)
0211         return 0;
0212
0213     hsuc = &chip->hsu->chan[nr];
0214     stat = this_cpu_ptr(hsuc->vchan.chan.local);
0215
0216     spin_lock_irqsave(&hsuc->vchan.lock, flags);
0217     desc = hsuc->desc;
0218     if (desc) {
0219         if (status & HSU_CH_SR_CHE) {
0220             desc->status = DMA_ERROR;
0221         } else if (desc->active < desc->nents) {
0222             hsu_dma_start_channel(hsuc);
0223         } else {
0224             vchan_cookie_complete(&desc->vdesc);
0225             desc->status = DMA_COMPLETE;
0226             stat->bytes_transferred += desc->length;
0227             hsu_dma_start_transfer(hsuc);
0228         }
0229     }
0230     spin_unlock_irqrestore(&hsuc->vchan.lock, flags);
0231
0232     return 1;
0233 }
0234 EXPORT_SYMBOL_GPL(hsu_dma_do_irq);
0235
0236 static struct hsu_dma_desc *hsu_dma_alloc_desc(unsigned int nents)
0237 {
0238     struct hsu_dma_desc *desc;
0239
0240     desc = kzalloc(sizeof(*desc), GFP_NOWAIT);
0241     if (!desc)
0242         return NULL;
0243
0244     desc->sg = kcalloc(nents, sizeof(*desc->sg), GFP_NOWAIT);
0245     if (!desc->sg) {
0246         kfree(desc);
0247         return NULL;
0248     }
0249
0250     return desc;
0251 }
0252
0253 static void hsu_dma_desc_free(struct virt_dma_desc *vdesc)
0254 {
0255     struct hsu_dma_desc *desc = to_hsu_dma_desc(vdesc);
0256
0257     kfree(desc->sg);
0258     kfree(desc);
0259 }
0260
0261 static struct dma_async_tx_descriptor *hsu_dma_prep_slave_sg(
0262         struct dma_chan *chan, struct scatterlist *sgl,
0263         unsigned int sg_len, enum dma_transfer_direction direction,
0264         unsigned long flags, void *context)
0265 {
0266     struct hsu_dma_chan *hsuc = to_hsu_dma_chan(chan);
0267     struct hsu_dma_desc *desc;
0268     struct scatterlist *sg;
0269     unsigned int i;
0270
0271     desc = hsu_dma_alloc_desc(sg_len);
0272     if (!desc)
0273         return NULL;
0274
0275     for_each_sg(sgl, sg, sg_len, i) {
0276         desc->sg[i].addr = sg_dma_address(sg);
0277         desc->sg[i].len = sg_dma_len(sg);
0278
0279         desc->length += sg_dma_len(sg);
0280     }
0281
0282     desc->nents = sg_len;
0283     desc->direction = direction;
0284     /* desc->active = 0 by kzalloc */
0285     desc->status = DMA_IN_PROGRESS;
0286
0287     return vchan_tx_prep(&hsuc->vchan, &desc->vdesc, flags);
0288 }
0289
0290 static void hsu_dma_issue_pending(struct dma_chan *chan)
0291 {
0292     struct hsu_dma_chan *hsuc = to_hsu_dma_chan(chan);
0293     unsigned long flags;
0294
0295     spin_lock_irqsave(&hsuc->vchan.lock, flags);
0296     if (vchan_issue_pending(&hsuc->vchan) && !hsuc->desc)
0297         hsu_dma_start_transfer(hsuc);
0298     spin_unlock_irqrestore(&hsuc->vchan.lock, flags);
0299 }
0300
0301 static size_t hsu_dma_active_desc_size(struct hsu_dma_chan *hsuc)
0302 {
0303     struct hsu_dma_desc *desc = hsuc->desc;
0304     size_t bytes = 0;
0305     int i;
0306
0307     for (i = desc->active; i < desc->nents; i++)
0308         bytes += desc->sg[i].len;
0309
0310     i = HSU_DMA_CHAN_NR_DESC - 1;
0311     do {
0312         bytes += hsu_chan_readl(hsuc, HSU_CH_DxTSR(i));
0313     } while (--i >= 0);
0314
0315     return bytes;
0316 }
0317
0318 static enum dma_status hsu_dma_tx_status(struct dma_chan *chan,
0319     dma_cookie_t cookie, struct dma_tx_state *state)
0320 {
0321     struct hsu_dma_chan *hsuc = to_hsu_dma_chan(chan);
0322     struct virt_dma_desc *vdesc;
0323     enum dma_status status;
0324     size_t bytes;
0325     unsigned long flags;
0326
0327     status = dma_cookie_status(chan, cookie, state);
0328     if (status == DMA_COMPLETE)
0329         return status;
0330
0331     spin_lock_irqsave(&hsuc->vchan.lock, flags);
0332     vdesc = vchan_find_desc(&hsuc->vchan, cookie);
0333     if (hsuc->desc && cookie == hsuc->desc->vdesc.tx.cookie) {
0334         bytes = hsu_dma_active_desc_size(hsuc);
0335         dma_set_residue(state, bytes);
0336         status = hsuc->desc->status;
0337     } else if (vdesc) {
0338         bytes = to_hsu_dma_desc(vdesc)->length;
0339         dma_set_residue(state, bytes);
0340     }
0341     spin_unlock_irqrestore(&hsuc->vchan.lock, flags);
0342
0343     return status;
0344 }
0345
0346 static int hsu_dma_slave_config(struct dma_chan *chan,
0347                 struct dma_slave_config *config)
0348 {
0349     struct hsu_dma_chan *hsuc = to_hsu_dma_chan(chan);
0350
0351     memcpy(&hsuc->config, config, sizeof(hsuc->config));
0352
0353     return 0;
0354 }
0355
0356 static int hsu_dma_pause(struct dma_chan *chan)
0357 {
0358     struct hsu_dma_chan *hsuc = to_hsu_dma_chan(chan);
0359     unsigned long flags;
0360
0361     spin_lock_irqsave(&hsuc->vchan.lock, flags);
0362     if (hsuc->desc && hsuc->desc->status == DMA_IN_PROGRESS) {
0363         hsu_chan_disable(hsuc);
0364         hsuc->desc->status = DMA_PAUSED;
0365     }
0366     spin_unlock_irqrestore(&hsuc->vchan.lock, flags);
0367
0368     return 0;
0369 }
0370
0371 static int hsu_dma_resume(struct dma_chan *chan)
0372 {
0373     struct hsu_dma_chan *hsuc = to_hsu_dma_chan(chan);
0374     unsigned long flags;
0375
0376     spin_lock_irqsave(&hsuc->vchan.lock, flags);
0377     if (hsuc->desc && hsuc->desc->status == DMA_PAUSED) {
0378         hsuc->desc->status = DMA_IN_PROGRESS;
0379         hsu_chan_enable(hsuc);
0380     }
0381     spin_unlock_irqrestore(&hsuc->vchan.lock, flags);
0382
0383     return 0;
0384 }
0385
0386 static int hsu_dma_terminate_all(struct dma_chan *chan)
0387 {
0388     struct hsu_dma_chan *hsuc = to_hsu_dma_chan(chan);
0389     unsigned long flags;
0390     LIST_HEAD(head);
0391
0392     spin_lock_irqsave(&hsuc->vchan.lock, flags);
0393
0394     hsu_dma_stop_channel(hsuc);
0395     if (hsuc->desc) {
0396         hsu_dma_desc_free(&hsuc->desc->vdesc);
0397         hsuc->desc = NULL;
0398     }
0399
0400     vchan_get_all_descriptors(&hsuc->vchan, &head);
0401     spin_unlock_irqrestore(&hsuc->vchan.lock, flags);
0402     vchan_dma_desc_free_list(&hsuc->vchan, &head);
0403
0404     return 0;
0405 }
0406
0407 static void hsu_dma_free_chan_resources(struct dma_chan *chan)
0408 {
0409     vchan_free_chan_resources(to_virt_chan(chan));
0410 }
0411
0412 static void hsu_dma_synchronize(struct dma_chan *chan)
0413 {
0414     struct hsu_dma_chan *hsuc = to_hsu_dma_chan(chan);
0415
0416     vchan_synchronize(&hsuc->vchan);
0417 }
0418
0419 int hsu_dma_probe(struct hsu_dma_chip *chip)
0420 {
0421     struct hsu_dma *hsu;
0422     void __iomem *addr = chip->regs + chip->offset;
0423     unsigned short i;
0424     int ret;
0425
0426     hsu = devm_kzalloc(chip->dev, sizeof(*hsu), GFP_KERNEL);
0427     if (!hsu)
0428         return -ENOMEM;
0429
0430     chip->hsu = hsu;
0431
0432     /* Calculate nr_channels from the IO space length */
0433     hsu->nr_channels = (chip->length - chip->offset) / HSU_DMA_CHAN_LENGTH;
0434
0435     hsu->chan = devm_kcalloc(chip->dev, hsu->nr_channels,
0436                  sizeof(*hsu->chan), GFP_KERNEL);
0437     if (!hsu->chan)
0438         return -ENOMEM;
0439
0440     INIT_LIST_HEAD(&hsu->dma.channels);
0441     for (i = 0; i < hsu->nr_channels; i++) {
0442         struct hsu_dma_chan *hsuc = &hsu->chan[i];
0443
0444         hsuc->vchan.desc_free = hsu_dma_desc_free;
0445         vchan_init(&hsuc->vchan, &hsu->dma);
0446
0447         hsuc->direction = (i & 0x1) ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV;
0448         hsuc->reg = addr + i * HSU_DMA_CHAN_LENGTH;
0449     }
0450
0451     dma_cap_set(DMA_SLAVE, hsu->dma.cap_mask);
0452     dma_cap_set(DMA_PRIVATE, hsu->dma.cap_mask);
0453
0454     hsu->dma.device_free_chan_resources = hsu_dma_free_chan_resources;
0455
0456     hsu->dma.device_prep_slave_sg = hsu_dma_prep_slave_sg;
0457
0458     hsu->dma.device_issue_pending = hsu_dma_issue_pending;
0459     hsu->dma.device_tx_status = hsu_dma_tx_status;
0460
0461     hsu->dma.device_config = hsu_dma_slave_config;
0462     hsu->dma.device_pause = hsu_dma_pause;
0463     hsu->dma.device_resume = hsu_dma_resume;
0464     hsu->dma.device_terminate_all = hsu_dma_terminate_all;
0465     hsu->dma.device_synchronize = hsu_dma_synchronize;
0466
0467     hsu->dma.src_addr_widths = HSU_DMA_BUSWIDTHS;
0468     hsu->dma.dst_addr_widths = HSU_DMA_BUSWIDTHS;
0469     hsu->dma.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
0470     hsu->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
0471
0472     hsu->dma.dev = chip->dev;
0473
0474     dma_set_max_seg_size(hsu->dma.dev, HSU_CH_DxTSR_MASK);
0475
0476     ret = dma_async_device_register(&hsu->dma);
0477     if (ret)
0478         return ret;
0479
0480     dev_info(chip->dev, "Found HSU DMA, %d channels\n", hsu->nr_channels);
0481     return 0;
0482 }
0483 EXPORT_SYMBOL_GPL(hsu_dma_probe);
0484
0485 int hsu_dma_remove(struct hsu_dma_chip *chip)
0486 {
0487     struct hsu_dma *hsu = chip->hsu;
0488     unsigned short i;
0489
0490     dma_async_device_unregister(&hsu->dma);
0491
0492     for (i = 0; i < hsu->nr_channels; i++) {
0493         struct hsu_dma_chan *hsuc = &hsu->chan[i];
0494
0495         tasklet_kill(&hsuc->vchan.task);
0496     }
0497
0498     return 0;
0499 }
0500 EXPORT_SYMBOL_GPL(hsu_dma_remove);
0501
0502 MODULE_LICENSE("GPL v2");
0503 MODULE_DESCRIPTION("High Speed UART DMA core driver");
0504 MODULE_AUTHOR("Andy Shevchenko <andriy.shevchenko@linux.intel.com>");