drivers/dma/apple-admac.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Driver for Audio DMA Controller (ADMAC) on t8103 (M1) and other Apple chips
0004  *
0005  * Copyright (C) The Asahi Linux Contributors
0006  */
0007
0008 #include <linux/bits.h>
0009 #include <linux/bitfield.h>
0010 #include <linux/device.h>
0011 #include <linux/init.h>
0012 #include <linux/module.h>
0013 #include <linux/of_device.h>
0014 #include <linux/of_dma.h>
0015 #include <linux/interrupt.h>
0016 #include <linux/spinlock.h>
0017
0018 #include "dmaengine.h"
0019
0020 #define NCHANNELS_MAX   64
0021 #define IRQ_NOUTPUTS    4
0022
0023 #define RING_WRITE_SLOT     GENMASK(1, 0)
0024 #define RING_READ_SLOT      GENMASK(5, 4)
0025 #define RING_FULL       BIT(9)
0026 #define RING_EMPTY      BIT(8)
0027 #define RING_ERR        BIT(10)
0028
0029 #define STATUS_DESC_DONE    BIT(0)
0030 #define STATUS_ERR      BIT(6)
0031
0032 #define FLAG_DESC_NOTIFY    BIT(16)
0033
0034 #define REG_TX_START        0x0000
0035 #define REG_TX_STOP     0x0004
0036 #define REG_RX_START        0x0008
0037 #define REG_RX_STOP     0x000c
0038
0039 #define REG_CHAN_CTL(ch)    (0x8000 + (ch) * 0x200)
0040 #define REG_CHAN_CTL_RST_RINGS  BIT(0)
0041
0042 #define REG_DESC_RING(ch)   (0x8070 + (ch) * 0x200)
0043 #define REG_REPORT_RING(ch) (0x8074 + (ch) * 0x200)
0044
0045 #define REG_RESIDUE(ch)     (0x8064 + (ch) * 0x200)
0046
0047 #define REG_BUS_WIDTH(ch)   (0x8040 + (ch) * 0x200)
0048
0049 #define BUS_WIDTH_8BIT      0x00
0050 #define BUS_WIDTH_16BIT     0x01
0051 #define BUS_WIDTH_32BIT     0x02
0052 #define BUS_WIDTH_FRAME_2_WORDS 0x10
0053 #define BUS_WIDTH_FRAME_4_WORDS 0x20
0054
0055 #define CHAN_BUFSIZE        0x8000
0056
0057 #define REG_CHAN_FIFOCTL(ch)    (0x8054 + (ch) * 0x200)
0058 #define CHAN_FIFOCTL_LIMIT  GENMASK(31, 16)
0059 #define CHAN_FIFOCTL_THRESHOLD  GENMASK(15, 0)
0060
0061 #define REG_DESC_WRITE(ch)  (0x10000 + ((ch) / 2) * 0x4 + ((ch) & 1) * 0x4000)
0062 #define REG_REPORT_READ(ch) (0x10100 + ((ch) / 2) * 0x4 + ((ch) & 1) * 0x4000)
0063
0064 #define REG_TX_INTSTATE(idx)        (0x0030 + (idx) * 4)
0065 #define REG_RX_INTSTATE(idx)        (0x0040 + (idx) * 4)
0066 #define REG_CHAN_INTSTATUS(ch, idx) (0x8010 + (ch) * 0x200 + (idx) * 4)
0067 #define REG_CHAN_INTMASK(ch, idx)   (0x8020 + (ch) * 0x200 + (idx) * 4)
0068
0069 struct admac_data;
0070 struct admac_tx;
0071
0072 struct admac_chan {
0073     unsigned int no;
0074     struct admac_data *host;
0075     struct dma_chan chan;
0076     struct tasklet_struct tasklet;
0077
0078     spinlock_t lock;
0079     struct admac_tx *current_tx;
0080     int nperiod_acks;
0081
0082     /*
0083      * We maintain a 'submitted' and 'issued' list mainly for interface
0084      * correctness. Typical use of the driver (per channel) will be
0085      * prepping, submitting and issuing a single cyclic transaction which
0086      * will stay current until terminate_all is called.
0087      */
0088     struct list_head submitted;
0089     struct list_head issued;
0090
0091     struct list_head to_free;
0092 };
0093
0094 struct admac_data {
0095     struct dma_device dma;
0096     struct device *dev;
0097     __iomem void *base;
0098
0099     int irq_index;
0100     int nchannels;
0101     struct admac_chan channels[];
0102 };
0103
0104 struct admac_tx {
0105     struct dma_async_tx_descriptor tx;
0106     bool cyclic;
0107     dma_addr_t buf_addr;
0108     dma_addr_t buf_end;
0109     size_t buf_len;
0110     size_t period_len;
0111
0112     size_t submitted_pos;
0113     size_t reclaimed_pos;
0114
0115     struct list_head node;
0116 };
0117
0118 static void admac_modify(struct admac_data *ad, int reg, u32 mask, u32 val)
0119 {
0120     void __iomem *addr = ad->base + reg;
0121     u32 curr = readl_relaxed(addr);
0122
0123     writel_relaxed((curr & ~mask) | (val & mask), addr);
0124 }
0125
0126 static struct admac_chan *to_admac_chan(struct dma_chan *chan)
0127 {
0128     return container_of(chan, struct admac_chan, chan);
0129 }
0130
0131 static struct admac_tx *to_admac_tx(struct dma_async_tx_descriptor *tx)
0132 {
0133     return container_of(tx, struct admac_tx, tx);
0134 }
0135
0136 static enum dma_transfer_direction admac_chan_direction(int channo)
0137 {
0138     /* Channel directions are hardwired */
0139     return (channo & 1) ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV;
0140 }
0141
0142 static dma_cookie_t admac_tx_submit(struct dma_async_tx_descriptor *tx)
0143 {
0144     struct admac_tx *adtx = to_admac_tx(tx);
0145     struct admac_chan *adchan = to_admac_chan(tx->chan);
0146     unsigned long flags;
0147     dma_cookie_t cookie;
0148
0149     spin_lock_irqsave(&adchan->lock, flags);
0150     cookie = dma_cookie_assign(tx);
0151     list_add_tail(&adtx->node, &adchan->submitted);
0152     spin_unlock_irqrestore(&adchan->lock, flags);
0153
0154     return cookie;
0155 }
0156
0157 static int admac_desc_free(struct dma_async_tx_descriptor *tx)
0158 {
0159     kfree(to_admac_tx(tx));
0160
0161     return 0;
0162 }
0163
0164 static struct dma_async_tx_descriptor *admac_prep_dma_cyclic(
0165         struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
0166         size_t period_len, enum dma_transfer_direction direction,
0167         unsigned long flags)
0168 {
0169     struct admac_chan *adchan = container_of(chan, struct admac_chan, chan);
0170     struct admac_tx *adtx;
0171
0172     if (direction != admac_chan_direction(adchan->no))
0173         return NULL;
0174
0175     adtx = kzalloc(sizeof(*adtx), GFP_NOWAIT);
0176     if (!adtx)
0177         return NULL;
0178
0179     adtx->cyclic = true;
0180
0181     adtx->buf_addr = buf_addr;
0182     adtx->buf_len = buf_len;
0183     adtx->buf_end = buf_addr + buf_len;
0184     adtx->period_len = period_len;
0185
0186     adtx->submitted_pos = 0;
0187     adtx->reclaimed_pos = 0;
0188
0189     dma_async_tx_descriptor_init(&adtx->tx, chan);
0190     adtx->tx.tx_submit = admac_tx_submit;
0191     adtx->tx.desc_free = admac_desc_free;
0192
0193     return &adtx->tx;
0194 }
0195
0196 /*
0197  * Write one hardware descriptor for a dmaengine cyclic transaction.
0198  */
0199 static void admac_cyclic_write_one_desc(struct admac_data *ad, int channo,
0200                     struct admac_tx *tx)
0201 {
0202     dma_addr_t addr;
0203
0204     addr = tx->buf_addr + (tx->submitted_pos % tx->buf_len);
0205
0206     /* If happens means we have buggy code */
0207     WARN_ON_ONCE(addr + tx->period_len > tx->buf_end);
0208
0209     dev_dbg(ad->dev, "ch%d descriptor: addr=0x%pad len=0x%zx flags=0x%lx\n",
0210         channo, &addr, tx->period_len, FLAG_DESC_NOTIFY);
0211
0212     writel_relaxed(lower_32_bits(addr), ad->base + REG_DESC_WRITE(channo));
0213     writel_relaxed(upper_32_bits(addr), ad->base + REG_DESC_WRITE(channo));
0214     writel_relaxed(tx->period_len,      ad->base + REG_DESC_WRITE(channo));
0215     writel_relaxed(FLAG_DESC_NOTIFY,    ad->base + REG_DESC_WRITE(channo));
0216
0217     tx->submitted_pos += tx->period_len;
0218     tx->submitted_pos %= 2 * tx->buf_len;
0219 }
0220
0221 /*
0222  * Write all the hardware descriptors for a dmaengine cyclic
0223  * transaction there is space for.
0224  */
0225 static void admac_cyclic_write_desc(struct admac_data *ad, int channo,
0226                     struct admac_tx *tx)
0227 {
0228     int i;
0229
0230     for (i = 0; i < 4; i++) {
0231         if (readl_relaxed(ad->base + REG_DESC_RING(channo)) & RING_FULL)
0232             break;
0233         admac_cyclic_write_one_desc(ad, channo, tx);
0234     }
0235 }
0236
0237 static int admac_ring_noccupied_slots(int ringval)
0238 {
0239     int wrslot = FIELD_GET(RING_WRITE_SLOT, ringval);
0240     int rdslot = FIELD_GET(RING_READ_SLOT, ringval);
0241
0242     if (wrslot != rdslot) {
0243         return (wrslot + 4 - rdslot) % 4;
0244     } else {
0245         WARN_ON((ringval & (RING_FULL | RING_EMPTY)) == 0);
0246
0247         if (ringval & RING_FULL)
0248             return 4;
0249         else
0250             return 0;
0251     }
0252 }
0253
0254 /*
0255  * Read from hardware the residue of a cyclic dmaengine transaction.
0256  */
0257 static u32 admac_cyclic_read_residue(struct admac_data *ad, int channo,
0258                      struct admac_tx *adtx)
0259 {
0260     u32 ring1, ring2;
0261     u32 residue1, residue2;
0262     int nreports;
0263     size_t pos;
0264
0265     ring1 =    readl_relaxed(ad->base + REG_REPORT_RING(channo));
0266     residue1 = readl_relaxed(ad->base + REG_RESIDUE(channo));
0267     ring2 =    readl_relaxed(ad->base + REG_REPORT_RING(channo));
0268     residue2 = readl_relaxed(ad->base + REG_RESIDUE(channo));
0269
0270     if (residue2 > residue1) {
0271         /*
0272          * Controller must have loaded next descriptor between
0273          * the two residue reads
0274          */
0275         nreports = admac_ring_noccupied_slots(ring1) + 1;
0276     } else {
0277         /* No descriptor load between the two reads, ring2 is safe to use */
0278         nreports = admac_ring_noccupied_slots(ring2);
0279     }
0280
0281     pos = adtx->reclaimed_pos + adtx->period_len * (nreports + 1) - residue2;
0282
0283     return adtx->buf_len - pos % adtx->buf_len;
0284 }
0285
0286 static enum dma_status admac_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
0287                        struct dma_tx_state *txstate)
0288 {
0289     struct admac_chan *adchan = to_admac_chan(chan);
0290     struct admac_data *ad = adchan->host;
0291     struct admac_tx *adtx;
0292
0293     enum dma_status ret;
0294     size_t residue;
0295     unsigned long flags;
0296
0297     ret = dma_cookie_status(chan, cookie, txstate);
0298     if (ret == DMA_COMPLETE || !txstate)
0299         return ret;
0300
0301     spin_lock_irqsave(&adchan->lock, flags);
0302     adtx = adchan->current_tx;
0303
0304     if (adtx && adtx->tx.cookie == cookie) {
0305         ret = DMA_IN_PROGRESS;
0306         residue = admac_cyclic_read_residue(ad, adchan->no, adtx);
0307     } else {
0308         ret = DMA_IN_PROGRESS;
0309         residue = 0;
0310         list_for_each_entry(adtx, &adchan->issued, node) {
0311             if (adtx->tx.cookie == cookie) {
0312                 residue = adtx->buf_len;
0313                 break;
0314             }
0315         }
0316     }
0317     spin_unlock_irqrestore(&adchan->lock, flags);
0318
0319     dma_set_residue(txstate, residue);
0320     return ret;
0321 }
0322
0323 static void admac_start_chan(struct admac_chan *adchan)
0324 {
0325     struct admac_data *ad = adchan->host;
0326     u32 startbit = 1 << (adchan->no / 2);
0327
0328     writel_relaxed(STATUS_DESC_DONE | STATUS_ERR,
0329                ad->base + REG_CHAN_INTSTATUS(adchan->no, ad->irq_index));
0330     writel_relaxed(STATUS_DESC_DONE | STATUS_ERR,
0331                ad->base + REG_CHAN_INTMASK(adchan->no, ad->irq_index));
0332
0333     switch (admac_chan_direction(adchan->no)) {
0334     case DMA_MEM_TO_DEV:
0335         writel_relaxed(startbit, ad->base + REG_TX_START);
0336         break;
0337     case DMA_DEV_TO_MEM:
0338         writel_relaxed(startbit, ad->base + REG_RX_START);
0339         break;
0340     default:
0341         break;
0342     }
0343     dev_dbg(adchan->host->dev, "ch%d start\n", adchan->no);
0344 }
0345
0346 static void admac_stop_chan(struct admac_chan *adchan)
0347 {
0348     struct admac_data *ad = adchan->host;
0349     u32 stopbit = 1 << (adchan->no / 2);
0350
0351     switch (admac_chan_direction(adchan->no)) {
0352     case DMA_MEM_TO_DEV:
0353         writel_relaxed(stopbit, ad->base + REG_TX_STOP);
0354         break;
0355     case DMA_DEV_TO_MEM:
0356         writel_relaxed(stopbit, ad->base + REG_RX_STOP);
0357         break;
0358     default:
0359         break;
0360     }
0361     dev_dbg(adchan->host->dev, "ch%d stop\n", adchan->no);
0362 }
0363
0364 static void admac_reset_rings(struct admac_chan *adchan)
0365 {
0366     struct admac_data *ad = adchan->host;
0367
0368     writel_relaxed(REG_CHAN_CTL_RST_RINGS,
0369                ad->base + REG_CHAN_CTL(adchan->no));
0370     writel_relaxed(0, ad->base + REG_CHAN_CTL(adchan->no));
0371 }
0372
0373 static void admac_start_current_tx(struct admac_chan *adchan)
0374 {
0375     struct admac_data *ad = adchan->host;
0376     int ch = adchan->no;
0377
0378     admac_reset_rings(adchan);
0379     writel_relaxed(0, ad->base + REG_CHAN_CTL(ch));
0380
0381     admac_cyclic_write_one_desc(ad, ch, adchan->current_tx);
0382     admac_start_chan(adchan);
0383     admac_cyclic_write_desc(ad, ch, adchan->current_tx);
0384 }
0385
0386 static void admac_issue_pending(struct dma_chan *chan)
0387 {
0388     struct admac_chan *adchan = to_admac_chan(chan);
0389     struct admac_tx *tx;
0390     unsigned long flags;
0391
0392     spin_lock_irqsave(&adchan->lock, flags);
0393     list_splice_tail_init(&adchan->submitted, &adchan->issued);
0394     if (!list_empty(&adchan->issued) && !adchan->current_tx) {
0395         tx = list_first_entry(&adchan->issued, struct admac_tx, node);
0396         list_del(&tx->node);
0397
0398         adchan->current_tx = tx;
0399         adchan->nperiod_acks = 0;
0400         admac_start_current_tx(adchan);
0401     }
0402     spin_unlock_irqrestore(&adchan->lock, flags);
0403 }
0404
0405 static int admac_pause(struct dma_chan *chan)
0406 {
0407     struct admac_chan *adchan = to_admac_chan(chan);
0408
0409     admac_stop_chan(adchan);
0410
0411     return 0;
0412 }
0413
0414 static int admac_resume(struct dma_chan *chan)
0415 {
0416     struct admac_chan *adchan = to_admac_chan(chan);
0417
0418     admac_start_chan(adchan);
0419
0420     return 0;
0421 }
0422
0423 static int admac_terminate_all(struct dma_chan *chan)
0424 {
0425     struct admac_chan *adchan = to_admac_chan(chan);
0426     unsigned long flags;
0427
0428     spin_lock_irqsave(&adchan->lock, flags);
0429     admac_stop_chan(adchan);
0430     admac_reset_rings(adchan);
0431
0432     adchan->current_tx = NULL;
0433     /*
0434      * Descriptors can only be freed after the tasklet
0435      * has been killed (in admac_synchronize).
0436      */
0437     list_splice_tail_init(&adchan->submitted, &adchan->to_free);
0438     list_splice_tail_init(&adchan->issued, &adchan->to_free);
0439     spin_unlock_irqrestore(&adchan->lock, flags);
0440
0441     return 0;
0442 }
0443
0444 static void admac_synchronize(struct dma_chan *chan)
0445 {
0446     struct admac_chan *adchan = to_admac_chan(chan);
0447     struct admac_tx *adtx, *_adtx;
0448     unsigned long flags;
0449     LIST_HEAD(head);
0450
0451     spin_lock_irqsave(&adchan->lock, flags);
0452     list_splice_tail_init(&adchan->to_free, &head);
0453     spin_unlock_irqrestore(&adchan->lock, flags);
0454
0455     tasklet_kill(&adchan->tasklet);
0456
0457     list_for_each_entry_safe(adtx, _adtx, &head, node) {
0458         list_del(&adtx->node);
0459         admac_desc_free(&adtx->tx);
0460     }
0461 }
0462
0463 static int admac_alloc_chan_resources(struct dma_chan *chan)
0464 {
0465     struct admac_chan *adchan = to_admac_chan(chan);
0466
0467     dma_cookie_init(&adchan->chan);
0468     return 0;
0469 }
0470
0471 static void admac_free_chan_resources(struct dma_chan *chan)
0472 {
0473     admac_terminate_all(chan);
0474     admac_synchronize(chan);
0475 }
0476
0477 static struct dma_chan *admac_dma_of_xlate(struct of_phandle_args *dma_spec,
0478                        struct of_dma *ofdma)
0479 {
0480     struct admac_data *ad = (struct admac_data *) ofdma->of_dma_data;
0481     unsigned int index;
0482
0483     if (dma_spec->args_count != 1)
0484         return NULL;
0485
0486     index = dma_spec->args[0];
0487
0488     if (index >= ad->nchannels) {
0489         dev_err(ad->dev, "channel index %u out of bounds\n", index);
0490         return NULL;
0491     }
0492
0493     return &ad->channels[index].chan;
0494 }
0495
0496 static int admac_drain_reports(struct admac_data *ad, int channo)
0497 {
0498     int count;
0499
0500     for (count = 0; count < 4; count++) {
0501         u32 countval_hi, countval_lo, unk1, flags;
0502
0503         if (readl_relaxed(ad->base + REG_REPORT_RING(channo)) & RING_EMPTY)
0504             break;
0505
0506         countval_lo = readl_relaxed(ad->base + REG_REPORT_READ(channo));
0507         countval_hi = readl_relaxed(ad->base + REG_REPORT_READ(channo));
0508         unk1 =        readl_relaxed(ad->base + REG_REPORT_READ(channo));
0509         flags =       readl_relaxed(ad->base + REG_REPORT_READ(channo));
0510
0511         dev_dbg(ad->dev, "ch%d report: countval=0x%llx unk1=0x%x flags=0x%x\n",
0512             channo, ((u64) countval_hi) << 32 | countval_lo, unk1, flags);
0513     }
0514
0515     return count;
0516 }
0517
0518 static void admac_handle_status_err(struct admac_data *ad, int channo)
0519 {
0520     bool handled = false;
0521
0522     if (readl_relaxed(ad->base + REG_DESC_RING(channo)) & RING_ERR) {
0523         writel_relaxed(RING_ERR, ad->base + REG_DESC_RING(channo));
0524         dev_err_ratelimited(ad->dev, "ch%d descriptor ring error\n", channo);
0525         handled = true;
0526     }
0527
0528     if (readl_relaxed(ad->base + REG_REPORT_RING(channo)) & RING_ERR) {
0529         writel_relaxed(RING_ERR, ad->base + REG_REPORT_RING(channo));
0530         dev_err_ratelimited(ad->dev, "ch%d report ring error\n", channo);
0531         handled = true;
0532     }
0533
0534     if (unlikely(!handled)) {
0535         dev_err(ad->dev, "ch%d unknown error, masking errors as cause of IRQs\n", channo);
0536         admac_modify(ad, REG_CHAN_INTMASK(channo, ad->irq_index),
0537                  STATUS_ERR, 0);
0538     }
0539 }
0540
0541 static void admac_handle_status_desc_done(struct admac_data *ad, int channo)
0542 {
0543     struct admac_chan *adchan = &ad->channels[channo];
0544     unsigned long flags;
0545     int nreports;
0546
0547     writel_relaxed(STATUS_DESC_DONE,
0548                ad->base + REG_CHAN_INTSTATUS(channo, ad->irq_index));
0549
0550     spin_lock_irqsave(&adchan->lock, flags);
0551     nreports = admac_drain_reports(ad, channo);
0552
0553     if (adchan->current_tx) {
0554         struct admac_tx *tx = adchan->current_tx;
0555
0556         adchan->nperiod_acks += nreports;
0557         tx->reclaimed_pos += nreports * tx->period_len;
0558         tx->reclaimed_pos %= 2 * tx->buf_len;
0559
0560         admac_cyclic_write_desc(ad, channo, tx);
0561         tasklet_schedule(&adchan->tasklet);
0562     }
0563     spin_unlock_irqrestore(&adchan->lock, flags);
0564 }
0565
0566 static void admac_handle_chan_int(struct admac_data *ad, int no)
0567 {
0568     u32 cause = readl_relaxed(ad->base + REG_CHAN_INTSTATUS(no, ad->irq_index));
0569
0570     if (cause & STATUS_ERR)
0571         admac_handle_status_err(ad, no);
0572
0573     if (cause & STATUS_DESC_DONE)
0574         admac_handle_status_desc_done(ad, no);
0575 }
0576
0577 static irqreturn_t admac_interrupt(int irq, void *devid)
0578 {
0579     struct admac_data *ad = devid;
0580     u32 rx_intstate, tx_intstate;
0581     int i;
0582
0583     rx_intstate = readl_relaxed(ad->base + REG_RX_INTSTATE(ad->irq_index));
0584     tx_intstate = readl_relaxed(ad->base + REG_TX_INTSTATE(ad->irq_index));
0585
0586     if (!tx_intstate && !rx_intstate)
0587         return IRQ_NONE;
0588
0589     for (i = 0; i < ad->nchannels; i += 2) {
0590         if (tx_intstate & 1)
0591             admac_handle_chan_int(ad, i);
0592         tx_intstate >>= 1;
0593     }
0594
0595     for (i = 1; i < ad->nchannels; i += 2) {
0596         if (rx_intstate & 1)
0597             admac_handle_chan_int(ad, i);
0598         rx_intstate >>= 1;
0599     }
0600
0601     return IRQ_HANDLED;
0602 }
0603
0604 static void admac_chan_tasklet(struct tasklet_struct *t)
0605 {
0606     struct admac_chan *adchan = from_tasklet(adchan, t, tasklet);
0607     struct admac_tx *adtx;
0608     struct dmaengine_desc_callback cb;
0609     struct dmaengine_result tx_result;
0610     int nacks;
0611
0612     spin_lock_irq(&adchan->lock);
0613     adtx = adchan->current_tx;
0614     nacks = adchan->nperiod_acks;
0615     adchan->nperiod_acks = 0;
0616     spin_unlock_irq(&adchan->lock);
0617
0618     if (!adtx || !nacks)
0619         return;
0620
0621     tx_result.result = DMA_TRANS_NOERROR;
0622     tx_result.residue = 0;
0623
0624     dmaengine_desc_get_callback(&adtx->tx, &cb);
0625     while (nacks--)
0626         dmaengine_desc_callback_invoke(&cb, &tx_result);
0627 }
0628
0629 static int admac_device_config(struct dma_chan *chan,
0630                    struct dma_slave_config *config)
0631 {
0632     struct admac_chan *adchan = to_admac_chan(chan);
0633     struct admac_data *ad = adchan->host;
0634     bool is_tx = admac_chan_direction(adchan->no) == DMA_MEM_TO_DEV;
0635     int wordsize = 0;
0636     u32 bus_width = 0;
0637
0638     switch (is_tx ? config->dst_addr_width : config->src_addr_width) {
0639     case DMA_SLAVE_BUSWIDTH_1_BYTE:
0640         wordsize = 1;
0641         bus_width |= BUS_WIDTH_8BIT;
0642         break;
0643     case DMA_SLAVE_BUSWIDTH_2_BYTES:
0644         wordsize = 2;
0645         bus_width |= BUS_WIDTH_16BIT;
0646         break;
0647     case DMA_SLAVE_BUSWIDTH_4_BYTES:
0648         wordsize = 4;
0649         bus_width |= BUS_WIDTH_32BIT;
0650         break;
0651     default:
0652         return -EINVAL;
0653     }
0654
0655     /*
0656      * We take port_window_size to be the number of words in a frame.
0657      *
0658      * The controller has some means of out-of-band signalling, to the peripheral,
0659      * of words position in a frame. That's where the importance of this control
0660      * comes from.
0661      */
0662     switch (is_tx ? config->dst_port_window_size : config->src_port_window_size) {
0663     case 0 ... 1:
0664         break;
0665     case 2:
0666         bus_width |= BUS_WIDTH_FRAME_2_WORDS;
0667         break;
0668     case 4:
0669         bus_width |= BUS_WIDTH_FRAME_4_WORDS;
0670         break;
0671     default:
0672         return -EINVAL;
0673     }
0674
0675     writel_relaxed(bus_width, ad->base + REG_BUS_WIDTH(adchan->no));
0676
0677     /*
0678      * By FIFOCTL_LIMIT we seem to set the maximal number of bytes allowed to be
0679      * held in controller's per-channel FIFO. Transfers seem to be triggered
0680      * around the time FIFO occupancy touches FIFOCTL_THRESHOLD.
0681      *
0682      * The numbers we set are more or less arbitrary.
0683      */
0684     writel_relaxed(FIELD_PREP(CHAN_FIFOCTL_LIMIT, 0x30 * wordsize)
0685                | FIELD_PREP(CHAN_FIFOCTL_THRESHOLD, 0x18 * wordsize),
0686                ad->base + REG_CHAN_FIFOCTL(adchan->no));
0687
0688     return 0;
0689 }
0690
0691 static int admac_probe(struct platform_device *pdev)
0692 {
0693     struct device_node *np = pdev->dev.of_node;
0694     struct admac_data *ad;
0695     struct dma_device *dma;
0696     int nchannels;
0697     int err, irq, i;
0698
0699     err = of_property_read_u32(np, "dma-channels", &nchannels);
0700     if (err || nchannels > NCHANNELS_MAX) {
0701         dev_err(&pdev->dev, "missing or invalid dma-channels property\n");
0702         return -EINVAL;
0703     }
0704
0705     ad = devm_kzalloc(&pdev->dev, struct_size(ad, channels, nchannels), GFP_KERNEL);
0706     if (!ad)
0707         return -ENOMEM;
0708
0709     platform_set_drvdata(pdev, ad);
0710     ad->dev = &pdev->dev;
0711     ad->nchannels = nchannels;
0712
0713     /*
0714      * The controller has 4 IRQ outputs. Try them all until
0715      * we find one we can use.
0716      */
0717     for (i = 0; i < IRQ_NOUTPUTS; i++) {
0718         irq = platform_get_irq_optional(pdev, i);
0719         if (irq >= 0) {
0720             ad->irq_index = i;
0721             break;
0722         }
0723     }
0724
0725     if (irq < 0)
0726         return dev_err_probe(&pdev->dev, irq, "no usable interrupt\n");
0727
0728     err = devm_request_irq(&pdev->dev, irq, admac_interrupt,
0729                    0, dev_name(&pdev->dev), ad);
0730     if (err)
0731         return dev_err_probe(&pdev->dev, err,
0732                      "unable to register interrupt\n");
0733
0734     ad->base = devm_platform_ioremap_resource(pdev, 0);
0735     if (IS_ERR(ad->base))
0736         return dev_err_probe(&pdev->dev, PTR_ERR(ad->base),
0737                      "unable to obtain MMIO resource\n");
0738
0739     dma = &ad->dma;
0740
0741     dma_cap_set(DMA_PRIVATE, dma->cap_mask);
0742     dma_cap_set(DMA_CYCLIC, dma->cap_mask);
0743
0744     dma->dev = &pdev->dev;
0745     dma->device_alloc_chan_resources = admac_alloc_chan_resources;
0746     dma->device_free_chan_resources = admac_free_chan_resources;
0747     dma->device_tx_status = admac_tx_status;
0748     dma->device_issue_pending = admac_issue_pending;
0749     dma->device_terminate_all = admac_terminate_all;
0750     dma->device_synchronize = admac_synchronize;
0751     dma->device_prep_dma_cyclic = admac_prep_dma_cyclic;
0752     dma->device_config = admac_device_config;
0753     dma->device_pause = admac_pause;
0754     dma->device_resume = admac_resume;
0755
0756     dma->directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM);
0757     dma->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
0758     dma->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
0759             BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
0760             BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
0761
0762     INIT_LIST_HEAD(&dma->channels);
0763     for (i = 0; i < nchannels; i++) {
0764         struct admac_chan *adchan = &ad->channels[i];
0765
0766         adchan->host = ad;
0767         adchan->no = i;
0768         adchan->chan.device = &ad->dma;
0769         spin_lock_init(&adchan->lock);
0770         INIT_LIST_HEAD(&adchan->submitted);
0771         INIT_LIST_HEAD(&adchan->issued);
0772         INIT_LIST_HEAD(&adchan->to_free);
0773         list_add_tail(&adchan->chan.device_node, &dma->channels);
0774         tasklet_setup(&adchan->tasklet, admac_chan_tasklet);
0775     }
0776
0777     err = dma_async_device_register(&ad->dma);
0778     if (err)
0779         return dev_err_probe(&pdev->dev, err, "failed to register DMA device\n");
0780
0781     err = of_dma_controller_register(pdev->dev.of_node, admac_dma_of_xlate, ad);
0782     if (err) {
0783         dma_async_device_unregister(&ad->dma);
0784         return dev_err_probe(&pdev->dev, err, "failed to register with OF\n");
0785     }
0786
0787     return 0;
0788 }
0789
0790 static int admac_remove(struct platform_device *pdev)
0791 {
0792     struct admac_data *ad = platform_get_drvdata(pdev);
0793
0794     of_dma_controller_free(pdev->dev.of_node);
0795     dma_async_device_unregister(&ad->dma);
0796
0797     return 0;
0798 }
0799
0800 static const struct of_device_id admac_of_match[] = {
0801     { .compatible = "apple,admac", },
0802     { }
0803 };
0804 MODULE_DEVICE_TABLE(of, admac_of_match);
0805
0806 static struct platform_driver apple_admac_driver = {
0807     .driver = {
0808         .name = "apple-admac",
0809         .of_match_table = admac_of_match,
0810     },
0811     .probe = admac_probe,
0812     .remove = admac_remove,
0813 };
0814 module_platform_driver(apple_admac_driver);
0815
0816 MODULE_AUTHOR("Martin Povišer <povik+lin@cutebit.org>");
0817 MODULE_DESCRIPTION("Driver for Audio DMA Controller (ADMAC) on Apple SoCs");
0818 MODULE_LICENSE("GPL");