dma/ppc4xx/adma.c

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  * Copyright (C) 2006-2009 DENX Software Engineering.
0004  *
0005  * Author: Yuri Tikhonov <yur@emcraft.com>
0006  *
0007  * Further porting to arch/powerpc by
0008  *  Anatolij Gustschin <agust@denx.de>
0009  */
0010
0011 /*
0012  * This driver supports the asynchrounous DMA copy and RAID engines available
0013  * on the AMCC PPC440SPe Processors.
0014  * Based on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x)
0015  * ADMA driver written by D.Williams.
0016  */
0017
0018 #include <linux/init.h>
0019 #include <linux/module.h>
0020 #include <linux/async_tx.h>
0021 #include <linux/delay.h>
0022 #include <linux/dma-mapping.h>
0023 #include <linux/spinlock.h>
0024 #include <linux/interrupt.h>
0025 #include <linux/slab.h>
0026 #include <linux/uaccess.h>
0027 #include <linux/proc_fs.h>
0028 #include <linux/of.h>
0029 #include <linux/of_address.h>
0030 #include <linux/of_irq.h>
0031 #include <linux/of_platform.h>
0032 #include <asm/dcr.h>
0033 #include <asm/dcr-regs.h>
0034 #include "adma.h"
0035 #include "../dmaengine.h"
0036
0037 enum ppc_adma_init_code {
0038     PPC_ADMA_INIT_OK = 0,
0039     PPC_ADMA_INIT_MEMRES,
0040     PPC_ADMA_INIT_MEMREG,
0041     PPC_ADMA_INIT_ALLOC,
0042     PPC_ADMA_INIT_COHERENT,
0043     PPC_ADMA_INIT_CHANNEL,
0044     PPC_ADMA_INIT_IRQ1,
0045     PPC_ADMA_INIT_IRQ2,
0046     PPC_ADMA_INIT_REGISTER
0047 };
0048
0049 static char *ppc_adma_errors[] = {
0050     [PPC_ADMA_INIT_OK] = "ok",
0051     [PPC_ADMA_INIT_MEMRES] = "failed to get memory resource",
0052     [PPC_ADMA_INIT_MEMREG] = "failed to request memory region",
0053     [PPC_ADMA_INIT_ALLOC] = "failed to allocate memory for adev "
0054                 "structure",
0055     [PPC_ADMA_INIT_COHERENT] = "failed to allocate coherent memory for "
0056                    "hardware descriptors",
0057     [PPC_ADMA_INIT_CHANNEL] = "failed to allocate memory for channel",
0058     [PPC_ADMA_INIT_IRQ1] = "failed to request first irq",
0059     [PPC_ADMA_INIT_IRQ2] = "failed to request second irq",
0060     [PPC_ADMA_INIT_REGISTER] = "failed to register dma async device",
0061 };
0062
0063 static enum ppc_adma_init_code
0064 ppc440spe_adma_devices[PPC440SPE_ADMA_ENGINES_NUM];
0065
0066 struct ppc_dma_chan_ref {
0067     struct dma_chan *chan;
0068     struct list_head node;
0069 };
0070
0071 /* The list of channels exported by ppc440spe ADMA */
0072 static struct list_head
0073 ppc440spe_adma_chan_list = LIST_HEAD_INIT(ppc440spe_adma_chan_list);
0074
0075 /* This flag is set when want to refetch the xor chain in the interrupt
0076  * handler
0077  */
0078 static u32 do_xor_refetch;
0079
0080 /* Pointer to DMA0, DMA1 CP/CS FIFO */
0081 static void *ppc440spe_dma_fifo_buf;
0082
0083 /* Pointers to last submitted to DMA0, DMA1 CDBs */
0084 static struct ppc440spe_adma_desc_slot *chan_last_sub[3];
0085 static struct ppc440spe_adma_desc_slot *chan_first_cdb[3];
0086
0087 /* Pointer to last linked and submitted xor CB */
0088 static struct ppc440spe_adma_desc_slot *xor_last_linked;
0089 static struct ppc440spe_adma_desc_slot *xor_last_submit;
0090
0091 /* This array is used in data-check operations for storing a pattern */
0092 static char ppc440spe_qword[16];
0093
0094 static atomic_t ppc440spe_adma_err_irq_ref;
0095 static dcr_host_t ppc440spe_mq_dcr_host;
0096 static unsigned int ppc440spe_mq_dcr_len;
0097
0098 /* Since RXOR operations use the common register (MQ0_CF2H) for setting-up
0099  * the block size in transactions, then we do not allow to activate more than
0100  * only one RXOR transactions simultaneously. So use this var to store
0101  * the information about is RXOR currently active (PPC440SPE_RXOR_RUN bit is
0102  * set) or not (PPC440SPE_RXOR_RUN is clear).
0103  */
0104 static unsigned long ppc440spe_rxor_state;
0105
0106 /* These are used in enable & check routines
0107  */
0108 static u32 ppc440spe_r6_enabled;
0109 static struct ppc440spe_adma_chan *ppc440spe_r6_tchan;
0110 static struct completion ppc440spe_r6_test_comp;
0111
0112 static int ppc440spe_adma_dma2rxor_prep_src(
0113         struct ppc440spe_adma_desc_slot *desc,
0114         struct ppc440spe_rxor *cursor, int index,
0115         int src_cnt, u32 addr);
0116 static void ppc440spe_adma_dma2rxor_set_src(
0117         struct ppc440spe_adma_desc_slot *desc,
0118         int index, dma_addr_t addr);
0119 static void ppc440spe_adma_dma2rxor_set_mult(
0120         struct ppc440spe_adma_desc_slot *desc,
0121         int index, u8 mult);
0122
0123 #ifdef ADMA_LL_DEBUG
0124 #define ADMA_LL_DBG(x) ({ if (1) x; 0; })
0125 #else
0126 #define ADMA_LL_DBG(x) ({ if (0) x; 0; })
0127 #endif
0128
0129 static void print_cb(struct ppc440spe_adma_chan *chan, void *block)
0130 {
0131     struct dma_cdb *cdb;
0132     struct xor_cb *cb;
0133     int i;
0134
0135     switch (chan->device->id) {
0136     case 0:
0137     case 1:
0138         cdb = block;
0139
0140         pr_debug("CDB at %p [%d]:\n"
0141             "\t attr 0x%02x opc 0x%02x cnt 0x%08x\n"
0142             "\t sg1u 0x%08x sg1l 0x%08x\n"
0143             "\t sg2u 0x%08x sg2l 0x%08x\n"
0144             "\t sg3u 0x%08x sg3l 0x%08x\n",
0145             cdb, chan->device->id,
0146             cdb->attr, cdb->opc, le32_to_cpu(cdb->cnt),
0147             le32_to_cpu(cdb->sg1u), le32_to_cpu(cdb->sg1l),
0148             le32_to_cpu(cdb->sg2u), le32_to_cpu(cdb->sg2l),
0149             le32_to_cpu(cdb->sg3u), le32_to_cpu(cdb->sg3l)
0150         );
0151         break;
0152     case 2:
0153         cb = block;
0154
0155         pr_debug("CB at %p [%d]:\n"
0156             "\t cbc 0x%08x cbbc 0x%08x cbs 0x%08x\n"
0157             "\t cbtah 0x%08x cbtal 0x%08x\n"
0158             "\t cblah 0x%08x cblal 0x%08x\n",
0159             cb, chan->device->id,
0160             cb->cbc, cb->cbbc, cb->cbs,
0161             cb->cbtah, cb->cbtal,
0162             cb->cblah, cb->cblal);
0163         for (i = 0; i < 16; i++) {
0164             if (i && !cb->ops[i].h && !cb->ops[i].l)
0165                 continue;
0166             pr_debug("\t ops[%2d]: h 0x%08x l 0x%08x\n",
0167                 i, cb->ops[i].h, cb->ops[i].l);
0168         }
0169         break;
0170     }
0171 }
0172
0173 static void print_cb_list(struct ppc440spe_adma_chan *chan,
0174               struct ppc440spe_adma_desc_slot *iter)
0175 {
0176     for (; iter; iter = iter->hw_next)
0177         print_cb(chan, iter->hw_desc);
0178 }
0179
0180 static void prep_dma_xor_dbg(int id, dma_addr_t dst, dma_addr_t *src,
0181                  unsigned int src_cnt)
0182 {
0183     int i;
0184
0185     pr_debug("\n%s(%d):\nsrc: ", __func__, id);
0186     for (i = 0; i < src_cnt; i++)
0187         pr_debug("\t0x%016llx ", src[i]);
0188     pr_debug("dst:\n\t0x%016llx\n", dst);
0189 }
0190
0191 static void prep_dma_pq_dbg(int id, dma_addr_t *dst, dma_addr_t *src,
0192                 unsigned int src_cnt)
0193 {
0194     int i;
0195
0196     pr_debug("\n%s(%d):\nsrc: ", __func__, id);
0197     for (i = 0; i < src_cnt; i++)
0198         pr_debug("\t0x%016llx ", src[i]);
0199     pr_debug("dst: ");
0200     for (i = 0; i < 2; i++)
0201         pr_debug("\t0x%016llx ", dst[i]);
0202 }
0203
0204 static void prep_dma_pqzero_sum_dbg(int id, dma_addr_t *src,
0205                     unsigned int src_cnt,
0206                     const unsigned char *scf)
0207 {
0208     int i;
0209
0210     pr_debug("\n%s(%d):\nsrc(coef): ", __func__, id);
0211     if (scf) {
0212         for (i = 0; i < src_cnt; i++)
0213             pr_debug("\t0x%016llx(0x%02x) ", src[i], scf[i]);
0214     } else {
0215         for (i = 0; i < src_cnt; i++)
0216             pr_debug("\t0x%016llx(no) ", src[i]);
0217     }
0218
0219     pr_debug("dst: ");
0220     for (i = 0; i < 2; i++)
0221         pr_debug("\t0x%016llx ", src[src_cnt + i]);
0222 }
0223
0224 /******************************************************************************
0225  * Command (Descriptor) Blocks low-level routines
0226  ******************************************************************************/
0227 /**
0228  * ppc440spe_desc_init_interrupt - initialize the descriptor for INTERRUPT
0229  * pseudo operation
0230  */
0231 static void ppc440spe_desc_init_interrupt(struct ppc440spe_adma_desc_slot *desc,
0232                       struct ppc440spe_adma_chan *chan)
0233 {
0234     struct xor_cb *p;
0235
0236     switch (chan->device->id) {
0237     case PPC440SPE_XOR_ID:
0238         p = desc->hw_desc;
0239         memset(desc->hw_desc, 0, sizeof(struct xor_cb));
0240         /* NOP with Command Block Complete Enable */
0241         p->cbc = XOR_CBCR_CBCE_BIT;
0242         break;
0243     case PPC440SPE_DMA0_ID:
0244     case PPC440SPE_DMA1_ID:
0245         memset(desc->hw_desc, 0, sizeof(struct dma_cdb));
0246         /* NOP with interrupt */
0247         set_bit(PPC440SPE_DESC_INT, &desc->flags);
0248         break;
0249     default:
0250         printk(KERN_ERR "Unsupported id %d in %s\n", chan->device->id,
0251                 __func__);
0252         break;
0253     }
0254 }
0255
0256 /**
0257  * ppc440spe_desc_init_null_xor - initialize the descriptor for NULL XOR
0258  * pseudo operation
0259  */
0260 static void ppc440spe_desc_init_null_xor(struct ppc440spe_adma_desc_slot *desc)
0261 {
0262     memset(desc->hw_desc, 0, sizeof(struct xor_cb));
0263     desc->hw_next = NULL;
0264     desc->src_cnt = 0;
0265     desc->dst_cnt = 1;
0266 }
0267
0268 /**
0269  * ppc440spe_desc_init_xor - initialize the descriptor for XOR operation
0270  */
0271 static void ppc440spe_desc_init_xor(struct ppc440spe_adma_desc_slot *desc,
0272                      int src_cnt, unsigned long flags)
0273 {
0274     struct xor_cb *hw_desc = desc->hw_desc;
0275
0276     memset(desc->hw_desc, 0, sizeof(struct xor_cb));
0277     desc->hw_next = NULL;
0278     desc->src_cnt = src_cnt;
0279     desc->dst_cnt = 1;
0280
0281     hw_desc->cbc = XOR_CBCR_TGT_BIT | src_cnt;
0282     if (flags & DMA_PREP_INTERRUPT)
0283         /* Enable interrupt on completion */
0284         hw_desc->cbc |= XOR_CBCR_CBCE_BIT;
0285 }
0286
0287 /**
0288  * ppc440spe_desc_init_dma2pq - initialize the descriptor for PQ
0289  * operation in DMA2 controller
0290  */
0291 static void ppc440spe_desc_init_dma2pq(struct ppc440spe_adma_desc_slot *desc,
0292         int dst_cnt, int src_cnt, unsigned long flags)
0293 {
0294     struct xor_cb *hw_desc = desc->hw_desc;
0295
0296     memset(desc->hw_desc, 0, sizeof(struct xor_cb));
0297     desc->hw_next = NULL;
0298     desc->src_cnt = src_cnt;
0299     desc->dst_cnt = dst_cnt;
0300     memset(desc->reverse_flags, 0, sizeof(desc->reverse_flags));
0301     desc->descs_per_op = 0;
0302
0303     hw_desc->cbc = XOR_CBCR_TGT_BIT;
0304     if (flags & DMA_PREP_INTERRUPT)
0305         /* Enable interrupt on completion */
0306         hw_desc->cbc |= XOR_CBCR_CBCE_BIT;
0307 }
0308
0309 #define DMA_CTRL_FLAGS_LAST DMA_PREP_FENCE
0310 #define DMA_PREP_ZERO_P     (DMA_CTRL_FLAGS_LAST << 1)
0311 #define DMA_PREP_ZERO_Q     (DMA_PREP_ZERO_P << 1)
0312
0313 /**
0314  * ppc440spe_desc_init_dma01pq - initialize the descriptors for PQ operation
0315  * with DMA0/1
0316  */
0317 static void ppc440spe_desc_init_dma01pq(struct ppc440spe_adma_desc_slot *desc,
0318                 int dst_cnt, int src_cnt, unsigned long flags,
0319                 unsigned long op)
0320 {
0321     struct dma_cdb *hw_desc;
0322     struct ppc440spe_adma_desc_slot *iter;
0323     u8 dopc;
0324
0325     /* Common initialization of a PQ descriptors chain */
0326     set_bits(op, &desc->flags);
0327     desc->src_cnt = src_cnt;
0328     desc->dst_cnt = dst_cnt;
0329
0330     /* WXOR MULTICAST if both P and Q are being computed
0331      * MV_SG1_SG2 if Q only
0332      */
0333     dopc = (desc->dst_cnt == DMA_DEST_MAX_NUM) ?
0334         DMA_CDB_OPC_MULTICAST : DMA_CDB_OPC_MV_SG1_SG2;
0335
0336     list_for_each_entry(iter, &desc->group_list, chain_node) {
0337         hw_desc = iter->hw_desc;
0338         memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
0339
0340         if (likely(!list_is_last(&iter->chain_node,
0341                 &desc->group_list))) {
0342             /* set 'next' pointer */
0343             iter->hw_next = list_entry(iter->chain_node.next,
0344                 struct ppc440spe_adma_desc_slot, chain_node);
0345             clear_bit(PPC440SPE_DESC_INT, &iter->flags);
0346         } else {
0347             /* this is the last descriptor.
0348              * this slot will be pasted from ADMA level
0349              * each time it wants to configure parameters
0350              * of the transaction (src, dst, ...)
0351              */
0352             iter->hw_next = NULL;
0353             if (flags & DMA_PREP_INTERRUPT)
0354                 set_bit(PPC440SPE_DESC_INT, &iter->flags);
0355             else
0356                 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
0357         }
0358     }
0359
0360     /* Set OPS depending on WXOR/RXOR type of operation */
0361     if (!test_bit(PPC440SPE_DESC_RXOR, &desc->flags)) {
0362         /* This is a WXOR only chain:
0363          * - first descriptors are for zeroing destinations
0364          *   if PPC440SPE_ZERO_P/Q set;
0365          * - descriptors remained are for GF-XOR operations.
0366          */
0367         iter = list_first_entry(&desc->group_list,
0368                     struct ppc440spe_adma_desc_slot,
0369                     chain_node);
0370
0371         if (test_bit(PPC440SPE_ZERO_P, &desc->flags)) {
0372             hw_desc = iter->hw_desc;
0373             hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
0374             iter = list_first_entry(&iter->chain_node,
0375                     struct ppc440spe_adma_desc_slot,
0376                     chain_node);
0377         }
0378
0379         if (test_bit(PPC440SPE_ZERO_Q, &desc->flags)) {
0380             hw_desc = iter->hw_desc;
0381             hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
0382             iter = list_first_entry(&iter->chain_node,
0383                     struct ppc440spe_adma_desc_slot,
0384                     chain_node);
0385         }
0386
0387         list_for_each_entry_from(iter, &desc->group_list, chain_node) {
0388             hw_desc = iter->hw_desc;
0389             hw_desc->opc = dopc;
0390         }
0391     } else {
0392         /* This is either RXOR-only or mixed RXOR/WXOR */
0393
0394         /* The first 1 or 2 slots in chain are always RXOR,
0395          * if need to calculate P & Q, then there are two
0396          * RXOR slots; if only P or only Q, then there is one
0397          */
0398         iter = list_first_entry(&desc->group_list,
0399                     struct ppc440spe_adma_desc_slot,
0400                     chain_node);
0401         hw_desc = iter->hw_desc;
0402         hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
0403
0404         if (desc->dst_cnt == DMA_DEST_MAX_NUM) {
0405             iter = list_first_entry(&iter->chain_node,
0406                         struct ppc440spe_adma_desc_slot,
0407                         chain_node);
0408             hw_desc = iter->hw_desc;
0409             hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
0410         }
0411
0412         /* The remaining descs (if any) are WXORs */
0413         if (test_bit(PPC440SPE_DESC_WXOR, &desc->flags)) {
0414             iter = list_first_entry(&iter->chain_node,
0415                         struct ppc440spe_adma_desc_slot,
0416                         chain_node);
0417             list_for_each_entry_from(iter, &desc->group_list,
0418                         chain_node) {
0419                 hw_desc = iter->hw_desc;
0420                 hw_desc->opc = dopc;
0421             }
0422         }
0423     }
0424 }
0425
0426 /**
0427  * ppc440spe_desc_init_dma01pqzero_sum - initialize the descriptor
0428  * for PQ_ZERO_SUM operation
0429  */
0430 static void ppc440spe_desc_init_dma01pqzero_sum(
0431                 struct ppc440spe_adma_desc_slot *desc,
0432                 int dst_cnt, int src_cnt)
0433 {
0434     struct dma_cdb *hw_desc;
0435     struct ppc440spe_adma_desc_slot *iter;
0436     int i = 0;
0437     u8 dopc = (dst_cnt == 2) ? DMA_CDB_OPC_MULTICAST :
0438                    DMA_CDB_OPC_MV_SG1_SG2;
0439     /*
0440      * Initialize starting from 2nd or 3rd descriptor dependent
0441      * on dst_cnt. First one or two slots are for cloning P
0442      * and/or Q to chan->pdest and/or chan->qdest as we have
0443      * to preserve original P/Q.
0444      */
0445     iter = list_first_entry(&desc->group_list,
0446                 struct ppc440spe_adma_desc_slot, chain_node);
0447     iter = list_entry(iter->chain_node.next,
0448               struct ppc440spe_adma_desc_slot, chain_node);
0449
0450     if (dst_cnt > 1) {
0451         iter = list_entry(iter->chain_node.next,
0452                   struct ppc440spe_adma_desc_slot, chain_node);
0453     }
0454     /* initialize each source descriptor in chain */
0455     list_for_each_entry_from(iter, &desc->group_list, chain_node) {
0456         hw_desc = iter->hw_desc;
0457         memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
0458         iter->src_cnt = 0;
0459         iter->dst_cnt = 0;
0460
0461         /* This is a ZERO_SUM operation:
0462          * - <src_cnt> descriptors starting from 2nd or 3rd
0463          *   descriptor are for GF-XOR operations;
0464          * - remaining <dst_cnt> descriptors are for checking the result
0465          */
0466         if (i++ < src_cnt)
0467             /* MV_SG1_SG2 if only Q is being verified
0468              * MULTICAST if both P and Q are being verified
0469              */
0470             hw_desc->opc = dopc;
0471         else
0472             /* DMA_CDB_OPC_DCHECK128 operation */
0473             hw_desc->opc = DMA_CDB_OPC_DCHECK128;
0474
0475         if (likely(!list_is_last(&iter->chain_node,
0476                      &desc->group_list))) {
0477             /* set 'next' pointer */
0478             iter->hw_next = list_entry(iter->chain_node.next,
0479                         struct ppc440spe_adma_desc_slot,
0480                         chain_node);
0481         } else {
0482             /* this is the last descriptor.
0483              * this slot will be pasted from ADMA level
0484              * each time it wants to configure parameters
0485              * of the transaction (src, dst, ...)
0486              */
0487             iter->hw_next = NULL;
0488             /* always enable interrupt generation since we get
0489              * the status of pqzero from the handler
0490              */
0491             set_bit(PPC440SPE_DESC_INT, &iter->flags);
0492         }
0493     }
0494     desc->src_cnt = src_cnt;
0495     desc->dst_cnt = dst_cnt;
0496 }
0497
0498 /**
0499  * ppc440spe_desc_init_memcpy - initialize the descriptor for MEMCPY operation
0500  */
0501 static void ppc440spe_desc_init_memcpy(struct ppc440spe_adma_desc_slot *desc,
0502                     unsigned long flags)
0503 {
0504     struct dma_cdb *hw_desc = desc->hw_desc;
0505
0506     memset(desc->hw_desc, 0, sizeof(struct dma_cdb));
0507     desc->hw_next = NULL;
0508     desc->src_cnt = 1;
0509     desc->dst_cnt = 1;
0510
0511     if (flags & DMA_PREP_INTERRUPT)
0512         set_bit(PPC440SPE_DESC_INT, &desc->flags);
0513     else
0514         clear_bit(PPC440SPE_DESC_INT, &desc->flags);
0515
0516     hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
0517 }
0518
0519 /**
0520  * ppc440spe_desc_set_src_addr - set source address into the descriptor
0521  */
0522 static void ppc440spe_desc_set_src_addr(struct ppc440spe_adma_desc_slot *desc,
0523                     struct ppc440spe_adma_chan *chan,
0524                     int src_idx, dma_addr_t addrh,
0525                     dma_addr_t addrl)
0526 {
0527     struct dma_cdb *dma_hw_desc;
0528     struct xor_cb *xor_hw_desc;
0529     phys_addr_t addr64, tmplow, tmphi;
0530
0531     switch (chan->device->id) {
0532     case PPC440SPE_DMA0_ID:
0533     case PPC440SPE_DMA1_ID:
0534         if (!addrh) {
0535             addr64 = addrl;
0536             tmphi = (addr64 >> 32);
0537             tmplow = (addr64 & 0xFFFFFFFF);
0538         } else {
0539             tmphi = addrh;
0540             tmplow = addrl;
0541         }
0542         dma_hw_desc = desc->hw_desc;
0543         dma_hw_desc->sg1l = cpu_to_le32((u32)tmplow);
0544         dma_hw_desc->sg1u |= cpu_to_le32((u32)tmphi);
0545         break;
0546     case PPC440SPE_XOR_ID:
0547         xor_hw_desc = desc->hw_desc;
0548         xor_hw_desc->ops[src_idx].l = addrl;
0549         xor_hw_desc->ops[src_idx].h |= addrh;
0550         break;
0551     }
0552 }
0553
0554 /**
0555  * ppc440spe_desc_set_src_mult - set source address mult into the descriptor
0556  */
0557 static void ppc440spe_desc_set_src_mult(struct ppc440spe_adma_desc_slot *desc,
0558             struct ppc440spe_adma_chan *chan, u32 mult_index,
0559             int sg_index, unsigned char mult_value)
0560 {
0561     struct dma_cdb *dma_hw_desc;
0562     u32 *psgu;
0563
0564     switch (chan->device->id) {
0565     case PPC440SPE_DMA0_ID:
0566     case PPC440SPE_DMA1_ID:
0567         dma_hw_desc = desc->hw_desc;
0568
0569         switch (sg_index) {
0570         /* for RXOR operations set multiplier
0571          * into source cued address
0572          */
0573         case DMA_CDB_SG_SRC:
0574             psgu = &dma_hw_desc->sg1u;
0575             break;
0576         /* for WXOR operations set multiplier
0577          * into destination cued address(es)
0578          */
0579         case DMA_CDB_SG_DST1:
0580             psgu = &dma_hw_desc->sg2u;
0581             break;
0582         case DMA_CDB_SG_DST2:
0583             psgu = &dma_hw_desc->sg3u;
0584             break;
0585         default:
0586             BUG();
0587         }
0588
0589         *psgu |= cpu_to_le32(mult_value << mult_index);
0590         break;
0591     case PPC440SPE_XOR_ID:
0592         break;
0593     default:
0594         BUG();
0595     }
0596 }
0597
0598 /**
0599  * ppc440spe_desc_set_dest_addr - set destination address into the descriptor
0600  */
0601 static void ppc440spe_desc_set_dest_addr(struct ppc440spe_adma_desc_slot *desc,
0602                 struct ppc440spe_adma_chan *chan,
0603                 dma_addr_t addrh, dma_addr_t addrl,
0604                 u32 dst_idx)
0605 {
0606     struct dma_cdb *dma_hw_desc;
0607     struct xor_cb *xor_hw_desc;
0608     phys_addr_t addr64, tmphi, tmplow;
0609     u32 *psgu, *psgl;
0610
0611     switch (chan->device->id) {
0612     case PPC440SPE_DMA0_ID:
0613     case PPC440SPE_DMA1_ID:
0614         if (!addrh) {
0615             addr64 = addrl;
0616             tmphi = (addr64 >> 32);
0617             tmplow = (addr64 & 0xFFFFFFFF);
0618         } else {
0619             tmphi = addrh;
0620             tmplow = addrl;
0621         }
0622         dma_hw_desc = desc->hw_desc;
0623
0624         psgu = dst_idx ? &dma_hw_desc->sg3u : &dma_hw_desc->sg2u;
0625         psgl = dst_idx ? &dma_hw_desc->sg3l : &dma_hw_desc->sg2l;
0626
0627         *psgl = cpu_to_le32((u32)tmplow);
0628         *psgu |= cpu_to_le32((u32)tmphi);
0629         break;
0630     case PPC440SPE_XOR_ID:
0631         xor_hw_desc = desc->hw_desc;
0632         xor_hw_desc->cbtal = addrl;
0633         xor_hw_desc->cbtah |= addrh;
0634         break;
0635     }
0636 }
0637
0638 /**
0639  * ppc440spe_desc_set_byte_count - set number of data bytes involved
0640  * into the operation
0641  */
0642 static void ppc440spe_desc_set_byte_count(struct ppc440spe_adma_desc_slot *desc,
0643                 struct ppc440spe_adma_chan *chan,
0644                 u32 byte_count)
0645 {
0646     struct dma_cdb *dma_hw_desc;
0647     struct xor_cb *xor_hw_desc;
0648
0649     switch (chan->device->id) {
0650     case PPC440SPE_DMA0_ID:
0651     case PPC440SPE_DMA1_ID:
0652         dma_hw_desc = desc->hw_desc;
0653         dma_hw_desc->cnt = cpu_to_le32(byte_count);
0654         break;
0655     case PPC440SPE_XOR_ID:
0656         xor_hw_desc = desc->hw_desc;
0657         xor_hw_desc->cbbc = byte_count;
0658         break;
0659     }
0660 }
0661
0662 /**
0663  * ppc440spe_desc_set_rxor_block_size - set RXOR block size
0664  */
0665 static inline void ppc440spe_desc_set_rxor_block_size(u32 byte_count)
0666 {
0667     /* assume that byte_count is aligned on the 512-boundary;
0668      * thus write it directly to the register (bits 23:31 are
0669      * reserved there).
0670      */
0671     dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CF2H, byte_count);
0672 }
0673
0674 /**
0675  * ppc440spe_desc_set_dcheck - set CHECK pattern
0676  */
0677 static void ppc440spe_desc_set_dcheck(struct ppc440spe_adma_desc_slot *desc,
0678                 struct ppc440spe_adma_chan *chan, u8 *qword)
0679 {
0680     struct dma_cdb *dma_hw_desc;
0681
0682     switch (chan->device->id) {
0683     case PPC440SPE_DMA0_ID:
0684     case PPC440SPE_DMA1_ID:
0685         dma_hw_desc = desc->hw_desc;
0686         iowrite32(qword[0], &dma_hw_desc->sg3l);
0687         iowrite32(qword[4], &dma_hw_desc->sg3u);
0688         iowrite32(qword[8], &dma_hw_desc->sg2l);
0689         iowrite32(qword[12], &dma_hw_desc->sg2u);
0690         break;
0691     default:
0692         BUG();
0693     }
0694 }
0695
0696 /**
0697  * ppc440spe_xor_set_link - set link address in xor CB
0698  */
0699 static void ppc440spe_xor_set_link(struct ppc440spe_adma_desc_slot *prev_desc,
0700                 struct ppc440spe_adma_desc_slot *next_desc)
0701 {
0702     struct xor_cb *xor_hw_desc = prev_desc->hw_desc;
0703
0704     if (unlikely(!next_desc || !(next_desc->phys))) {
0705         printk(KERN_ERR "%s: next_desc=0x%p; next_desc->phys=0x%llx\n",
0706             __func__, next_desc,
0707             next_desc ? next_desc->phys : 0);
0708         BUG();
0709     }
0710
0711     xor_hw_desc->cbs = 0;
0712     xor_hw_desc->cblal = next_desc->phys;
0713     xor_hw_desc->cblah = 0;
0714     xor_hw_desc->cbc |= XOR_CBCR_LNK_BIT;
0715 }
0716
0717 /**
0718  * ppc440spe_desc_set_link - set the address of descriptor following this
0719  * descriptor in chain
0720  */
0721 static void ppc440spe_desc_set_link(struct ppc440spe_adma_chan *chan,
0722                 struct ppc440spe_adma_desc_slot *prev_desc,
0723                 struct ppc440spe_adma_desc_slot *next_desc)
0724 {
0725     unsigned long flags;
0726     struct ppc440spe_adma_desc_slot *tail = next_desc;
0727
0728     if (unlikely(!prev_desc || !next_desc ||
0729         (prev_desc->hw_next && prev_desc->hw_next != next_desc))) {
0730         /* If previous next is overwritten something is wrong.
0731          * though we may refetch from append to initiate list
0732          * processing; in this case - it's ok.
0733          */
0734         printk(KERN_ERR "%s: prev_desc=0x%p; next_desc=0x%p; "
0735             "prev->hw_next=0x%p\n", __func__, prev_desc,
0736             next_desc, prev_desc ? prev_desc->hw_next : 0);
0737         BUG();
0738     }
0739
0740     local_irq_save(flags);
0741
0742     /* do s/w chaining both for DMA and XOR descriptors */
0743     prev_desc->hw_next = next_desc;
0744
0745     switch (chan->device->id) {
0746     case PPC440SPE_DMA0_ID:
0747     case PPC440SPE_DMA1_ID:
0748         break;
0749     case PPC440SPE_XOR_ID:
0750         /* bind descriptor to the chain */
0751         while (tail->hw_next)
0752             tail = tail->hw_next;
0753         xor_last_linked = tail;
0754
0755         if (prev_desc == xor_last_submit)
0756             /* do not link to the last submitted CB */
0757             break;
0758         ppc440spe_xor_set_link(prev_desc, next_desc);
0759         break;
0760     }
0761
0762     local_irq_restore(flags);
0763 }
0764
0765 /**
0766  * ppc440spe_desc_get_link - get the address of the descriptor that
0767  * follows this one
0768  */
0769 static inline u32 ppc440spe_desc_get_link(struct ppc440spe_adma_desc_slot *desc,
0770                     struct ppc440spe_adma_chan *chan)
0771 {
0772     if (!desc->hw_next)
0773         return 0;
0774
0775     return desc->hw_next->phys;
0776 }
0777
0778 /**
0779  * ppc440spe_desc_is_aligned - check alignment
0780  */
0781 static inline int ppc440spe_desc_is_aligned(
0782     struct ppc440spe_adma_desc_slot *desc, int num_slots)
0783 {
0784     return (desc->idx & (num_slots - 1)) ? 0 : 1;
0785 }
0786
0787 /**
0788  * ppc440spe_chan_xor_slot_count - get the number of slots necessary for
0789  * XOR operation
0790  */
0791 static int ppc440spe_chan_xor_slot_count(size_t len, int src_cnt,
0792             int *slots_per_op)
0793 {
0794     int slot_cnt;
0795
0796     /* each XOR descriptor provides up to 16 source operands */
0797     slot_cnt = *slots_per_op = (src_cnt + XOR_MAX_OPS - 1)/XOR_MAX_OPS;
0798
0799     if (likely(len <= PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT))
0800         return slot_cnt;
0801
0802     printk(KERN_ERR "%s: len %d > max %d !!\n",
0803         __func__, len, PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT);
0804     BUG();
0805     return slot_cnt;
0806 }
0807
0808 /**
0809  * ppc440spe_dma2_pq_slot_count - get the number of slots necessary for
0810  * DMA2 PQ operation
0811  */
0812 static int ppc440spe_dma2_pq_slot_count(dma_addr_t *srcs,
0813         int src_cnt, size_t len)
0814 {
0815     signed long long order = 0;
0816     int state = 0;
0817     int addr_count = 0;
0818     int i;
0819     for (i = 1; i < src_cnt; i++) {
0820         dma_addr_t cur_addr = srcs[i];
0821         dma_addr_t old_addr = srcs[i-1];
0822         switch (state) {
0823         case 0:
0824             if (cur_addr == old_addr + len) {
0825                 /* direct RXOR */
0826                 order = 1;
0827                 state = 1;
0828                 if (i == src_cnt-1)
0829                     addr_count++;
0830             } else if (old_addr == cur_addr + len) {
0831                 /* reverse RXOR */
0832                 order = -1;
0833                 state = 1;
0834                 if (i == src_cnt-1)
0835                     addr_count++;
0836             } else {
0837                 state = 3;
0838             }
0839             break;
0840         case 1:
0841             if (i == src_cnt-2 || (order == -1
0842                 && cur_addr != old_addr - len)) {
0843                 order = 0;
0844                 state = 0;
0845                 addr_count++;
0846             } else if (cur_addr == old_addr + len*order) {
0847                 state = 2;
0848                 if (i == src_cnt-1)
0849                     addr_count++;
0850             } else if (cur_addr == old_addr + 2*len) {
0851                 state = 2;
0852                 if (i == src_cnt-1)
0853                     addr_count++;
0854             } else if (cur_addr == old_addr + 3*len) {
0855                 state = 2;
0856                 if (i == src_cnt-1)
0857                     addr_count++;
0858             } else {
0859                 order = 0;
0860                 state = 0;
0861                 addr_count++;
0862             }
0863             break;
0864         case 2:
0865             order = 0;
0866             state = 0;
0867             addr_count++;
0868                 break;
0869         }
0870         if (state == 3)
0871             break;
0872     }
0873     if (src_cnt <= 1 || (state != 1 && state != 2)) {
0874         pr_err("%s: src_cnt=%d, state=%d, addr_count=%d, order=%lld\n",
0875             __func__, src_cnt, state, addr_count, order);
0876         for (i = 0; i < src_cnt; i++)
0877             pr_err("\t[%d] 0x%llx \n", i, srcs[i]);
0878         BUG();
0879     }
0880
0881     return (addr_count + XOR_MAX_OPS - 1) / XOR_MAX_OPS;
0882 }
0883
0884
0885 /******************************************************************************
0886  * ADMA channel low-level routines
0887  ******************************************************************************/
0888
0889 static u32
0890 ppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan);
0891 static void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan);
0892
0893 /**
0894  * ppc440spe_adma_device_clear_eot_status - interrupt ack to XOR or DMA engine
0895  */
0896 static void ppc440spe_adma_device_clear_eot_status(
0897                     struct ppc440spe_adma_chan *chan)
0898 {
0899     struct dma_regs *dma_reg;
0900     struct xor_regs *xor_reg;
0901     u8 *p = chan->device->dma_desc_pool_virt;
0902     struct dma_cdb *cdb;
0903     u32 rv, i;
0904
0905     switch (chan->device->id) {
0906     case PPC440SPE_DMA0_ID:
0907     case PPC440SPE_DMA1_ID:
0908         /* read FIFO to ack */
0909         dma_reg = chan->device->dma_reg;
0910         while ((rv = ioread32(&dma_reg->csfpl))) {
0911             i = rv & DMA_CDB_ADDR_MSK;
0912             cdb = (struct dma_cdb *)&p[i -
0913                 (u32)chan->device->dma_desc_pool];
0914
0915             /* Clear opcode to ack. This is necessary for
0916              * ZeroSum operations only
0917              */
0918             cdb->opc = 0;
0919
0920             if (test_bit(PPC440SPE_RXOR_RUN,
0921                 &ppc440spe_rxor_state)) {
0922                 /* probably this is a completed RXOR op,
0923                  * get pointer to CDB using the fact that
0924                  * physical and virtual addresses of CDB
0925                  * in pools have the same offsets
0926                  */
0927                 if (le32_to_cpu(cdb->sg1u) &
0928                     DMA_CUED_XOR_BASE) {
0929                     /* this is a RXOR */
0930                     clear_bit(PPC440SPE_RXOR_RUN,
0931                           &ppc440spe_rxor_state);
0932                 }
0933             }
0934
0935             if (rv & DMA_CDB_STATUS_MSK) {
0936                 /* ZeroSum check failed
0937                  */
0938                 struct ppc440spe_adma_desc_slot *iter;
0939                 dma_addr_t phys = rv & ~DMA_CDB_MSK;
0940
0941                 /*
0942                  * Update the status of corresponding
0943                  * descriptor.
0944                  */
0945                 list_for_each_entry(iter, &chan->chain,
0946                     chain_node) {
0947                     if (iter->phys == phys)
0948                         break;
0949                 }
0950                 /*
0951                  * if cannot find the corresponding
0952                  * slot it's a bug
0953                  */
0954                 BUG_ON(&iter->chain_node == &chan->chain);
0955
0956                 if (iter->xor_check_result) {
0957                     if (test_bit(PPC440SPE_DESC_PCHECK,
0958                              &iter->flags)) {
0959                         *iter->xor_check_result |=
0960                             SUM_CHECK_P_RESULT;
0961                     } else
0962                     if (test_bit(PPC440SPE_DESC_QCHECK,
0963                              &iter->flags)) {
0964                         *iter->xor_check_result |=
0965                             SUM_CHECK_Q_RESULT;
0966                     } else
0967                         BUG();
0968                 }
0969             }
0970         }
0971
0972         rv = ioread32(&dma_reg->dsts);
0973         if (rv) {
0974             pr_err("DMA%d err status: 0x%x\n",
0975                    chan->device->id, rv);
0976             /* write back to clear */
0977             iowrite32(rv, &dma_reg->dsts);
0978         }
0979         break;
0980     case PPC440SPE_XOR_ID:
0981         /* reset status bits to ack */
0982         xor_reg = chan->device->xor_reg;
0983         rv = ioread32be(&xor_reg->sr);
0984         iowrite32be(rv, &xor_reg->sr);
0985
0986         if (rv & (XOR_IE_ICBIE_BIT|XOR_IE_ICIE_BIT|XOR_IE_RPTIE_BIT)) {
0987             if (rv & XOR_IE_RPTIE_BIT) {
0988                 /* Read PLB Timeout Error.
0989                  * Try to resubmit the CB
0990                  */
0991                 u32 val = ioread32be(&xor_reg->ccbalr);
0992
0993                 iowrite32be(val, &xor_reg->cblalr);
0994
0995                 val = ioread32be(&xor_reg->crsr);
0996                 iowrite32be(val | XOR_CRSR_XAE_BIT,
0997                         &xor_reg->crsr);
0998             } else
0999                 pr_err("XOR ERR 0x%x status\n", rv);
1000             break;
1001         }
1002
1003         /*  if the XORcore is idle, but there are unprocessed CBs
1004          * then refetch the s/w chain here
1005          */
1006         if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) &&
1007             do_xor_refetch)
1008             ppc440spe_chan_append(chan);
1009         break;
1010     }
1011 }
1012
1013 /**
1014  * ppc440spe_chan_is_busy - get the channel status
1015  */
1016 static int ppc440spe_chan_is_busy(struct ppc440spe_adma_chan *chan)
1017 {
1018     struct dma_regs *dma_reg;
1019     struct xor_regs *xor_reg;
1020     int busy = 0;
1021
1022     switch (chan->device->id) {
1023     case PPC440SPE_DMA0_ID:
1024     case PPC440SPE_DMA1_ID:
1025         dma_reg = chan->device->dma_reg;
1026         /*  if command FIFO's head and tail pointers are equal and
1027          * status tail is the same as command, then channel is free
1028          */
1029         if (ioread16(&dma_reg->cpfhp) != ioread16(&dma_reg->cpftp) ||
1030             ioread16(&dma_reg->cpftp) != ioread16(&dma_reg->csftp))
1031             busy = 1;
1032         break;
1033     case PPC440SPE_XOR_ID:
1034         /* use the special status bit for the XORcore
1035          */
1036         xor_reg = chan->device->xor_reg;
1037         busy = (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) ? 1 : 0;
1038         break;
1039     }
1040
1041     return busy;
1042 }
1043
1044 /**
1045  * ppc440spe_chan_set_first_xor_descriptor -  init XORcore chain
1046  */
1047 static void ppc440spe_chan_set_first_xor_descriptor(
1048                 struct ppc440spe_adma_chan *chan,
1049                 struct ppc440spe_adma_desc_slot *next_desc)
1050 {
1051     struct xor_regs *xor_reg = chan->device->xor_reg;
1052
1053     if (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT)
1054         printk(KERN_INFO "%s: Warn: XORcore is running "
1055             "when try to set the first CDB!\n",
1056             __func__);
1057
1058     xor_last_submit = xor_last_linked = next_desc;
1059
1060     iowrite32be(XOR_CRSR_64BA_BIT, &xor_reg->crsr);
1061
1062     iowrite32be(next_desc->phys, &xor_reg->cblalr);
1063     iowrite32be(0, &xor_reg->cblahr);
1064     iowrite32be(ioread32be(&xor_reg->cbcr) | XOR_CBCR_LNK_BIT,
1065             &xor_reg->cbcr);
1066
1067     chan->hw_chain_inited = 1;
1068 }
1069
1070 /**
1071  * ppc440spe_dma_put_desc - put DMA0,1 descriptor to FIFO.
1072  * called with irqs disabled
1073  */
1074 static void ppc440spe_dma_put_desc(struct ppc440spe_adma_chan *chan,
1075         struct ppc440spe_adma_desc_slot *desc)
1076 {
1077     u32 pcdb;
1078     struct dma_regs *dma_reg = chan->device->dma_reg;
1079
1080     pcdb = desc->phys;
1081     if (!test_bit(PPC440SPE_DESC_INT, &desc->flags))
1082         pcdb |= DMA_CDB_NO_INT;
1083
1084     chan_last_sub[chan->device->id] = desc;
1085
1086     ADMA_LL_DBG(print_cb(chan, desc->hw_desc));
1087
1088     iowrite32(pcdb, &dma_reg->cpfpl);
1089 }
1090
1091 /**
1092  * ppc440spe_chan_append - update the h/w chain in the channel
1093  */
1094 static void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan)
1095 {
1096     struct xor_regs *xor_reg;
1097     struct ppc440spe_adma_desc_slot *iter;
1098     struct xor_cb *xcb;
1099     u32 cur_desc;
1100     unsigned long flags;
1101
1102     local_irq_save(flags);
1103
1104     switch (chan->device->id) {
1105     case PPC440SPE_DMA0_ID:
1106     case PPC440SPE_DMA1_ID:
1107         cur_desc = ppc440spe_chan_get_current_descriptor(chan);
1108
1109         if (likely(cur_desc)) {
1110             iter = chan_last_sub[chan->device->id];
1111             BUG_ON(!iter);
1112         } else {
1113             /* first peer */
1114             iter = chan_first_cdb[chan->device->id];
1115             BUG_ON(!iter);
1116             ppc440spe_dma_put_desc(chan, iter);
1117             chan->hw_chain_inited = 1;
1118         }
1119
1120         /* is there something new to append */
1121         if (!iter->hw_next)
1122             break;
1123
1124         /* flush descriptors from the s/w queue to fifo */
1125         list_for_each_entry_continue(iter, &chan->chain, chain_node) {
1126             ppc440spe_dma_put_desc(chan, iter);
1127             if (!iter->hw_next)
1128                 break;
1129         }
1130         break;
1131     case PPC440SPE_XOR_ID:
1132         /* update h/w links and refetch */
1133         if (!xor_last_submit->hw_next)
1134             break;
1135
1136         xor_reg = chan->device->xor_reg;
1137         /* the last linked CDB has to generate an interrupt
1138          * that we'd be able to append the next lists to h/w
1139          * regardless of the XOR engine state at the moment of
1140          * appending of these next lists
1141          */
1142         xcb = xor_last_linked->hw_desc;
1143         xcb->cbc |= XOR_CBCR_CBCE_BIT;
1144
1145         if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT)) {
1146             /* XORcore is idle. Refetch now */
1147             do_xor_refetch = 0;
1148             ppc440spe_xor_set_link(xor_last_submit,
1149                 xor_last_submit->hw_next);
1150
1151             ADMA_LL_DBG(print_cb_list(chan,
1152                 xor_last_submit->hw_next));
1153
1154             xor_last_submit = xor_last_linked;
1155             iowrite32be(ioread32be(&xor_reg->crsr) |
1156                     XOR_CRSR_RCBE_BIT | XOR_CRSR_64BA_BIT,
1157                     &xor_reg->crsr);
1158         } else {
1159             /* XORcore is running. Refetch later in the handler */
1160             do_xor_refetch = 1;
1161         }
1162
1163         break;
1164     }
1165
1166     local_irq_restore(flags);
1167 }
1168
1169 /**
1170  * ppc440spe_chan_get_current_descriptor - get the currently executed descriptor
1171  */
1172 static u32
1173 ppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan)
1174 {
1175     struct dma_regs *dma_reg;
1176     struct xor_regs *xor_reg;
1177
1178     if (unlikely(!chan->hw_chain_inited))
1179         /* h/w descriptor chain is not initialized yet */
1180         return 0;
1181
1182     switch (chan->device->id) {
1183     case PPC440SPE_DMA0_ID:
1184     case PPC440SPE_DMA1_ID:
1185         dma_reg = chan->device->dma_reg;
1186         return ioread32(&dma_reg->acpl) & (~DMA_CDB_MSK);
1187     case PPC440SPE_XOR_ID:
1188         xor_reg = chan->device->xor_reg;
1189         return ioread32be(&xor_reg->ccbalr);
1190     }
1191     return 0;
1192 }
1193
1194 /**
1195  * ppc440spe_chan_run - enable the channel
1196  */
1197 static void ppc440spe_chan_run(struct ppc440spe_adma_chan *chan)
1198 {
1199     struct xor_regs *xor_reg;
1200
1201     switch (chan->device->id) {
1202     case PPC440SPE_DMA0_ID:
1203     case PPC440SPE_DMA1_ID:
1204         /* DMAs are always enabled, do nothing */
1205         break;
1206     case PPC440SPE_XOR_ID:
1207         /* drain write buffer */
1208         xor_reg = chan->device->xor_reg;
1209
1210         /* fetch descriptor pointed to in <link> */
1211         iowrite32be(XOR_CRSR_64BA_BIT | XOR_CRSR_XAE_BIT,
1212                 &xor_reg->crsr);
1213         break;
1214     }
1215 }
1216
1217 /******************************************************************************
1218  * ADMA device level
1219  ******************************************************************************/
1220
1221 static void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan);
1222 static int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan);
1223
1224 static dma_cookie_t
1225 ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx);
1226
1227 static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *tx,
1228                     dma_addr_t addr, int index);
1229 static void
1230 ppc440spe_adma_memcpy_xor_set_src(struct ppc440spe_adma_desc_slot *tx,
1231                   dma_addr_t addr, int index);
1232
1233 static void
1234 ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *tx,
1235                dma_addr_t *paddr, unsigned long flags);
1236 static void
1237 ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *tx,
1238               dma_addr_t addr, int index);
1239 static void
1240 ppc440spe_adma_pq_set_src_mult(struct ppc440spe_adma_desc_slot *tx,
1241                    unsigned char mult, int index, int dst_pos);
1242 static void
1243 ppc440spe_adma_pqzero_sum_set_dest(struct ppc440spe_adma_desc_slot *tx,
1244                    dma_addr_t paddr, dma_addr_t qaddr);
1245
1246 static struct page *ppc440spe_rxor_srcs[32];
1247
1248 /**
1249  * ppc440spe_can_rxor - check if the operands may be processed with RXOR
1250  */
1251 static int ppc440spe_can_rxor(struct page **srcs, int src_cnt, size_t len)
1252 {
1253     int i, order = 0, state = 0;
1254     int idx = 0;
1255
1256     if (unlikely(!(src_cnt > 1)))
1257         return 0;
1258
1259     BUG_ON(src_cnt > ARRAY_SIZE(ppc440spe_rxor_srcs));
1260
1261     /* Skip holes in the source list before checking */
1262     for (i = 0; i < src_cnt; i++) {
1263         if (!srcs[i])
1264             continue;
1265         ppc440spe_rxor_srcs[idx++] = srcs[i];
1266     }
1267     src_cnt = idx;
1268
1269     for (i = 1; i < src_cnt; i++) {
1270         char *cur_addr = page_address(ppc440spe_rxor_srcs[i]);
1271         char *old_addr = page_address(ppc440spe_rxor_srcs[i - 1]);
1272
1273         switch (state) {
1274         case 0:
1275             if (cur_addr == old_addr + len) {
1276                 /* direct RXOR */
1277                 order = 1;
1278                 state = 1;
1279             } else if (old_addr == cur_addr + len) {
1280                 /* reverse RXOR */
1281                 order = -1;
1282                 state = 1;
1283             } else
1284                 goto out;
1285             break;
1286         case 1:
1287             if ((i == src_cnt - 2) ||
1288                 (order == -1 && cur_addr != old_addr - len)) {
1289                 order = 0;
1290                 state = 0;
1291             } else if ((cur_addr == old_addr + len * order) ||
1292                    (cur_addr == old_addr + 2 * len) ||
1293                    (cur_addr == old_addr + 3 * len)) {
1294                 state = 2;
1295             } else {
1296                 order = 0;
1297                 state = 0;
1298             }
1299             break;
1300         case 2:
1301             order = 0;
1302             state = 0;
1303             break;
1304         }
1305     }
1306
1307 out:
1308     if (state == 1 || state == 2)
1309         return 1;
1310
1311     return 0;
1312 }
1313
1314 /**
1315  * ppc440spe_adma_device_estimate - estimate the efficiency of processing
1316  *  the operation given on this channel. It's assumed that 'chan' is
1317  *  capable to process 'cap' type of operation.
1318  * @chan: channel to use
1319  * @cap: type of transaction
1320  * @dst_lst: array of destination pointers
1321  * @dst_cnt: number of destination operands
1322  * @src_lst: array of source pointers
1323  * @src_cnt: number of source operands
1324  * @src_sz: size of each source operand
1325  */
1326 static int ppc440spe_adma_estimate(struct dma_chan *chan,
1327     enum dma_transaction_type cap, struct page **dst_lst, int dst_cnt,
1328     struct page **src_lst, int src_cnt, size_t src_sz)
1329 {
1330     int ef = 1;
1331
1332     if (cap == DMA_PQ || cap == DMA_PQ_VAL) {
1333         /* If RAID-6 capabilities were not activated don't try
1334          * to use them
1335          */
1336         if (unlikely(!ppc440spe_r6_enabled))
1337             return -1;
1338     }
1339     /*  In the current implementation of ppc440spe ADMA driver it
1340      * makes sense to pick out only pq case, because it may be
1341      * processed:
1342      * (1) either using Biskup method on DMA2;
1343      * (2) or on DMA0/1.
1344      *  Thus we give a favour to (1) if the sources are suitable;
1345      * else let it be processed on one of the DMA0/1 engines.
1346      *  In the sum_product case where destination is also the
1347      * source process it on DMA0/1 only.
1348      */
1349     if (cap == DMA_PQ && chan->chan_id == PPC440SPE_XOR_ID) {
1350
1351         if (dst_cnt == 1 && src_cnt == 2 && dst_lst[0] == src_lst[1])
1352             ef = 0; /* sum_product case, process on DMA0/1 */
1353         else if (ppc440spe_can_rxor(src_lst, src_cnt, src_sz))
1354             ef = 3; /* override (DMA0/1 + idle) */
1355         else
1356             ef = 0; /* can't process on DMA2 if !rxor */
1357     }
1358
1359     /* channel idleness increases the priority */
1360     if (likely(ef) &&
1361         !ppc440spe_chan_is_busy(to_ppc440spe_adma_chan(chan)))
1362         ef++;
1363
1364     return ef;
1365 }
1366
1367 struct dma_chan *
1368 ppc440spe_async_tx_find_best_channel(enum dma_transaction_type cap,
1369     struct page **dst_lst, int dst_cnt, struct page **src_lst,
1370     int src_cnt, size_t src_sz)
1371 {
1372     struct dma_chan *best_chan = NULL;
1373     struct ppc_dma_chan_ref *ref;
1374     int best_rank = -1;
1375
1376     if (unlikely(!src_sz))
1377         return NULL;
1378     if (src_sz > PAGE_SIZE) {
1379         /*
1380          * should a user of the api ever pass > PAGE_SIZE requests
1381          * we sort out cases where temporary page-sized buffers
1382          * are used.
1383          */
1384         switch (cap) {
1385         case DMA_PQ:
1386             if (src_cnt == 1 && dst_lst[1] == src_lst[0])
1387                 return NULL;
1388             if (src_cnt == 2 && dst_lst[1] == src_lst[1])
1389                 return NULL;
1390             break;
1391         case DMA_PQ_VAL:
1392         case DMA_XOR_VAL:
1393             return NULL;
1394         default:
1395             break;
1396         }
1397     }
1398
1399     list_for_each_entry(ref, &ppc440spe_adma_chan_list, node) {
1400         if (dma_has_cap(cap, ref->chan->device->cap_mask)) {
1401             int rank;
1402
1403             rank = ppc440spe_adma_estimate(ref->chan, cap, dst_lst,
1404                     dst_cnt, src_lst, src_cnt, src_sz);
1405             if (rank > best_rank) {
1406                 best_rank = rank;
1407                 best_chan = ref->chan;
1408             }
1409         }
1410     }
1411
1412     return best_chan;
1413 }
1414 EXPORT_SYMBOL_GPL(ppc440spe_async_tx_find_best_channel);
1415
1416 /**
1417  * ppc440spe_get_group_entry - get group entry with index idx
1418  * @tdesc: is the last allocated slot in the group.
1419  */
1420 static struct ppc440spe_adma_desc_slot *
1421 ppc440spe_get_group_entry(struct ppc440spe_adma_desc_slot *tdesc, u32 entry_idx)
1422 {
1423     struct ppc440spe_adma_desc_slot *iter = tdesc->group_head;
1424     int i = 0;
1425
1426     if (entry_idx < 0 || entry_idx >= (tdesc->src_cnt + tdesc->dst_cnt)) {
1427         printk("%s: entry_idx %d, src_cnt %d, dst_cnt %d\n",
1428             __func__, entry_idx, tdesc->src_cnt, tdesc->dst_cnt);
1429         BUG();
1430     }
1431
1432     list_for_each_entry(iter, &tdesc->group_list, chain_node) {
1433         if (i++ == entry_idx)
1434             break;
1435     }
1436     return iter;
1437 }
1438
1439 /**
1440  * ppc440spe_adma_free_slots - flags descriptor slots for reuse
1441  * @slot: Slot to free
1442  * Caller must hold &ppc440spe_chan->lock while calling this function
1443  */
1444 static void ppc440spe_adma_free_slots(struct ppc440spe_adma_desc_slot *slot,
1445                       struct ppc440spe_adma_chan *chan)
1446 {
1447     int stride = slot->slots_per_op;
1448
1449     while (stride--) {
1450         slot->slots_per_op = 0;
1451         slot = list_entry(slot->slot_node.next,
1452                 struct ppc440spe_adma_desc_slot,
1453                 slot_node);
1454     }
1455 }
1456
1457 /**
1458  * ppc440spe_adma_run_tx_complete_actions - call functions to be called
1459  * upon completion
1460  */
1461 static dma_cookie_t ppc440spe_adma_run_tx_complete_actions(
1462         struct ppc440spe_adma_desc_slot *desc,
1463         struct ppc440spe_adma_chan *chan,
1464         dma_cookie_t cookie)
1465 {
1466     BUG_ON(desc->async_tx.cookie < 0);
1467     if (desc->async_tx.cookie > 0) {
1468         cookie = desc->async_tx.cookie;
1469         desc->async_tx.cookie = 0;
1470
1471         dma_descriptor_unmap(&desc->async_tx);
1472         /* call the callback (must not sleep or submit new
1473          * operations to this channel)
1474          */
1475         dmaengine_desc_get_callback_invoke(&desc->async_tx, NULL);
1476     }
1477
1478     /* run dependent operations */
1479     dma_run_dependencies(&desc->async_tx);
1480
1481     return cookie;
1482 }
1483
1484 /**
1485  * ppc440spe_adma_clean_slot - clean up CDB slot (if ack is set)
1486  */
1487 static int ppc440spe_adma_clean_slot(struct ppc440spe_adma_desc_slot *desc,
1488         struct ppc440spe_adma_chan *chan)
1489 {
1490     /* the client is allowed to attach dependent operations
1491      * until 'ack' is set
1492      */
1493     if (!async_tx_test_ack(&desc->async_tx))
1494         return 0;
1495
1496     /* leave the last descriptor in the chain
1497      * so we can append to it
1498      */
1499     if (list_is_last(&desc->chain_node, &chan->chain) ||
1500         desc->phys == ppc440spe_chan_get_current_descriptor(chan))
1501         return 1;
1502
1503     if (chan->device->id != PPC440SPE_XOR_ID) {
1504         /* our DMA interrupt handler clears opc field of
1505          * each processed descriptor. For all types of
1506          * operations except for ZeroSum we do not actually
1507          * need ack from the interrupt handler. ZeroSum is a
1508          * special case since the result of this operation
1509          * is available from the handler only, so if we see
1510          * such type of descriptor (which is unprocessed yet)
1511          * then leave it in chain.
1512          */
1513         struct dma_cdb *cdb = desc->hw_desc;
1514         if (cdb->opc == DMA_CDB_OPC_DCHECK128)
1515             return 1;
1516     }
1517
1518     dev_dbg(chan->device->common.dev, "\tfree slot %llx: %d stride: %d\n",
1519         desc->phys, desc->idx, desc->slots_per_op);
1520
1521     list_del(&desc->chain_node);
1522     ppc440spe_adma_free_slots(desc, chan);
1523     return 0;
1524 }
1525
1526 /**
1527  * __ppc440spe_adma_slot_cleanup - this is the common clean-up routine
1528  *  which runs through the channel CDBs list until reach the descriptor
1529  *  currently processed. When routine determines that all CDBs of group
1530  *  are completed then corresponding callbacks (if any) are called and slots
1531  *  are freed.
1532  */
1533 static void __ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan)
1534 {
1535     struct ppc440spe_adma_desc_slot *iter, *_iter, *group_start = NULL;
1536     dma_cookie_t cookie = 0;
1537     u32 current_desc = ppc440spe_chan_get_current_descriptor(chan);
1538     int busy = ppc440spe_chan_is_busy(chan);
1539     int seen_current = 0, slot_cnt = 0, slots_per_op = 0;
1540
1541     dev_dbg(chan->device->common.dev, "ppc440spe adma%d: %s\n",
1542         chan->device->id, __func__);
1543
1544     if (!current_desc) {
1545         /*  There were no transactions yet, so
1546          * nothing to clean
1547          */
1548         return;
1549     }
1550
1551     /* free completed slots from the chain starting with
1552      * the oldest descriptor
1553      */
1554     list_for_each_entry_safe(iter, _iter, &chan->chain,
1555                     chain_node) {
1556         dev_dbg(chan->device->common.dev, "\tcookie: %d slot: %d "
1557             "busy: %d this_desc: %#llx next_desc: %#x "
1558             "cur: %#x ack: %d\n",
1559             iter->async_tx.cookie, iter->idx, busy, iter->phys,
1560             ppc440spe_desc_get_link(iter, chan), current_desc,
1561             async_tx_test_ack(&iter->async_tx));
1562         prefetch(_iter);
1563         prefetch(&_iter->async_tx);
1564
1565         /* do not advance past the current descriptor loaded into the
1566          * hardware channel,subsequent descriptors are either in process
1567          * or have not been submitted
1568          */
1569         if (seen_current)
1570             break;
1571
1572         /* stop the search if we reach the current descriptor and the
1573          * channel is busy, or if it appears that the current descriptor
1574          * needs to be re-read (i.e. has been appended to)
1575          */
1576         if (iter->phys == current_desc) {
1577             BUG_ON(seen_current++);
1578             if (busy || ppc440spe_desc_get_link(iter, chan)) {
1579                 /* not all descriptors of the group have
1580                  * been completed; exit.
1581                  */
1582                 break;
1583             }
1584         }
1585
1586         /* detect the start of a group transaction */
1587         if (!slot_cnt && !slots_per_op) {
1588             slot_cnt = iter->slot_cnt;
1589             slots_per_op = iter->slots_per_op;
1590             if (slot_cnt <= slots_per_op) {
1591                 slot_cnt = 0;
1592                 slots_per_op = 0;
1593             }
1594         }
1595
1596         if (slot_cnt) {
1597             if (!group_start)
1598                 group_start = iter;
1599             slot_cnt -= slots_per_op;
1600         }
1601
1602         /* all the members of a group are complete */
1603         if (slots_per_op != 0 && slot_cnt == 0) {
1604             struct ppc440spe_adma_desc_slot *grp_iter, *_grp_iter;
1605             int end_of_chain = 0;
1606
1607             /* clean up the group */
1608             slot_cnt = group_start->slot_cnt;
1609             grp_iter = group_start;
1610             list_for_each_entry_safe_from(grp_iter, _grp_iter,
1611                 &chan->chain, chain_node) {
1612
1613                 cookie = ppc440spe_adma_run_tx_complete_actions(
1614                     grp_iter, chan, cookie);
1615
1616                 slot_cnt -= slots_per_op;
1617                 end_of_chain = ppc440spe_adma_clean_slot(
1618                     grp_iter, chan);
1619                 if (end_of_chain && slot_cnt) {
1620                     /* Should wait for ZeroSum completion */
1621                     if (cookie > 0)
1622                         chan->common.completed_cookie = cookie;
1623                     return;
1624                 }
1625
1626                 if (slot_cnt == 0 || end_of_chain)
1627                     break;
1628             }
1629
1630             /* the group should be complete at this point */
1631             BUG_ON(slot_cnt);
1632
1633             slots_per_op = 0;
1634             group_start = NULL;
1635             if (end_of_chain)
1636                 break;
1637             else
1638                 continue;
1639         } else if (slots_per_op) /* wait for group completion */
1640             continue;
1641
1642         cookie = ppc440spe_adma_run_tx_complete_actions(iter, chan,
1643             cookie);
1644
1645         if (ppc440spe_adma_clean_slot(iter, chan))
1646             break;
1647     }
1648
1649     BUG_ON(!seen_current);
1650
1651     if (cookie > 0) {
1652         chan->common.completed_cookie = cookie;
1653         pr_debug("\tcompleted cookie %d\n", cookie);
1654     }
1655
1656 }
1657
1658 /**
1659  * ppc440spe_adma_tasklet - clean up watch-dog initiator
1660  */
1661 static void ppc440spe_adma_tasklet(struct tasklet_struct *t)
1662 {
1663     struct ppc440spe_adma_chan *chan = from_tasklet(chan, t, irq_tasklet);
1664
1665     spin_lock_nested(&chan->lock, SINGLE_DEPTH_NESTING);
1666     __ppc440spe_adma_slot_cleanup(chan);
1667     spin_unlock(&chan->lock);
1668 }
1669
1670 /**
1671  * ppc440spe_adma_slot_cleanup - clean up scheduled initiator
1672  */
1673 static void ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan)
1674 {
1675     spin_lock_bh(&chan->lock);
1676     __ppc440spe_adma_slot_cleanup(chan);
1677     spin_unlock_bh(&chan->lock);
1678 }
1679
1680 /**
1681  * ppc440spe_adma_alloc_slots - allocate free slots (if any)
1682  */
1683 static struct ppc440spe_adma_desc_slot *ppc440spe_adma_alloc_slots(
1684         struct ppc440spe_adma_chan *chan, int num_slots,
1685         int slots_per_op)
1686 {
1687     struct ppc440spe_adma_desc_slot *iter = NULL, *_iter;
1688     struct ppc440spe_adma_desc_slot *alloc_start = NULL;
1689     int slots_found, retry = 0;
1690     LIST_HEAD(chain);
1691
1692
1693     BUG_ON(!num_slots || !slots_per_op);
1694     /* start search from the last allocated descrtiptor
1695      * if a contiguous allocation can not be found start searching
1696      * from the beginning of the list
1697      */
1698 retry:
1699     slots_found = 0;
1700     if (retry == 0)
1701         iter = chan->last_used;
1702     else
1703         iter = list_entry(&chan->all_slots,
1704                   struct ppc440spe_adma_desc_slot,
1705                   slot_node);
1706     list_for_each_entry_safe_continue(iter, _iter, &chan->all_slots,
1707         slot_node) {
1708         prefetch(_iter);
1709         prefetch(&_iter->async_tx);
1710         if (iter->slots_per_op) {
1711             slots_found = 0;
1712             continue;
1713         }
1714
1715         /* start the allocation if the slot is correctly aligned */
1716         if (!slots_found++)
1717             alloc_start = iter;
1718
1719         if (slots_found == num_slots) {
1720             struct ppc440spe_adma_desc_slot *alloc_tail = NULL;
1721             struct ppc440spe_adma_desc_slot *last_used = NULL;
1722
1723             iter = alloc_start;
1724             while (num_slots) {
1725                 int i;
1726                 /* pre-ack all but the last descriptor */
1727                 if (num_slots != slots_per_op)
1728                     async_tx_ack(&iter->async_tx);
1729
1730                 list_add_tail(&iter->chain_node, &chain);
1731                 alloc_tail = iter;
1732                 iter->async_tx.cookie = 0;
1733                 iter->hw_next = NULL;
1734                 iter->flags = 0;
1735                 iter->slot_cnt = num_slots;
1736                 iter->xor_check_result = NULL;
1737                 for (i = 0; i < slots_per_op; i++) {
1738                     iter->slots_per_op = slots_per_op - i;
1739                     last_used = iter;
1740                     iter = list_entry(iter->slot_node.next,
1741                         struct ppc440spe_adma_desc_slot,
1742                         slot_node);
1743                 }
1744                 num_slots -= slots_per_op;
1745             }
1746             alloc_tail->group_head = alloc_start;
1747             alloc_tail->async_tx.cookie = -EBUSY;
1748             list_splice(&chain, &alloc_tail->group_list);
1749             chan->last_used = last_used;
1750             return alloc_tail;
1751         }
1752     }
1753     if (!retry++)
1754         goto retry;
1755
1756     /* try to free some slots if the allocation fails */
1757     tasklet_schedule(&chan->irq_tasklet);
1758     return NULL;
1759 }
1760
1761 /**
1762  * ppc440spe_adma_alloc_chan_resources -  allocate pools for CDB slots
1763  */
1764 static int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan)
1765 {
1766     struct ppc440spe_adma_chan *ppc440spe_chan;
1767     struct ppc440spe_adma_desc_slot *slot = NULL;
1768     char *hw_desc;
1769     int i, db_sz;
1770     int init;
1771
1772     ppc440spe_chan = to_ppc440spe_adma_chan(chan);
1773     init = ppc440spe_chan->slots_allocated ? 0 : 1;
1774     chan->chan_id = ppc440spe_chan->device->id;
1775
1776     /* Allocate descriptor slots */
1777     i = ppc440spe_chan->slots_allocated;
1778     if (ppc440spe_chan->device->id != PPC440SPE_XOR_ID)
1779         db_sz = sizeof(struct dma_cdb);
1780     else
1781         db_sz = sizeof(struct xor_cb);
1782
1783     for (; i < (ppc440spe_chan->device->pool_size / db_sz); i++) {
1784         slot = kzalloc(sizeof(struct ppc440spe_adma_desc_slot),
1785                    GFP_KERNEL);
1786         if (!slot) {
1787             printk(KERN_INFO "SPE ADMA Channel only initialized"
1788                 " %d descriptor slots", i--);
1789             break;
1790         }
1791
1792         hw_desc = (char *) ppc440spe_chan->device->dma_desc_pool_virt;
1793         slot->hw_desc = (void *) &hw_desc[i * db_sz];
1794         dma_async_tx_descriptor_init(&slot->async_tx, chan);
1795         slot->async_tx.tx_submit = ppc440spe_adma_tx_submit;
1796         INIT_LIST_HEAD(&slot->chain_node);
1797         INIT_LIST_HEAD(&slot->slot_node);
1798         INIT_LIST_HEAD(&slot->group_list);
1799         slot->phys = ppc440spe_chan->device->dma_desc_pool + i * db_sz;
1800         slot->idx = i;
1801
1802         spin_lock_bh(&ppc440spe_chan->lock);
1803         ppc440spe_chan->slots_allocated++;
1804         list_add_tail(&slot->slot_node, &ppc440spe_chan->all_slots);
1805         spin_unlock_bh(&ppc440spe_chan->lock);
1806     }
1807
1808     if (i && !ppc440spe_chan->last_used) {
1809         ppc440spe_chan->last_used =
1810             list_entry(ppc440spe_chan->all_slots.next,
1811                 struct ppc440spe_adma_desc_slot,
1812                 slot_node);
1813     }
1814
1815     dev_dbg(ppc440spe_chan->device->common.dev,
1816         "ppc440spe adma%d: allocated %d descriptor slots\n",
1817         ppc440spe_chan->device->id, i);
1818
1819     /* initialize the channel and the chain with a null operation */
1820     if (init) {
1821         switch (ppc440spe_chan->device->id) {
1822         case PPC440SPE_DMA0_ID:
1823         case PPC440SPE_DMA1_ID:
1824             ppc440spe_chan->hw_chain_inited = 0;
1825             /* Use WXOR for self-testing */
1826             if (!ppc440spe_r6_tchan)
1827                 ppc440spe_r6_tchan = ppc440spe_chan;
1828             break;
1829         case PPC440SPE_XOR_ID:
1830             ppc440spe_chan_start_null_xor(ppc440spe_chan);
1831             break;
1832         default:
1833             BUG();
1834         }
1835         ppc440spe_chan->needs_unmap = 1;
1836     }
1837
1838     return (i > 0) ? i : -ENOMEM;
1839 }
1840
1841 /**
1842  * ppc440spe_rxor_set_region_data -
1843  */
1844 static void ppc440spe_rxor_set_region(struct ppc440spe_adma_desc_slot *desc,
1845     u8 xor_arg_no, u32 mask)
1846 {
1847     struct xor_cb *xcb = desc->hw_desc;
1848
1849     xcb->ops[xor_arg_no].h |= mask;
1850 }
1851
1852 /**
1853  * ppc440spe_rxor_set_src -
1854  */
1855 static void ppc440spe_rxor_set_src(struct ppc440spe_adma_desc_slot *desc,
1856     u8 xor_arg_no, dma_addr_t addr)
1857 {
1858     struct xor_cb *xcb = desc->hw_desc;
1859
1860     xcb->ops[xor_arg_no].h |= DMA_CUED_XOR_BASE;
1861     xcb->ops[xor_arg_no].l = addr;
1862 }
1863
1864 /**
1865  * ppc440spe_rxor_set_mult -
1866  */
1867 static void ppc440spe_rxor_set_mult(struct ppc440spe_adma_desc_slot *desc,
1868     u8 xor_arg_no, u8 idx, u8 mult)
1869 {
1870     struct xor_cb *xcb = desc->hw_desc;
1871
1872     xcb->ops[xor_arg_no].h |= mult << (DMA_CUED_MULT1_OFF + idx * 8);
1873 }
1874
1875 /**
1876  * ppc440spe_adma_check_threshold - append CDBs to h/w chain if threshold
1877  *  has been achieved
1878  */
1879 static void ppc440spe_adma_check_threshold(struct ppc440spe_adma_chan *chan)
1880 {
1881     dev_dbg(chan->device->common.dev, "ppc440spe adma%d: pending: %d\n",
1882         chan->device->id, chan->pending);
1883
1884     if (chan->pending >= PPC440SPE_ADMA_THRESHOLD) {
1885         chan->pending = 0;
1886         ppc440spe_chan_append(chan);
1887     }
1888 }
1889
1890 /**
1891  * ppc440spe_adma_tx_submit - submit new descriptor group to the channel
1892  *  (it's not necessary that descriptors will be submitted to the h/w
1893  *  chains too right now)
1894  */
1895 static dma_cookie_t ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx)
1896 {
1897     struct ppc440spe_adma_desc_slot *sw_desc;
1898     struct ppc440spe_adma_chan *chan = to_ppc440spe_adma_chan(tx->chan);
1899     struct ppc440spe_adma_desc_slot *group_start, *old_chain_tail;
1900     int slot_cnt;
1901     int slots_per_op;
1902     dma_cookie_t cookie;
1903
1904     sw_desc = tx_to_ppc440spe_adma_slot(tx);
1905
1906     group_start = sw_desc->group_head;
1907     slot_cnt = group_start->slot_cnt;
1908     slots_per_op = group_start->slots_per_op;
1909
1910     spin_lock_bh(&chan->lock);
1911     cookie = dma_cookie_assign(tx);
1912
1913     if (unlikely(list_empty(&chan->chain))) {
1914         /* first peer */
1915         list_splice_init(&sw_desc->group_list, &chan->chain);
1916         chan_first_cdb[chan->device->id] = group_start;
1917     } else {
1918         /* isn't first peer, bind CDBs to chain */
1919         old_chain_tail = list_entry(chan->chain.prev,
1920                     struct ppc440spe_adma_desc_slot,
1921                     chain_node);
1922         list_splice_init(&sw_desc->group_list,
1923             &old_chain_tail->chain_node);
1924         /* fix up the hardware chain */
1925         ppc440spe_desc_set_link(chan, old_chain_tail, group_start);
1926     }
1927
1928     /* increment the pending count by the number of operations */
1929     chan->pending += slot_cnt / slots_per_op;
1930     ppc440spe_adma_check_threshold(chan);
1931     spin_unlock_bh(&chan->lock);
1932
1933     dev_dbg(chan->device->common.dev,
1934         "ppc440spe adma%d: %s cookie: %d slot: %d tx %p\n",
1935         chan->device->id, __func__,
1936         sw_desc->async_tx.cookie, sw_desc->idx, sw_desc);
1937
1938     return cookie;
1939 }
1940
1941 /**
1942  * ppc440spe_adma_prep_dma_interrupt - prepare CDB for a pseudo DMA operation
1943  */
1944 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_interrupt(
1945         struct dma_chan *chan, unsigned long flags)
1946 {
1947     struct ppc440spe_adma_chan *ppc440spe_chan;
1948     struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
1949     int slot_cnt, slots_per_op;
1950
1951     ppc440spe_chan = to_ppc440spe_adma_chan(chan);
1952
1953     dev_dbg(ppc440spe_chan->device->common.dev,
1954         "ppc440spe adma%d: %s\n", ppc440spe_chan->device->id,
1955         __func__);
1956
1957     spin_lock_bh(&ppc440spe_chan->lock);
1958     slot_cnt = slots_per_op = 1;
1959     sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
1960             slots_per_op);
1961     if (sw_desc) {
1962         group_start = sw_desc->group_head;
1963         ppc440spe_desc_init_interrupt(group_start, ppc440spe_chan);
1964         group_start->unmap_len = 0;
1965         sw_desc->async_tx.flags = flags;
1966     }
1967     spin_unlock_bh(&ppc440spe_chan->lock);
1968
1969     return sw_desc ? &sw_desc->async_tx : NULL;
1970 }
1971
1972 /**
1973  * ppc440spe_adma_prep_dma_memcpy - prepare CDB for a MEMCPY operation
1974  */
1975 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_memcpy(
1976         struct dma_chan *chan, dma_addr_t dma_dest,
1977         dma_addr_t dma_src, size_t len, unsigned long flags)
1978 {
1979     struct ppc440spe_adma_chan *ppc440spe_chan;
1980     struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
1981     int slot_cnt, slots_per_op;
1982
1983     ppc440spe_chan = to_ppc440spe_adma_chan(chan);
1984
1985     if (unlikely(!len))
1986         return NULL;
1987
1988     BUG_ON(len > PPC440SPE_ADMA_DMA_MAX_BYTE_COUNT);
1989
1990     spin_lock_bh(&ppc440spe_chan->lock);
1991
1992     dev_dbg(ppc440spe_chan->device->common.dev,
1993         "ppc440spe adma%d: %s len: %u int_en %d\n",
1994         ppc440spe_chan->device->id, __func__, len,
1995         flags & DMA_PREP_INTERRUPT ? 1 : 0);
1996     slot_cnt = slots_per_op = 1;
1997     sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
1998         slots_per_op);
1999     if (sw_desc) {
2000         group_start = sw_desc->group_head;
2001         ppc440spe_desc_init_memcpy(group_start, flags);
2002         ppc440spe_adma_set_dest(group_start, dma_dest, 0);
2003         ppc440spe_adma_memcpy_xor_set_src(group_start, dma_src, 0);
2004         ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len);
2005         sw_desc->unmap_len = len;
2006         sw_desc->async_tx.flags = flags;
2007     }
2008     spin_unlock_bh(&ppc440spe_chan->lock);
2009
2010     return sw_desc ? &sw_desc->async_tx : NULL;
2011 }
2012
2013 /**
2014  * ppc440spe_adma_prep_dma_xor - prepare CDB for a XOR operation
2015  */
2016 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor(
2017         struct dma_chan *chan, dma_addr_t dma_dest,
2018         dma_addr_t *dma_src, u32 src_cnt, size_t len,
2019         unsigned long flags)
2020 {
2021     struct ppc440spe_adma_chan *ppc440spe_chan;
2022     struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
2023     int slot_cnt, slots_per_op;
2024
2025     ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2026
2027     ADMA_LL_DBG(prep_dma_xor_dbg(ppc440spe_chan->device->id,
2028                      dma_dest, dma_src, src_cnt));
2029     if (unlikely(!len))
2030         return NULL;
2031     BUG_ON(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT);
2032
2033     dev_dbg(ppc440spe_chan->device->common.dev,
2034         "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n",
2035         ppc440spe_chan->device->id, __func__, src_cnt, len,
2036         flags & DMA_PREP_INTERRUPT ? 1 : 0);
2037
2038     spin_lock_bh(&ppc440spe_chan->lock);
2039     slot_cnt = ppc440spe_chan_xor_slot_count(len, src_cnt, &slots_per_op);
2040     sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2041             slots_per_op);
2042     if (sw_desc) {
2043         group_start = sw_desc->group_head;
2044         ppc440spe_desc_init_xor(group_start, src_cnt, flags);
2045         ppc440spe_adma_set_dest(group_start, dma_dest, 0);
2046         while (src_cnt--)
2047             ppc440spe_adma_memcpy_xor_set_src(group_start,
2048                 dma_src[src_cnt], src_cnt);
2049         ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len);
2050         sw_desc->unmap_len = len;
2051         sw_desc->async_tx.flags = flags;
2052     }
2053     spin_unlock_bh(&ppc440spe_chan->lock);
2054
2055     return sw_desc ? &sw_desc->async_tx : NULL;
2056 }
2057
2058 static inline void
2059 ppc440spe_desc_set_xor_src_cnt(struct ppc440spe_adma_desc_slot *desc,
2060                 int src_cnt);
2061 static void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor);
2062
2063 /**
2064  * ppc440spe_adma_init_dma2rxor_slot -
2065  */
2066 static void ppc440spe_adma_init_dma2rxor_slot(
2067         struct ppc440spe_adma_desc_slot *desc,
2068         dma_addr_t *src, int src_cnt)
2069 {
2070     int i;
2071
2072     /* initialize CDB */
2073     for (i = 0; i < src_cnt; i++) {
2074         ppc440spe_adma_dma2rxor_prep_src(desc, &desc->rxor_cursor, i,
2075                          desc->src_cnt, (u32)src[i]);
2076     }
2077 }
2078
2079 /**
2080  * ppc440spe_dma01_prep_mult -
2081  * for Q operation where destination is also the source
2082  */
2083 static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_mult(
2084         struct ppc440spe_adma_chan *ppc440spe_chan,
2085         dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
2086         const unsigned char *scf, size_t len, unsigned long flags)
2087 {
2088     struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2089     unsigned long op = 0;
2090     int slot_cnt;
2091
2092     set_bit(PPC440SPE_DESC_WXOR, &op);
2093     slot_cnt = 2;
2094
2095     spin_lock_bh(&ppc440spe_chan->lock);
2096
2097     /* use WXOR, each descriptor occupies one slot */
2098     sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2099     if (sw_desc) {
2100         struct ppc440spe_adma_chan *chan;
2101         struct ppc440spe_adma_desc_slot *iter;
2102         struct dma_cdb *hw_desc;
2103
2104         chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2105         set_bits(op, &sw_desc->flags);
2106         sw_desc->src_cnt = src_cnt;
2107         sw_desc->dst_cnt = dst_cnt;
2108         /* First descriptor, zero data in the destination and copy it
2109          * to q page using MULTICAST transfer.
2110          */
2111         iter = list_first_entry(&sw_desc->group_list,
2112                     struct ppc440spe_adma_desc_slot,
2113                     chain_node);
2114         memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2115         /* set 'next' pointer */
2116         iter->hw_next = list_entry(iter->chain_node.next,
2117                        struct ppc440spe_adma_desc_slot,
2118                        chain_node);
2119         clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2120         hw_desc = iter->hw_desc;
2121         hw_desc->opc = DMA_CDB_OPC_MULTICAST;
2122
2123         ppc440spe_desc_set_dest_addr(iter, chan,
2124                          DMA_CUED_XOR_BASE, dst[0], 0);
2125         ppc440spe_desc_set_dest_addr(iter, chan, 0, dst[1], 1);
2126         ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2127                         src[0]);
2128         ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2129         iter->unmap_len = len;
2130
2131         /*
2132          * Second descriptor, multiply data from the q page
2133          * and store the result in real destination.
2134          */
2135         iter = list_first_entry(&iter->chain_node,
2136                     struct ppc440spe_adma_desc_slot,
2137                     chain_node);
2138         memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2139         iter->hw_next = NULL;
2140         if (flags & DMA_PREP_INTERRUPT)
2141             set_bit(PPC440SPE_DESC_INT, &iter->flags);
2142         else
2143             clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2144
2145         hw_desc = iter->hw_desc;
2146         hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2147         ppc440spe_desc_set_src_addr(iter, chan, 0,
2148                         DMA_CUED_XOR_HB, dst[1]);
2149         ppc440spe_desc_set_dest_addr(iter, chan,
2150                          DMA_CUED_XOR_BASE, dst[0], 0);
2151
2152         ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2153                         DMA_CDB_SG_DST1, scf[0]);
2154         ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2155         iter->unmap_len = len;
2156         sw_desc->async_tx.flags = flags;
2157     }
2158
2159     spin_unlock_bh(&ppc440spe_chan->lock);
2160
2161     return sw_desc;
2162 }
2163
2164 /**
2165  * ppc440spe_dma01_prep_sum_product -
2166  * Dx = A*(P+Pxy) + B*(Q+Qxy) operation where destination is also
2167  * the source.
2168  */
2169 static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_sum_product(
2170         struct ppc440spe_adma_chan *ppc440spe_chan,
2171         dma_addr_t *dst, dma_addr_t *src, int src_cnt,
2172         const unsigned char *scf, size_t len, unsigned long flags)
2173 {
2174     struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2175     unsigned long op = 0;
2176     int slot_cnt;
2177
2178     set_bit(PPC440SPE_DESC_WXOR, &op);
2179     slot_cnt = 3;
2180
2181     spin_lock_bh(&ppc440spe_chan->lock);
2182
2183     /* WXOR, each descriptor occupies one slot */
2184     sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2185     if (sw_desc) {
2186         struct ppc440spe_adma_chan *chan;
2187         struct ppc440spe_adma_desc_slot *iter;
2188         struct dma_cdb *hw_desc;
2189
2190         chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2191         set_bits(op, &sw_desc->flags);
2192         sw_desc->src_cnt = src_cnt;
2193         sw_desc->dst_cnt = 1;
2194         /* 1st descriptor, src[1] data to q page and zero destination */
2195         iter = list_first_entry(&sw_desc->group_list,
2196                     struct ppc440spe_adma_desc_slot,
2197                     chain_node);
2198         memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2199         iter->hw_next = list_entry(iter->chain_node.next,
2200                        struct ppc440spe_adma_desc_slot,
2201                        chain_node);
2202         clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2203         hw_desc = iter->hw_desc;
2204         hw_desc->opc = DMA_CDB_OPC_MULTICAST;
2205
2206         ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE,
2207                          *dst, 0);
2208         ppc440spe_desc_set_dest_addr(iter, chan, 0,
2209                          ppc440spe_chan->qdest, 1);
2210         ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2211                         src[1]);
2212         ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2213         iter->unmap_len = len;
2214
2215         /* 2nd descriptor, multiply src[1] data and store the
2216          * result in destination */
2217         iter = list_first_entry(&iter->chain_node,
2218                     struct ppc440spe_adma_desc_slot,
2219                     chain_node);
2220         memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2221         /* set 'next' pointer */
2222         iter->hw_next = list_entry(iter->chain_node.next,
2223                        struct ppc440spe_adma_desc_slot,
2224                        chain_node);
2225         if (flags & DMA_PREP_INTERRUPT)
2226             set_bit(PPC440SPE_DESC_INT, &iter->flags);
2227         else
2228             clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2229
2230         hw_desc = iter->hw_desc;
2231         hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2232         ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2233                         ppc440spe_chan->qdest);
2234         ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE,
2235                          *dst, 0);
2236         ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2237                         DMA_CDB_SG_DST1, scf[1]);
2238         ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2239         iter->unmap_len = len;
2240
2241         /*
2242          * 3rd descriptor, multiply src[0] data and xor it
2243          * with destination
2244          */
2245         iter = list_first_entry(&iter->chain_node,
2246                     struct ppc440spe_adma_desc_slot,
2247                     chain_node);
2248         memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2249         iter->hw_next = NULL;
2250         if (flags & DMA_PREP_INTERRUPT)
2251             set_bit(PPC440SPE_DESC_INT, &iter->flags);
2252         else
2253             clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2254
2255         hw_desc = iter->hw_desc;
2256         hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2257         ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2258                         src[0]);
2259         ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE,
2260                          *dst, 0);
2261         ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2262                         DMA_CDB_SG_DST1, scf[0]);
2263         ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2264         iter->unmap_len = len;
2265         sw_desc->async_tx.flags = flags;
2266     }
2267
2268     spin_unlock_bh(&ppc440spe_chan->lock);
2269
2270     return sw_desc;
2271 }
2272
2273 static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_pq(
2274         struct ppc440spe_adma_chan *ppc440spe_chan,
2275         dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
2276         const unsigned char *scf, size_t len, unsigned long flags)
2277 {
2278     int slot_cnt;
2279     struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter;
2280     unsigned long op = 0;
2281     unsigned char mult = 1;
2282
2283     pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n",
2284          __func__, dst_cnt, src_cnt, len);
2285     /*  select operations WXOR/RXOR depending on the
2286      * source addresses of operators and the number
2287      * of destinations (RXOR support only Q-parity calculations)
2288      */
2289     set_bit(PPC440SPE_DESC_WXOR, &op);
2290     if (!test_and_set_bit(PPC440SPE_RXOR_RUN, &ppc440spe_rxor_state)) {
2291         /* no active RXOR;
2292          * do RXOR if:
2293          * - there are more than 1 source,
2294          * - len is aligned on 512-byte boundary,
2295          * - source addresses fit to one of 4 possible regions.
2296          */
2297         if (src_cnt > 1 &&
2298             !(len & MQ0_CF2H_RXOR_BS_MASK) &&
2299             (src[0] + len) == src[1]) {
2300             /* may do RXOR R1 R2 */
2301             set_bit(PPC440SPE_DESC_RXOR, &op);
2302             if (src_cnt != 2) {
2303                 /* may try to enhance region of RXOR */
2304                 if ((src[1] + len) == src[2]) {
2305                     /* do RXOR R1 R2 R3 */
2306                     set_bit(PPC440SPE_DESC_RXOR123,
2307                         &op);
2308                 } else if ((src[1] + len * 2) == src[2]) {
2309                     /* do RXOR R1 R2 R4 */
2310                     set_bit(PPC440SPE_DESC_RXOR124, &op);
2311                 } else if ((src[1] + len * 3) == src[2]) {
2312                     /* do RXOR R1 R2 R5 */
2313                     set_bit(PPC440SPE_DESC_RXOR125,
2314                         &op);
2315                 } else {
2316                     /* do RXOR R1 R2 */
2317                     set_bit(PPC440SPE_DESC_RXOR12,
2318                         &op);
2319                 }
2320             } else {
2321                 /* do RXOR R1 R2 */
2322                 set_bit(PPC440SPE_DESC_RXOR12, &op);
2323             }
2324         }
2325
2326         if (!test_bit(PPC440SPE_DESC_RXOR, &op)) {
2327             /* can not do this operation with RXOR */
2328             clear_bit(PPC440SPE_RXOR_RUN,
2329                 &ppc440spe_rxor_state);
2330         } else {
2331             /* can do; set block size right now */
2332             ppc440spe_desc_set_rxor_block_size(len);
2333         }
2334     }
2335
2336     /* Number of necessary slots depends on operation type selected */
2337     if (!test_bit(PPC440SPE_DESC_RXOR, &op)) {
2338         /*  This is a WXOR only chain. Need descriptors for each
2339          * source to GF-XOR them with WXOR, and need descriptors
2340          * for each destination to zero them with WXOR
2341          */
2342         slot_cnt = src_cnt;
2343
2344         if (flags & DMA_PREP_ZERO_P) {
2345             slot_cnt++;
2346             set_bit(PPC440SPE_ZERO_P, &op);
2347         }
2348         if (flags & DMA_PREP_ZERO_Q) {
2349             slot_cnt++;
2350             set_bit(PPC440SPE_ZERO_Q, &op);
2351         }
2352     } else {
2353         /*  Need 1/2 descriptor for RXOR operation, and
2354          * need (src_cnt - (2 or 3)) for WXOR of sources
2355          * remained (if any)
2356          */
2357         slot_cnt = dst_cnt;
2358
2359         if (flags & DMA_PREP_ZERO_P)
2360             set_bit(PPC440SPE_ZERO_P, &op);
2361         if (flags & DMA_PREP_ZERO_Q)
2362             set_bit(PPC440SPE_ZERO_Q, &op);
2363
2364         if (test_bit(PPC440SPE_DESC_RXOR12, &op))
2365             slot_cnt += src_cnt - 2;
2366         else
2367             slot_cnt += src_cnt - 3;
2368
2369         /*  Thus we have either RXOR only chain or
2370          * mixed RXOR/WXOR
2371          */
2372         if (slot_cnt == dst_cnt)
2373             /* RXOR only chain */
2374             clear_bit(PPC440SPE_DESC_WXOR, &op);
2375     }
2376
2377     spin_lock_bh(&ppc440spe_chan->lock);
2378     /* for both RXOR/WXOR each descriptor occupies one slot */
2379     sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2380     if (sw_desc) {
2381         ppc440spe_desc_init_dma01pq(sw_desc, dst_cnt, src_cnt,
2382                 flags, op);
2383
2384         /* setup dst/src/mult */
2385         pr_debug("%s: set dst descriptor 0, 1: 0x%016llx, 0x%016llx\n",
2386              __func__, dst[0], dst[1]);
2387         ppc440spe_adma_pq_set_dest(sw_desc, dst, flags);
2388         while (src_cnt--) {
2389             ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt],
2390                           src_cnt);
2391
2392             /* NOTE: "Multi = 0 is equivalent to = 1" as it
2393              * stated in 440SPSPe_RAID6_Addendum_UM_1_17.pdf
2394              * doesn't work for RXOR with DMA0/1! Instead, multi=0
2395              * leads to zeroing source data after RXOR.
2396              * So, for P case set-up mult=1 explicitly.
2397              */
2398             if (!(flags & DMA_PREP_PQ_DISABLE_Q))
2399                 mult = scf[src_cnt];
2400             ppc440spe_adma_pq_set_src_mult(sw_desc,
2401                 mult, src_cnt,  dst_cnt - 1);
2402         }
2403
2404         /* Setup byte count foreach slot just allocated */
2405         sw_desc->async_tx.flags = flags;
2406         list_for_each_entry(iter, &sw_desc->group_list,
2407                 chain_node) {
2408             ppc440spe_desc_set_byte_count(iter,
2409                 ppc440spe_chan, len);
2410             iter->unmap_len = len;
2411         }
2412     }
2413     spin_unlock_bh(&ppc440spe_chan->lock);
2414
2415     return sw_desc;
2416 }
2417
2418 static struct ppc440spe_adma_desc_slot *ppc440spe_dma2_prep_pq(
2419         struct ppc440spe_adma_chan *ppc440spe_chan,
2420         dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
2421         const unsigned char *scf, size_t len, unsigned long flags)
2422 {
2423     int slot_cnt, descs_per_op;
2424     struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter;
2425     unsigned long op = 0;
2426     unsigned char mult = 1;
2427
2428     BUG_ON(!dst_cnt);
2429     /*pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n",
2430          __func__, dst_cnt, src_cnt, len);*/
2431
2432     spin_lock_bh(&ppc440spe_chan->lock);
2433     descs_per_op = ppc440spe_dma2_pq_slot_count(src, src_cnt, len);
2434     if (descs_per_op < 0) {
2435         spin_unlock_bh(&ppc440spe_chan->lock);
2436         return NULL;
2437     }
2438
2439     /* depending on number of sources we have 1 or 2 RXOR chains */
2440     slot_cnt = descs_per_op * dst_cnt;
2441
2442     sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2443     if (sw_desc) {
2444         op = slot_cnt;
2445         sw_desc->async_tx.flags = flags;
2446         list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2447             ppc440spe_desc_init_dma2pq(iter, dst_cnt, src_cnt,
2448                 --op ? 0 : flags);
2449             ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2450                 len);
2451             iter->unmap_len = len;
2452
2453             ppc440spe_init_rxor_cursor(&(iter->rxor_cursor));
2454             iter->rxor_cursor.len = len;
2455             iter->descs_per_op = descs_per_op;
2456         }
2457         op = 0;
2458         list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2459             op++;
2460             if (op % descs_per_op == 0)
2461                 ppc440spe_adma_init_dma2rxor_slot(iter, src,
2462                                   src_cnt);
2463             if (likely(!list_is_last(&iter->chain_node,
2464                          &sw_desc->group_list))) {
2465                 /* set 'next' pointer */
2466                 iter->hw_next =
2467                     list_entry(iter->chain_node.next,
2468                         struct ppc440spe_adma_desc_slot,
2469                         chain_node);
2470                 ppc440spe_xor_set_link(iter, iter->hw_next);
2471             } else {
2472                 /* this is the last descriptor. */
2473                 iter->hw_next = NULL;
2474             }
2475         }
2476
2477         /* fixup head descriptor */
2478         sw_desc->dst_cnt = dst_cnt;
2479         if (flags & DMA_PREP_ZERO_P)
2480             set_bit(PPC440SPE_ZERO_P, &sw_desc->flags);
2481         if (flags & DMA_PREP_ZERO_Q)
2482             set_bit(PPC440SPE_ZERO_Q, &sw_desc->flags);
2483
2484         /* setup dst/src/mult */
2485         ppc440spe_adma_pq_set_dest(sw_desc, dst, flags);
2486
2487         while (src_cnt--) {
2488             /* handle descriptors (if dst_cnt == 2) inside
2489              * the ppc440spe_adma_pq_set_srcxxx() functions
2490              */
2491             ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt],
2492                           src_cnt);
2493             if (!(flags & DMA_PREP_PQ_DISABLE_Q))
2494                 mult = scf[src_cnt];
2495             ppc440spe_adma_pq_set_src_mult(sw_desc,
2496                     mult, src_cnt, dst_cnt - 1);
2497         }
2498     }
2499     spin_unlock_bh(&ppc440spe_chan->lock);
2500     ppc440spe_desc_set_rxor_block_size(len);
2501     return sw_desc;
2502 }
2503
2504 /**
2505  * ppc440spe_adma_prep_dma_pq - prepare CDB (group) for a GF-XOR operation
2506  */
2507 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pq(
2508         struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
2509         unsigned int src_cnt, const unsigned char *scf,
2510         size_t len, unsigned long flags)
2511 {
2512     struct ppc440spe_adma_chan *ppc440spe_chan;
2513     struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2514     int dst_cnt = 0;
2515
2516     ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2517
2518     ADMA_LL_DBG(prep_dma_pq_dbg(ppc440spe_chan->device->id,
2519                     dst, src, src_cnt));
2520     BUG_ON(!len);
2521     BUG_ON(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT);
2522     BUG_ON(!src_cnt);
2523
2524     if (src_cnt == 1 && dst[1] == src[0]) {
2525         dma_addr_t dest[2];
2526
2527         /* dst[1] is real destination (Q) */
2528         dest[0] = dst[1];
2529         /* this is the page to multicast source data to */
2530         dest[1] = ppc440spe_chan->qdest;
2531         sw_desc = ppc440spe_dma01_prep_mult(ppc440spe_chan,
2532                 dest, 2, src, src_cnt, scf, len, flags);
2533         return sw_desc ? &sw_desc->async_tx : NULL;
2534     }
2535
2536     if (src_cnt == 2 && dst[1] == src[1]) {
2537         sw_desc = ppc440spe_dma01_prep_sum_product(ppc440spe_chan,
2538                     &dst[1], src, 2, scf, len, flags);
2539         return sw_desc ? &sw_desc->async_tx : NULL;
2540     }
2541
2542     if (!(flags & DMA_PREP_PQ_DISABLE_P)) {
2543         BUG_ON(!dst[0]);
2544         dst_cnt++;
2545         flags |= DMA_PREP_ZERO_P;
2546     }
2547
2548     if (!(flags & DMA_PREP_PQ_DISABLE_Q)) {
2549         BUG_ON(!dst[1]);
2550         dst_cnt++;
2551         flags |= DMA_PREP_ZERO_Q;
2552     }
2553
2554     BUG_ON(!dst_cnt);
2555
2556     dev_dbg(ppc440spe_chan->device->common.dev,
2557         "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n",
2558         ppc440spe_chan->device->id, __func__, src_cnt, len,
2559         flags & DMA_PREP_INTERRUPT ? 1 : 0);
2560
2561     switch (ppc440spe_chan->device->id) {
2562     case PPC440SPE_DMA0_ID:
2563     case PPC440SPE_DMA1_ID:
2564         sw_desc = ppc440spe_dma01_prep_pq(ppc440spe_chan,
2565                 dst, dst_cnt, src, src_cnt, scf,
2566                 len, flags);
2567         break;
2568
2569     case PPC440SPE_XOR_ID:
2570         sw_desc = ppc440spe_dma2_prep_pq(ppc440spe_chan,
2571                 dst, dst_cnt, src, src_cnt, scf,
2572                 len, flags);
2573         break;
2574     }
2575
2576     return sw_desc ? &sw_desc->async_tx : NULL;
2577 }
2578
2579 /**
2580  * ppc440spe_adma_prep_dma_pqzero_sum - prepare CDB group for
2581  * a PQ_ZERO_SUM operation
2582  */
2583 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pqzero_sum(
2584         struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
2585         unsigned int src_cnt, const unsigned char *scf, size_t len,
2586         enum sum_check_flags *pqres, unsigned long flags)
2587 {
2588     struct ppc440spe_adma_chan *ppc440spe_chan;
2589     struct ppc440spe_adma_desc_slot *sw_desc, *iter;
2590     dma_addr_t pdest, qdest;
2591     int slot_cnt, slots_per_op, idst, dst_cnt;
2592
2593     ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2594
2595     if (flags & DMA_PREP_PQ_DISABLE_P)
2596         pdest = 0;
2597     else
2598         pdest = pq[0];
2599
2600     if (flags & DMA_PREP_PQ_DISABLE_Q)
2601         qdest = 0;
2602     else
2603         qdest = pq[1];
2604
2605     ADMA_LL_DBG(prep_dma_pqzero_sum_dbg(ppc440spe_chan->device->id,
2606                         src, src_cnt, scf));
2607
2608     /* Always use WXOR for P/Q calculations (two destinations).
2609      * Need 1 or 2 extra slots to verify results are zero.
2610      */
2611     idst = dst_cnt = (pdest && qdest) ? 2 : 1;
2612
2613     /* One additional slot per destination to clone P/Q
2614      * before calculation (we have to preserve destinations).
2615      */
2616     slot_cnt = src_cnt + dst_cnt * 2;
2617     slots_per_op = 1;
2618
2619     spin_lock_bh(&ppc440spe_chan->lock);
2620     sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2621                          slots_per_op);
2622     if (sw_desc) {
2623         ppc440spe_desc_init_dma01pqzero_sum(sw_desc, dst_cnt, src_cnt);
2624
2625         /* Setup byte count for each slot just allocated */
2626         sw_desc->async_tx.flags = flags;
2627         list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2628             ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2629                               len);
2630             iter->unmap_len = len;
2631         }
2632
2633         if (pdest) {
2634             struct dma_cdb *hw_desc;
2635             struct ppc440spe_adma_chan *chan;
2636
2637             iter = sw_desc->group_head;
2638             chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
2639             memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2640             iter->hw_next = list_entry(iter->chain_node.next,
2641                         struct ppc440spe_adma_desc_slot,
2642                         chain_node);
2643             hw_desc = iter->hw_desc;
2644             hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2645             iter->src_cnt = 0;
2646             iter->dst_cnt = 0;
2647             ppc440spe_desc_set_dest_addr(iter, chan, 0,
2648                              ppc440spe_chan->pdest, 0);
2649             ppc440spe_desc_set_src_addr(iter, chan, 0, 0, pdest);
2650             ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2651                               len);
2652             iter->unmap_len = 0;
2653             /* override pdest to preserve original P */
2654             pdest = ppc440spe_chan->pdest;
2655         }
2656         if (qdest) {
2657             struct dma_cdb *hw_desc;
2658             struct ppc440spe_adma_chan *chan;
2659
2660             iter = list_first_entry(&sw_desc->group_list,
2661                         struct ppc440spe_adma_desc_slot,
2662                         chain_node);
2663             chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
2664
2665             if (pdest) {
2666                 iter = list_entry(iter->chain_node.next,
2667                         struct ppc440spe_adma_desc_slot,
2668                         chain_node);
2669             }
2670
2671             memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2672             iter->hw_next = list_entry(iter->chain_node.next,
2673                         struct ppc440spe_adma_desc_slot,
2674                         chain_node);
2675             hw_desc = iter->hw_desc;
2676             hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2677             iter->src_cnt = 0;
2678             iter->dst_cnt = 0;
2679             ppc440spe_desc_set_dest_addr(iter, chan, 0,
2680                              ppc440spe_chan->qdest, 0);
2681             ppc440spe_desc_set_src_addr(iter, chan, 0, 0, qdest);
2682             ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2683                               len);
2684             iter->unmap_len = 0;
2685             /* override qdest to preserve original Q */
2686             qdest = ppc440spe_chan->qdest;
2687         }
2688
2689         /* Setup destinations for P/Q ops */
2690         ppc440spe_adma_pqzero_sum_set_dest(sw_desc, pdest, qdest);
2691
2692         /* Setup zero QWORDs into DCHECK CDBs */
2693         idst = dst_cnt;
2694         list_for_each_entry_reverse(iter, &sw_desc->group_list,
2695                         chain_node) {
2696             /*
2697              * The last CDB corresponds to Q-parity check,
2698              * the one before last CDB corresponds
2699              * P-parity check
2700              */
2701             if (idst == DMA_DEST_MAX_NUM) {
2702                 if (idst == dst_cnt) {
2703                     set_bit(PPC440SPE_DESC_QCHECK,
2704                         &iter->flags);
2705                 } else {
2706                     set_bit(PPC440SPE_DESC_PCHECK,
2707                         &iter->flags);
2708                 }
2709             } else {
2710                 if (qdest) {
2711                     set_bit(PPC440SPE_DESC_QCHECK,
2712                         &iter->flags);
2713                 } else {
2714                     set_bit(PPC440SPE_DESC_PCHECK,
2715                         &iter->flags);
2716                 }
2717             }
2718             iter->xor_check_result = pqres;
2719
2720             /*
2721              * set it to zero, if check fail then result will
2722              * be updated
2723              */
2724             *iter->xor_check_result = 0;
2725             ppc440spe_desc_set_dcheck(iter, ppc440spe_chan,
2726                 ppc440spe_qword);
2727
2728             if (!(--dst_cnt))
2729                 break;
2730         }
2731
2732         /* Setup sources and mults for P/Q ops */
2733         list_for_each_entry_continue_reverse(iter, &sw_desc->group_list,
2734                              chain_node) {
2735             struct ppc440spe_adma_chan *chan;
2736             u32 mult_dst;
2737
2738             chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
2739             ppc440spe_desc_set_src_addr(iter, chan, 0,
2740                             DMA_CUED_XOR_HB,
2741                             src[src_cnt - 1]);
2742             if (qdest) {
2743                 mult_dst = (dst_cnt - 1) ? DMA_CDB_SG_DST2 :
2744                                DMA_CDB_SG_DST1;
2745                 ppc440spe_desc_set_src_mult(iter, chan,
2746                                 DMA_CUED_MULT1_OFF,
2747                                 mult_dst,
2748                                 scf[src_cnt - 1]);
2749             }
2750             if (!(--src_cnt))
2751                 break;
2752         }
2753     }
2754     spin_unlock_bh(&ppc440spe_chan->lock);
2755     return sw_desc ? &sw_desc->async_tx : NULL;
2756 }
2757
2758 /**
2759  * ppc440spe_adma_prep_dma_xor_zero_sum - prepare CDB group for
2760  * XOR ZERO_SUM operation
2761  */
2762 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor_zero_sum(
2763         struct dma_chan *chan, dma_addr_t *src, unsigned int src_cnt,
2764         size_t len, enum sum_check_flags *result, unsigned long flags)
2765 {
2766     struct dma_async_tx_descriptor *tx;
2767     dma_addr_t pq[2];
2768
2769     /* validate P, disable Q */
2770     pq[0] = src[0];
2771     pq[1] = 0;
2772     flags |= DMA_PREP_PQ_DISABLE_Q;
2773
2774     tx = ppc440spe_adma_prep_dma_pqzero_sum(chan, pq, &src[1],
2775                         src_cnt - 1, 0, len,
2776                         result, flags);
2777     return tx;
2778 }
2779
2780 /**
2781  * ppc440spe_adma_set_dest - set destination address into descriptor
2782  */
2783 static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *sw_desc,
2784         dma_addr_t addr, int index)
2785 {
2786     struct ppc440spe_adma_chan *chan;
2787
2788     BUG_ON(index >= sw_desc->dst_cnt);
2789
2790     chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2791
2792     switch (chan->device->id) {
2793     case PPC440SPE_DMA0_ID:
2794     case PPC440SPE_DMA1_ID:
2795         /* to do: support transfers lengths >
2796          * PPC440SPE_ADMA_DMA/XOR_MAX_BYTE_COUNT
2797          */
2798         ppc440spe_desc_set_dest_addr(sw_desc->group_head,
2799             chan, 0, addr, index);
2800         break;
2801     case PPC440SPE_XOR_ID:
2802         sw_desc = ppc440spe_get_group_entry(sw_desc, index);
2803         ppc440spe_desc_set_dest_addr(sw_desc,
2804             chan, 0, addr, index);
2805         break;
2806     }
2807 }
2808
2809 static void ppc440spe_adma_pq_zero_op(struct ppc440spe_adma_desc_slot *iter,
2810         struct ppc440spe_adma_chan *chan, dma_addr_t addr)
2811 {
2812     /*  To clear destinations update the descriptor
2813      * (P or Q depending on index) as follows:
2814      * addr is destination (0 corresponds to SG2):
2815      */
2816     ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, addr, 0);
2817
2818     /* ... and the addr is source: */
2819     ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, addr);
2820
2821     /* addr is always SG2 then the mult is always DST1 */
2822     ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2823                     DMA_CDB_SG_DST1, 1);
2824 }
2825
2826 /**
2827  * ppc440spe_adma_pq_set_dest - set destination address into descriptor
2828  * for the PQXOR operation
2829  */
2830 static void ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *sw_desc,
2831         dma_addr_t *addrs, unsigned long flags)
2832 {
2833     struct ppc440spe_adma_desc_slot *iter;
2834     struct ppc440spe_adma_chan *chan;
2835     dma_addr_t paddr, qaddr;
2836     dma_addr_t addr = 0, ppath, qpath;
2837     int index = 0, i;
2838
2839     chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2840
2841     if (flags & DMA_PREP_PQ_DISABLE_P)
2842         paddr = 0;
2843     else
2844         paddr = addrs[0];
2845
2846     if (flags & DMA_PREP_PQ_DISABLE_Q)
2847         qaddr = 0;
2848     else
2849         qaddr = addrs[1];
2850
2851     if (!paddr || !qaddr)
2852         addr = paddr ? paddr : qaddr;
2853
2854     switch (chan->device->id) {
2855     case PPC440SPE_DMA0_ID:
2856     case PPC440SPE_DMA1_ID:
2857         /* walk through the WXOR source list and set P/Q-destinations
2858          * for each slot:
2859          */
2860         if (!test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
2861             /* This is WXOR-only chain; may have 1/2 zero descs */
2862             if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
2863                 index++;
2864             if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
2865                 index++;
2866
2867             iter = ppc440spe_get_group_entry(sw_desc, index);
2868             if (addr) {
2869                 /* one destination */
2870                 list_for_each_entry_from(iter,
2871                     &sw_desc->group_list, chain_node)
2872                     ppc440spe_desc_set_dest_addr(iter, chan,
2873                         DMA_CUED_XOR_BASE, addr, 0);
2874             } else {
2875                 /* two destinations */
2876                 list_for_each_entry_from(iter,
2877                     &sw_desc->group_list, chain_node) {
2878                     ppc440spe_desc_set_dest_addr(iter, chan,
2879                         DMA_CUED_XOR_BASE, paddr, 0);
2880                     ppc440spe_desc_set_dest_addr(iter, chan,
2881                         DMA_CUED_XOR_BASE, qaddr, 1);
2882                 }
2883             }
2884
2885             if (index) {
2886                 /*  To clear destinations update the descriptor
2887                  * (1st,2nd, or both depending on flags)
2888                  */
2889                 index = 0;
2890                 if (test_bit(PPC440SPE_ZERO_P,
2891                         &sw_desc->flags)) {
2892                     iter = ppc440spe_get_group_entry(
2893                             sw_desc, index++);
2894                     ppc440spe_adma_pq_zero_op(iter, chan,
2895                             paddr);
2896                 }
2897
2898                 if (test_bit(PPC440SPE_ZERO_Q,
2899                         &sw_desc->flags)) {
2900                     iter = ppc440spe_get_group_entry(
2901                             sw_desc, index++);
2902                     ppc440spe_adma_pq_zero_op(iter, chan,
2903                             qaddr);
2904                 }
2905
2906                 return;
2907             }
2908         } else {
2909             /* This is RXOR-only or RXOR/WXOR mixed chain */
2910
2911             /* If we want to include destination into calculations,
2912              * then make dest addresses cued with mult=1 (XOR).
2913              */
2914             ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ?
2915                     DMA_CUED_XOR_HB :
2916                     DMA_CUED_XOR_BASE |
2917                         (1 << DMA_CUED_MULT1_OFF);
2918             qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ?
2919                     DMA_CUED_XOR_HB :
2920                     DMA_CUED_XOR_BASE |
2921                         (1 << DMA_CUED_MULT1_OFF);
2922
2923             /* Setup destination(s) in RXOR slot(s) */
2924             iter = ppc440spe_get_group_entry(sw_desc, index++);
2925             ppc440spe_desc_set_dest_addr(iter, chan,
2926                         paddr ? ppath : qpath,
2927                         paddr ? paddr : qaddr, 0);
2928             if (!addr) {
2929                 /* two destinations */
2930                 iter = ppc440spe_get_group_entry(sw_desc,
2931                                  index++);
2932                 ppc440spe_desc_set_dest_addr(iter, chan,
2933                         qpath, qaddr, 0);
2934             }
2935
2936             if (test_bit(PPC440SPE_DESC_WXOR, &sw_desc->flags)) {
2937                 /* Setup destination(s) in remaining WXOR
2938                  * slots
2939                  */
2940                 iter = ppc440spe_get_group_entry(sw_desc,
2941                                  index);
2942                 if (addr) {
2943                     /* one destination */
2944                     list_for_each_entry_from(iter,
2945                         &sw_desc->group_list,
2946                         chain_node)
2947                         ppc440spe_desc_set_dest_addr(
2948                             iter, chan,
2949                             DMA_CUED_XOR_BASE,
2950                             addr, 0);
2951
2952                 } else {
2953                     /* two destinations */
2954                     list_for_each_entry_from(iter,
2955                         &sw_desc->group_list,
2956                         chain_node) {
2957                         ppc440spe_desc_set_dest_addr(
2958                             iter, chan,
2959                             DMA_CUED_XOR_BASE,
2960                             paddr, 0);
2961                         ppc440spe_desc_set_dest_addr(
2962                             iter, chan,
2963                             DMA_CUED_XOR_BASE,
2964                             qaddr, 1);
2965                     }
2966                 }
2967             }
2968
2969         }
2970         break;
2971
2972     case PPC440SPE_XOR_ID:
2973         /* DMA2 descriptors have only 1 destination, so there are
2974          * two chains - one for each dest.
2975          * If we want to include destination into calculations,
2976          * then make dest addresses cued with mult=1 (XOR).
2977          */
2978         ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ?
2979                 DMA_CUED_XOR_HB :
2980                 DMA_CUED_XOR_BASE |
2981                     (1 << DMA_CUED_MULT1_OFF);
2982
2983         qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ?
2984                 DMA_CUED_XOR_HB :
2985                 DMA_CUED_XOR_BASE |
2986                     (1 << DMA_CUED_MULT1_OFF);
2987
2988         iter = ppc440spe_get_group_entry(sw_desc, 0);
2989         for (i = 0; i < sw_desc->descs_per_op; i++) {
2990             ppc440spe_desc_set_dest_addr(iter, chan,
2991                 paddr ? ppath : qpath,
2992                 paddr ? paddr : qaddr, 0);
2993             iter = list_entry(iter->chain_node.next,
2994                       struct ppc440spe_adma_desc_slot,
2995                       chain_node);
2996         }
2997
2998         if (!addr) {
2999             /* Two destinations; setup Q here */
3000             iter = ppc440spe_get_group_entry(sw_desc,
3001                 sw_desc->descs_per_op);
3002             for (i = 0; i < sw_desc->descs_per_op; i++) {
3003                 ppc440spe_desc_set_dest_addr(iter,
3004                     chan, qpath, qaddr, 0);
3005                 iter = list_entry(iter->chain_node.next,
3006                         struct ppc440spe_adma_desc_slot,
3007                         chain_node);
3008             }
3009         }
3010
3011         break;
3012     }
3013 }
3014
3015 /**
3016  * ppc440spe_adma_pq_zero_sum_set_dest - set destination address into descriptor
3017  * for the PQ_ZERO_SUM operation
3018  */
3019 static void ppc440spe_adma_pqzero_sum_set_dest(
3020         struct ppc440spe_adma_desc_slot *sw_desc,
3021         dma_addr_t paddr, dma_addr_t qaddr)
3022 {
3023     struct ppc440spe_adma_desc_slot *iter, *end;
3024     struct ppc440spe_adma_chan *chan;
3025     dma_addr_t addr = 0;
3026     int idx;
3027
3028     chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3029
3030     /* walk through the WXOR source list and set P/Q-destinations
3031      * for each slot
3032      */
3033     idx = (paddr && qaddr) ? 2 : 1;
3034     /* set end */
3035     list_for_each_entry_reverse(end, &sw_desc->group_list,
3036                     chain_node) {
3037         if (!(--idx))
3038             break;
3039     }
3040     /* set start */
3041     idx = (paddr && qaddr) ? 2 : 1;
3042     iter = ppc440spe_get_group_entry(sw_desc, idx);
3043
3044     if (paddr && qaddr) {
3045         /* two destinations */
3046         list_for_each_entry_from(iter, &sw_desc->group_list,
3047                      chain_node) {
3048             if (unlikely(iter == end))
3049                 break;
3050             ppc440spe_desc_set_dest_addr(iter, chan,
3051                         DMA_CUED_XOR_BASE, paddr, 0);
3052             ppc440spe_desc_set_dest_addr(iter, chan,
3053                         DMA_CUED_XOR_BASE, qaddr, 1);
3054         }
3055     } else {
3056         /* one destination */
3057         addr = paddr ? paddr : qaddr;
3058         list_for_each_entry_from(iter, &sw_desc->group_list,
3059                      chain_node) {
3060             if (unlikely(iter == end))
3061                 break;
3062             ppc440spe_desc_set_dest_addr(iter, chan,
3063                         DMA_CUED_XOR_BASE, addr, 0);
3064         }
3065     }
3066
3067     /*  The remaining descriptors are DATACHECK. These have no need in
3068      * destination. Actually, these destinations are used there
3069      * as sources for check operation. So, set addr as source.
3070      */
3071     ppc440spe_desc_set_src_addr(end, chan, 0, 0, addr ? addr : paddr);
3072
3073     if (!addr) {
3074         end = list_entry(end->chain_node.next,
3075                  struct ppc440spe_adma_desc_slot, chain_node);
3076         ppc440spe_desc_set_src_addr(end, chan, 0, 0, qaddr);
3077     }
3078 }
3079
3080 /**
3081  * ppc440spe_desc_set_xor_src_cnt - set source count into descriptor
3082  */
3083 static inline void ppc440spe_desc_set_xor_src_cnt(
3084             struct ppc440spe_adma_desc_slot *desc,
3085             int src_cnt)
3086 {
3087     struct xor_cb *hw_desc = desc->hw_desc;
3088
3089     hw_desc->cbc &= ~XOR_CDCR_OAC_MSK;
3090     hw_desc->cbc |= src_cnt;
3091 }
3092
3093 /**
3094  * ppc440spe_adma_pq_set_src - set source address into descriptor
3095  */
3096 static void ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *sw_desc,
3097         dma_addr_t addr, int index)
3098 {
3099     struct ppc440spe_adma_chan *chan;
3100     dma_addr_t haddr = 0;
3101     struct ppc440spe_adma_desc_slot *iter = NULL;
3102
3103     chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3104
3105     switch (chan->device->id) {
3106     case PPC440SPE_DMA0_ID:
3107     case PPC440SPE_DMA1_ID:
3108         /* DMA0,1 may do: WXOR, RXOR, RXOR+WXORs chain
3109          */
3110         if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
3111             /* RXOR-only or RXOR/WXOR operation */
3112             int iskip = test_bit(PPC440SPE_DESC_RXOR12,
3113                 &sw_desc->flags) ?  2 : 3;
3114
3115             if (index == 0) {
3116                 /* 1st slot (RXOR) */
3117                 /* setup sources region (R1-2-3, R1-2-4,
3118                  * or R1-2-5)
3119                  */
3120                 if (test_bit(PPC440SPE_DESC_RXOR12,
3121                         &sw_desc->flags))
3122                     haddr = DMA_RXOR12 <<
3123                         DMA_CUED_REGION_OFF;
3124                 else if (test_bit(PPC440SPE_DESC_RXOR123,
3125                     &sw_desc->flags))
3126                     haddr = DMA_RXOR123 <<
3127                         DMA_CUED_REGION_OFF;
3128                 else if (test_bit(PPC440SPE_DESC_RXOR124,
3129                     &sw_desc->flags))
3130                     haddr = DMA_RXOR124 <<
3131                         DMA_CUED_REGION_OFF;
3132                 else if (test_bit(PPC440SPE_DESC_RXOR125,
3133                     &sw_desc->flags))
3134                     haddr = DMA_RXOR125 <<
3135                         DMA_CUED_REGION_OFF;
3136                 else
3137                     BUG();
3138                 haddr |= DMA_CUED_XOR_BASE;
3139                 iter = ppc440spe_get_group_entry(sw_desc, 0);
3140             } else if (index < iskip) {
3141                 /* 1st slot (RXOR)
3142                  * shall actually set source address only once
3143                  * instead of first <iskip>
3144                  */
3145                 iter = NULL;
3146             } else {
3147                 /* 2nd/3d and next slots (WXOR);
3148                  * skip first slot with RXOR
3149                  */
3150                 haddr = DMA_CUED_XOR_HB;
3151                 iter = ppc440spe_get_group_entry(sw_desc,
3152                     index - iskip + sw_desc->dst_cnt);
3153             }
3154         } else {
3155             int znum = 0;
3156
3157             /* WXOR-only operation; skip first slots with
3158              * zeroing destinations
3159              */
3160             if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
3161                 znum++;
3162             if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
3163                 znum++;
3164
3165             haddr = DMA_CUED_XOR_HB;
3166             iter = ppc440spe_get_group_entry(sw_desc,
3167                     index + znum);
3168         }
3169
3170         if (likely(iter)) {
3171             ppc440spe_desc_set_src_addr(iter, chan, 0, haddr, addr);
3172
3173             if (!index &&
3174                 test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags) &&
3175                 sw_desc->dst_cnt == 2) {
3176                 /* if we have two destinations for RXOR, then
3177                  * setup source in the second descr too
3178                  */
3179                 iter = ppc440spe_get_group_entry(sw_desc, 1);
3180                 ppc440spe_desc_set_src_addr(iter, chan, 0,
3181                     haddr, addr);
3182             }
3183         }
3184         break;
3185
3186     case PPC440SPE_XOR_ID:
3187         /* DMA2 may do Biskup */
3188         iter = sw_desc->group_head;
3189         if (iter->dst_cnt == 2) {
3190             /* both P & Q calculations required; set P src here */
3191             ppc440spe_adma_dma2rxor_set_src(iter, index, addr);
3192
3193             /* this is for Q */
3194             iter = ppc440spe_get_group_entry(sw_desc,
3195                 sw_desc->descs_per_op);
3196         }
3197         ppc440spe_adma_dma2rxor_set_src(iter, index, addr);
3198         break;
3199     }
3200 }
3201
3202 /**
3203  * ppc440spe_adma_memcpy_xor_set_src - set source address into descriptor
3204  */
3205 static void ppc440spe_adma_memcpy_xor_set_src(
3206         struct ppc440spe_adma_desc_slot *sw_desc,
3207         dma_addr_t addr, int index)
3208 {
3209     struct ppc440spe_adma_chan *chan;
3210
3211     chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3212     sw_desc = sw_desc->group_head;
3213
3214     if (likely(sw_desc))
3215         ppc440spe_desc_set_src_addr(sw_desc, chan, index, 0, addr);
3216 }
3217
3218 /**
3219  * ppc440spe_adma_dma2rxor_inc_addr  -
3220  */
3221 static void ppc440spe_adma_dma2rxor_inc_addr(
3222         struct ppc440spe_adma_desc_slot *desc,
3223         struct ppc440spe_rxor *cursor, int index, int src_cnt)
3224 {
3225     cursor->addr_count++;
3226     if (index == src_cnt - 1) {
3227         ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count);
3228     } else if (cursor->addr_count == XOR_MAX_OPS) {
3229         ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count);
3230         cursor->addr_count = 0;
3231         cursor->desc_count++;
3232     }
3233 }
3234
3235 /**
3236  * ppc440spe_adma_dma2rxor_prep_src - setup RXOR types in DMA2 CDB
3237  */
3238 static int ppc440spe_adma_dma2rxor_prep_src(
3239         struct ppc440spe_adma_desc_slot *hdesc,
3240         struct ppc440spe_rxor *cursor, int index,
3241         int src_cnt, u32 addr)
3242 {
3243     u32 sign;
3244     struct ppc440spe_adma_desc_slot *desc = hdesc;
3245     int i;
3246
3247     for (i = 0; i < cursor->desc_count; i++) {
3248         desc = list_entry(hdesc->chain_node.next,
3249                   struct ppc440spe_adma_desc_slot,
3250                   chain_node);
3251     }
3252
3253     switch (cursor->state) {
3254     case 0:
3255         if (addr == cursor->addrl + cursor->len) {
3256             /* direct RXOR */
3257             cursor->state = 1;
3258             cursor->xor_count++;
3259             if (index == src_cnt-1) {
3260                 ppc440spe_rxor_set_region(desc,
3261                     cursor->addr_count,
3262                     DMA_RXOR12 << DMA_CUED_REGION_OFF);
3263                 ppc440spe_adma_dma2rxor_inc_addr(
3264                     desc, cursor, index, src_cnt);
3265             }
3266         } else if (cursor->addrl == addr + cursor->len) {
3267             /* reverse RXOR */
3268             cursor->state = 1;
3269             cursor->xor_count++;
3270             set_bit(cursor->addr_count, &desc->reverse_flags[0]);
3271             if (index == src_cnt-1) {
3272                 ppc440spe_rxor_set_region(desc,
3273                     cursor->addr_count,
3274                     DMA_RXOR12 << DMA_CUED_REGION_OFF);
3275                 ppc440spe_adma_dma2rxor_inc_addr(
3276                     desc, cursor, index, src_cnt);
3277             }
3278         } else {
3279             printk(KERN_ERR "Cannot build "
3280                 "DMA2 RXOR command block.\n");
3281             BUG();
3282         }
3283         break;
3284     case 1:
3285         sign = test_bit(cursor->addr_count,
3286                 desc->reverse_flags)
3287             ? -1 : 1;
3288         if (index == src_cnt-2 || (sign == -1
3289             && addr != cursor->addrl - 2*cursor->len)) {
3290             cursor->state = 0;
3291             cursor->xor_count = 1;
3292             cursor->addrl = addr;
3293             ppc440spe_rxor_set_region(desc,
3294                 cursor->addr_count,
3295                 DMA_RXOR12 << DMA_CUED_REGION_OFF);
3296             ppc440spe_adma_dma2rxor_inc_addr(
3297                 desc, cursor, index, src_cnt);
3298         } else if (addr == cursor->addrl + 2*sign*cursor->len) {
3299             cursor->state = 2;
3300             cursor->xor_count = 0;
3301             ppc440spe_rxor_set_region(desc,
3302                 cursor->addr_count,
3303                 DMA_RXOR123 << DMA_CUED_REGION_OFF);
3304             if (index == src_cnt-1) {
3305                 ppc440spe_adma_dma2rxor_inc_addr(
3306                     desc, cursor, index, src_cnt);
3307             }
3308         } else if (addr == cursor->addrl + 3*cursor->len) {
3309             cursor->state = 2;
3310             cursor->xor_count = 0;
3311             ppc440spe_rxor_set_region(desc,
3312                 cursor->addr_count,
3313                 DMA_RXOR124 << DMA_CUED_REGION_OFF);
3314             if (index == src_cnt-1) {
3315                 ppc440spe_adma_dma2rxor_inc_addr(
3316                     desc, cursor, index, src_cnt);
3317             }
3318         } else if (addr == cursor->addrl + 4*cursor->len) {
3319             cursor->state = 2;
3320             cursor->xor_count = 0;
3321             ppc440spe_rxor_set_region(desc,
3322                 cursor->addr_count,
3323                 DMA_RXOR125 << DMA_CUED_REGION_OFF);
3324             if (index == src_cnt-1) {
3325                 ppc440spe_adma_dma2rxor_inc_addr(
3326                     desc, cursor, index, src_cnt);
3327             }
3328         } else {
3329             cursor->state = 0;
3330             cursor->xor_count = 1;
3331             cursor->addrl = addr;
3332             ppc440spe_rxor_set_region(desc,
3333                 cursor->addr_count,
3334                 DMA_RXOR12 << DMA_CUED_REGION_OFF);
3335             ppc440spe_adma_dma2rxor_inc_addr(
3336                 desc, cursor, index, src_cnt);
3337         }
3338         break;
3339     case 2:
3340         cursor->state = 0;
3341         cursor->addrl = addr;
3342         cursor->xor_count++;
3343         if (index) {
3344             ppc440spe_adma_dma2rxor_inc_addr(
3345                 desc, cursor, index, src_cnt);
3346         }
3347         break;
3348     }
3349
3350     return 0;
3351 }
3352
3353 /**
3354  * ppc440spe_adma_dma2rxor_set_src - set RXOR source address; it's assumed that
3355  *  ppc440spe_adma_dma2rxor_prep_src() has already done prior this call
3356  */
3357 static void ppc440spe_adma_dma2rxor_set_src(
3358         struct ppc440spe_adma_desc_slot *desc,
3359         int index, dma_addr_t addr)
3360 {
3361     struct xor_cb *xcb = desc->hw_desc;
3362     int k = 0, op = 0, lop = 0;
3363
3364     /* get the RXOR operand which corresponds to index addr */
3365     while (op <= index) {
3366         lop = op;
3367         if (k == XOR_MAX_OPS) {
3368             k = 0;
3369             desc = list_entry(desc->chain_node.next,
3370                 struct ppc440spe_adma_desc_slot, chain_node);
3371             xcb = desc->hw_desc;
3372
3373         }
3374         if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) ==
3375             (DMA_RXOR12 << DMA_CUED_REGION_OFF))
3376             op += 2;
3377         else
3378             op += 3;
3379     }
3380
3381     BUG_ON(k < 1);
3382
3383     if (test_bit(k-1, desc->reverse_flags)) {
3384         /* reverse operand order; put last op in RXOR group */
3385         if (index == op - 1)
3386             ppc440spe_rxor_set_src(desc, k - 1, addr);
3387     } else {
3388         /* direct operand order; put first op in RXOR group */
3389         if (index == lop)
3390             ppc440spe_rxor_set_src(desc, k - 1, addr);
3391     }
3392 }
3393
3394 /**
3395  * ppc440spe_adma_dma2rxor_set_mult - set RXOR multipliers; it's assumed that
3396  *  ppc440spe_adma_dma2rxor_prep_src() has already done prior this call
3397  */
3398 static void ppc440spe_adma_dma2rxor_set_mult(
3399         struct ppc440spe_adma_desc_slot *desc,
3400         int index, u8 mult)
3401 {
3402     struct xor_cb *xcb = desc->hw_desc;
3403     int k = 0, op = 0, lop = 0;
3404
3405     /* get the RXOR operand which corresponds to index mult */
3406     while (op <= index) {
3407         lop = op;
3408         if (k == XOR_MAX_OPS) {
3409             k = 0;
3410             desc = list_entry(desc->chain_node.next,
3411                       struct ppc440spe_adma_desc_slot,
3412                       chain_node);
3413             xcb = desc->hw_desc;
3414
3415         }
3416         if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) ==
3417             (DMA_RXOR12 << DMA_CUED_REGION_OFF))
3418             op += 2;
3419         else
3420             op += 3;
3421     }
3422
3423     BUG_ON(k < 1);
3424     if (test_bit(k-1, desc->reverse_flags)) {
3425         /* reverse order */
3426         ppc440spe_rxor_set_mult(desc, k - 1, op - index - 1, mult);
3427     } else {
3428         /* direct order */
3429         ppc440spe_rxor_set_mult(desc, k - 1, index - lop, mult);
3430     }
3431 }
3432
3433 /**
3434  * ppc440spe_init_rxor_cursor -
3435  */
3436 static void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor)
3437 {
3438     memset(cursor, 0, sizeof(struct ppc440spe_rxor));
3439     cursor->state = 2;
3440 }
3441
3442 /**
3443  * ppc440spe_adma_pq_set_src_mult - set multiplication coefficient into
3444  * descriptor for the PQXOR operation
3445  */
3446 static void ppc440spe_adma_pq_set_src_mult(
3447         struct ppc440spe_adma_desc_slot *sw_desc,
3448         unsigned char mult, int index, int dst_pos)
3449 {
3450     struct ppc440spe_adma_chan *chan;
3451     u32 mult_idx, mult_dst;
3452     struct ppc440spe_adma_desc_slot *iter = NULL, *iter1 = NULL;
3453
3454     chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3455
3456     switch (chan->device->id) {
3457     case PPC440SPE_DMA0_ID:
3458     case PPC440SPE_DMA1_ID:
3459         if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
3460             int region = test_bit(PPC440SPE_DESC_RXOR12,
3461                     &sw_desc->flags) ? 2 : 3;
3462
3463             if (index < region) {
3464                 /* RXOR multipliers */
3465                 iter = ppc440spe_get_group_entry(sw_desc,
3466                     sw_desc->dst_cnt - 1);
3467                 if (sw_desc->dst_cnt == 2)
3468                     iter1 = ppc440spe_get_group_entry(
3469                             sw_desc, 0);
3470
3471                 mult_idx = DMA_CUED_MULT1_OFF + (index << 3);
3472                 mult_dst = DMA_CDB_SG_SRC;
3473             } else {
3474                 /* WXOR multiplier */
3475                 iter = ppc440spe_get_group_entry(sw_desc,
3476                             index - region +
3477                             sw_desc->dst_cnt);
3478                 mult_idx = DMA_CUED_MULT1_OFF;
3479                 mult_dst = dst_pos ? DMA_CDB_SG_DST2 :
3480                              DMA_CDB_SG_DST1;
3481             }
3482         } else {
3483             int znum = 0;
3484
3485             /* WXOR-only;
3486              * skip first slots with destinations (if ZERO_DST has
3487              * place)
3488              */
3489             if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
3490                 znum++;
3491             if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
3492                 znum++;
3493
3494             iter = ppc440spe_get_group_entry(sw_desc, index + znum);
3495             mult_idx = DMA_CUED_MULT1_OFF;
3496             mult_dst = dst_pos ? DMA_CDB_SG_DST2 : DMA_CDB_SG_DST1;
3497         }
3498
3499         if (likely(iter)) {
3500             ppc440spe_desc_set_src_mult(iter, chan,
3501                 mult_idx, mult_dst, mult);
3502
3503             if (unlikely(iter1)) {
3504                 /* if we have two destinations for RXOR, then
3505                  * we've just set Q mult. Set-up P now.
3506                  */
3507                 ppc440spe_desc_set_src_mult(iter1, chan,
3508                     mult_idx, mult_dst, 1);
3509             }
3510
3511         }
3512         break;
3513
3514     case PPC440SPE_XOR_ID:
3515         iter = sw_desc->group_head;
3516         if (sw_desc->dst_cnt == 2) {
3517             /* both P & Q calculations required; set P mult here */
3518             ppc440spe_adma_dma2rxor_set_mult(iter, index, 1);
3519
3520             /* and then set Q mult */
3521             iter = ppc440spe_get_group_entry(sw_desc,
3522                    sw_desc->descs_per_op);
3523         }
3524         ppc440spe_adma_dma2rxor_set_mult(iter, index, mult);
3525         break;
3526     }
3527 }
3528
3529 /**
3530  * ppc440spe_adma_free_chan_resources - free the resources allocated
3531  */
3532 static void ppc440spe_adma_free_chan_resources(struct dma_chan *chan)
3533 {
3534     struct ppc440spe_adma_chan *ppc440spe_chan;
3535     struct ppc440spe_adma_desc_slot *iter, *_iter;
3536     int in_use_descs = 0;
3537
3538     ppc440spe_chan = to_ppc440spe_adma_chan(chan);
3539     ppc440spe_adma_slot_cleanup(ppc440spe_chan);
3540
3541     spin_lock_bh(&ppc440spe_chan->lock);
3542     list_for_each_entry_safe(iter, _iter, &ppc440spe_chan->chain,
3543                     chain_node) {
3544         in_use_descs++;
3545         list_del(&iter->chain_node);
3546     }
3547     list_for_each_entry_safe_reverse(iter, _iter,
3548             &ppc440spe_chan->all_slots, slot_node) {
3549         list_del(&iter->slot_node);
3550         kfree(iter);
3551         ppc440spe_chan->slots_allocated--;
3552     }
3553     ppc440spe_chan->last_used = NULL;
3554
3555     dev_dbg(ppc440spe_chan->device->common.dev,
3556         "ppc440spe adma%d %s slots_allocated %d\n",
3557         ppc440spe_chan->device->id,
3558         __func__, ppc440spe_chan->slots_allocated);
3559     spin_unlock_bh(&ppc440spe_chan->lock);
3560
3561     /* one is ok since we left it on there on purpose */
3562     if (in_use_descs > 1)
3563         printk(KERN_ERR "SPE: Freeing %d in use descriptors!\n",
3564             in_use_descs - 1);
3565 }
3566
3567 /**
3568  * ppc440spe_adma_tx_status - poll the status of an ADMA transaction
3569  * @chan: ADMA channel handle
3570  * @cookie: ADMA transaction identifier
3571  * @txstate: a holder for the current state of the channel
3572  */
3573 static enum dma_status ppc440spe_adma_tx_status(struct dma_chan *chan,
3574             dma_cookie_t cookie, struct dma_tx_state *txstate)
3575 {
3576     struct ppc440spe_adma_chan *ppc440spe_chan;
3577     enum dma_status ret;
3578
3579     ppc440spe_chan = to_ppc440spe_adma_chan(chan);
3580     ret = dma_cookie_status(chan, cookie, txstate);
3581     if (ret == DMA_COMPLETE)
3582         return ret;
3583
3584     ppc440spe_adma_slot_cleanup(ppc440spe_chan);
3585
3586     return dma_cookie_status(chan, cookie, txstate);
3587 }
3588
3589 /**
3590  * ppc440spe_adma_eot_handler - end of transfer interrupt handler
3591  */
3592 static irqreturn_t ppc440spe_adma_eot_handler(int irq, void *data)
3593 {
3594     struct ppc440spe_adma_chan *chan = data;
3595
3596     dev_dbg(chan->device->common.dev,
3597         "ppc440spe adma%d: %s\n", chan->device->id, __func__);
3598
3599     tasklet_schedule(&chan->irq_tasklet);
3600     ppc440spe_adma_device_clear_eot_status(chan);
3601
3602     return IRQ_HANDLED;
3603 }
3604
3605 /**
3606  * ppc440spe_adma_err_handler - DMA error interrupt handler;
3607  *  do the same things as a eot handler
3608  */
3609 static irqreturn_t ppc440spe_adma_err_handler(int irq, void *data)
3610 {
3611     struct ppc440spe_adma_chan *chan = data;
3612
3613     dev_dbg(chan->device->common.dev,
3614         "ppc440spe adma%d: %s\n", chan->device->id, __func__);
3615
3616     tasklet_schedule(&chan->irq_tasklet);
3617     ppc440spe_adma_device_clear_eot_status(chan);
3618
3619     return IRQ_HANDLED;
3620 }
3621
3622 /**
3623  * ppc440spe_test_callback - called when test operation has been done
3624  */
3625 static void ppc440spe_test_callback(void *unused)
3626 {
3627     complete(&ppc440spe_r6_test_comp);
3628 }
3629
3630 /**
3631  * ppc440spe_adma_issue_pending - flush all pending descriptors to h/w
3632  */
3633 static void ppc440spe_adma_issue_pending(struct dma_chan *chan)
3634 {
3635     struct ppc440spe_adma_chan *ppc440spe_chan;
3636
3637     ppc440spe_chan = to_ppc440spe_adma_chan(chan);
3638     dev_dbg(ppc440spe_chan->device->common.dev,
3639         "ppc440spe adma%d: %s %d \n", ppc440spe_chan->device->id,
3640         __func__, ppc440spe_chan->pending);
3641
3642     if (ppc440spe_chan->pending) {
3643         ppc440spe_chan->pending = 0;
3644         ppc440spe_chan_append(ppc440spe_chan);
3645     }
3646 }
3647
3648 /**
3649  * ppc440spe_chan_start_null_xor - initiate the first XOR operation (DMA engines
3650  *  use FIFOs (as opposite to chains used in XOR) so this is a XOR
3651  *  specific operation)
3652  */
3653 static void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan)
3654 {
3655     struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
3656     dma_cookie_t cookie;
3657     int slot_cnt, slots_per_op;
3658
3659     dev_dbg(chan->device->common.dev,
3660         "ppc440spe adma%d: %s\n", chan->device->id, __func__);
3661
3662     spin_lock_bh(&chan->lock);
3663     slot_cnt = ppc440spe_chan_xor_slot_count(0, 2, &slots_per_op);
3664     sw_desc = ppc440spe_adma_alloc_slots(chan, slot_cnt, slots_per_op);
3665     if (sw_desc) {
3666         group_start = sw_desc->group_head;
3667         list_splice_init(&sw_desc->group_list, &chan->chain);
3668         async_tx_ack(&sw_desc->async_tx);
3669         ppc440spe_desc_init_null_xor(group_start);
3670
3671         cookie = dma_cookie_assign(&sw_desc->async_tx);
3672
3673         /* initialize the completed cookie to be less than
3674          * the most recently used cookie
3675          */
3676         chan->common.completed_cookie = cookie - 1;
3677
3678         /* channel should not be busy */
3679         BUG_ON(ppc440spe_chan_is_busy(chan));
3680
3681         /* set the descriptor address */
3682         ppc440spe_chan_set_first_xor_descriptor(chan, sw_desc);
3683
3684         /* run the descriptor */
3685         ppc440spe_chan_run(chan);
3686     } else
3687         printk(KERN_ERR "ppc440spe adma%d"
3688             " failed to allocate null descriptor\n",
3689             chan->device->id);
3690     spin_unlock_bh(&chan->lock);
3691 }
3692
3693 /**
3694  * ppc440spe_test_raid6 - test are RAID-6 capabilities enabled successfully.
3695  *  For this we just perform one WXOR operation with the same source
3696  *  and destination addresses, the GF-multiplier is 1; so if RAID-6
3697  *  capabilities are enabled then we'll get src/dst filled with zero.
3698  */
3699 static int ppc440spe_test_raid6(struct ppc440spe_adma_chan *chan)
3700 {
3701     struct ppc440spe_adma_desc_slot *sw_desc, *iter;
3702     struct page *pg;
3703     char *a;
3704     dma_addr_t dma_addr, addrs[2];
3705     unsigned long op = 0;
3706     int rval = 0;
3707
3708     set_bit(PPC440SPE_DESC_WXOR, &op);
3709
3710     pg = alloc_page(GFP_KERNEL);
3711     if (!pg)
3712         return -ENOMEM;
3713
3714     spin_lock_bh(&chan->lock);
3715     sw_desc = ppc440spe_adma_alloc_slots(chan, 1, 1);
3716     if (sw_desc) {
3717         /* 1 src, 1 dsr, int_ena, WXOR */
3718         ppc440spe_desc_init_dma01pq(sw_desc, 1, 1, 1, op);
3719         list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
3720             ppc440spe_desc_set_byte_count(iter, chan, PAGE_SIZE);
3721             iter->unmap_len = PAGE_SIZE;
3722         }
3723     } else {
3724         rval = -EFAULT;
3725         spin_unlock_bh(&chan->lock);
3726         goto exit;
3727     }
3728     spin_unlock_bh(&chan->lock);
3729
3730     /* Fill the test page with ones */
3731     memset(page_address(pg), 0xFF, PAGE_SIZE);
3732     dma_addr = dma_map_page(chan->device->dev, pg, 0,
3733                 PAGE_SIZE, DMA_BIDIRECTIONAL);
3734
3735     /* Setup addresses */
3736     ppc440spe_adma_pq_set_src(sw_desc, dma_addr, 0);
3737     ppc440spe_adma_pq_set_src_mult(sw_desc, 1, 0, 0);
3738     addrs[0] = dma_addr;
3739     addrs[1] = 0;
3740     ppc440spe_adma_pq_set_dest(sw_desc, addrs, DMA_PREP_PQ_DISABLE_Q);
3741
3742     async_tx_ack(&sw_desc->async_tx);
3743     sw_desc->async_tx.callback = ppc440spe_test_callback;
3744     sw_desc->async_tx.callback_param = NULL;
3745
3746     init_completion(&ppc440spe_r6_test_comp);
3747
3748     ppc440spe_adma_tx_submit(&sw_desc->async_tx);
3749     ppc440spe_adma_issue_pending(&chan->common);
3750
3751     wait_for_completion(&ppc440spe_r6_test_comp);
3752
3753     /* Now check if the test page is zeroed */
3754     a = page_address(pg);
3755     if ((*(u32 *)a) == 0 && memcmp(a, a+4, PAGE_SIZE-4) == 0) {
3756         /* page is zero - RAID-6 enabled */
3757         rval = 0;
3758     } else {
3759         /* RAID-6 was not enabled */
3760         rval = -EINVAL;
3761     }
3762 exit:
3763     __free_page(pg);
3764     return rval;
3765 }
3766
3767 static void ppc440spe_adma_init_capabilities(struct ppc440spe_adma_device *adev)
3768 {
3769     switch (adev->id) {
3770     case PPC440SPE_DMA0_ID:
3771     case PPC440SPE_DMA1_ID:
3772         dma_cap_set(DMA_MEMCPY, adev->common.cap_mask);
3773         dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask);
3774         dma_cap_set(DMA_PQ, adev->common.cap_mask);
3775         dma_cap_set(DMA_PQ_VAL, adev->common.cap_mask);
3776         dma_cap_set(DMA_XOR_VAL, adev->common.cap_mask);
3777         break;
3778     case PPC440SPE_XOR_ID:
3779         dma_cap_set(DMA_XOR, adev->common.cap_mask);
3780         dma_cap_set(DMA_PQ, adev->common.cap_mask);
3781         dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask);
3782         adev->common.cap_mask = adev->common.cap_mask;
3783         break;
3784     }
3785
3786     /* Set base routines */
3787     adev->common.device_alloc_chan_resources =
3788                 ppc440spe_adma_alloc_chan_resources;
3789     adev->common.device_free_chan_resources =
3790                 ppc440spe_adma_free_chan_resources;
3791     adev->common.device_tx_status = ppc440spe_adma_tx_status;
3792     adev->common.device_issue_pending = ppc440spe_adma_issue_pending;
3793
3794     /* Set prep routines based on capability */
3795     if (dma_has_cap(DMA_MEMCPY, adev->common.cap_mask)) {
3796         adev->common.device_prep_dma_memcpy =
3797             ppc440spe_adma_prep_dma_memcpy;
3798     }
3799     if (dma_has_cap(DMA_XOR, adev->common.cap_mask)) {
3800         adev->common.max_xor = XOR_MAX_OPS;
3801         adev->common.device_prep_dma_xor =
3802             ppc440spe_adma_prep_dma_xor;
3803     }
3804     if (dma_has_cap(DMA_PQ, adev->common.cap_mask)) {
3805         switch (adev->id) {
3806         case PPC440SPE_DMA0_ID:
3807             dma_set_maxpq(&adev->common,
3808                 DMA0_FIFO_SIZE / sizeof(struct dma_cdb), 0);
3809             break;
3810         case PPC440SPE_DMA1_ID:
3811             dma_set_maxpq(&adev->common,
3812                 DMA1_FIFO_SIZE / sizeof(struct dma_cdb), 0);
3813             break;
3814         case PPC440SPE_XOR_ID:
3815             adev->common.max_pq = XOR_MAX_OPS * 3;
3816             break;
3817         }
3818         adev->common.device_prep_dma_pq =
3819             ppc440spe_adma_prep_dma_pq;
3820     }
3821     if (dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask)) {
3822         switch (adev->id) {
3823         case PPC440SPE_DMA0_ID:
3824             adev->common.max_pq = DMA0_FIFO_SIZE /
3825                         sizeof(struct dma_cdb);
3826             break;
3827         case PPC440SPE_DMA1_ID:
3828             adev->common.max_pq = DMA1_FIFO_SIZE /
3829                         sizeof(struct dma_cdb);
3830             break;
3831         }
3832         adev->common.device_prep_dma_pq_val =
3833             ppc440spe_adma_prep_dma_pqzero_sum;
3834     }
3835     if (dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask)) {
3836         switch (adev->id) {
3837         case PPC440SPE_DMA0_ID:
3838             adev->common.max_xor = DMA0_FIFO_SIZE /
3839                         sizeof(struct dma_cdb);
3840             break;
3841         case PPC440SPE_DMA1_ID:
3842             adev->common.max_xor = DMA1_FIFO_SIZE /
3843                         sizeof(struct dma_cdb);
3844             break;
3845         }
3846         adev->common.device_prep_dma_xor_val =
3847             ppc440spe_adma_prep_dma_xor_zero_sum;
3848     }
3849     if (dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask)) {
3850         adev->common.device_prep_dma_interrupt =
3851             ppc440spe_adma_prep_dma_interrupt;
3852     }
3853     pr_info("%s: AMCC(R) PPC440SP(E) ADMA Engine: "
3854       "( %s%s%s%s%s%s)\n",
3855       dev_name(adev->dev),
3856       dma_has_cap(DMA_PQ, adev->common.cap_mask) ? "pq " : "",
3857       dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask) ? "pq_val " : "",
3858       dma_has_cap(DMA_XOR, adev->common.cap_mask) ? "xor " : "",
3859       dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask) ? "xor_val " : "",
3860       dma_has_cap(DMA_MEMCPY, adev->common.cap_mask) ? "memcpy " : "",
3861       dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask) ? "intr " : "");
3862 }
3863
3864 static int ppc440spe_adma_setup_irqs(struct ppc440spe_adma_device *adev,
3865                      struct ppc440spe_adma_chan *chan,
3866                      int *initcode)
3867 {
3868     struct platform_device *ofdev;
3869     struct device_node *np;
3870     int ret;
3871
3872     ofdev = container_of(adev->dev, struct platform_device, dev);
3873     np = ofdev->dev.of_node;
3874     if (adev->id != PPC440SPE_XOR_ID) {
3875         adev->err_irq = irq_of_parse_and_map(np, 1);
3876         if (!adev->err_irq) {
3877             dev_warn(adev->dev, "no err irq resource?\n");
3878             *initcode = PPC_ADMA_INIT_IRQ2;
3879             adev->err_irq = -ENXIO;
3880         } else
3881             atomic_inc(&ppc440spe_adma_err_irq_ref);
3882     } else {
3883         adev->err_irq = -ENXIO;
3884     }
3885
3886     adev->irq = irq_of_parse_and_map(np, 0);
3887     if (!adev->irq) {
3888         dev_err(adev->dev, "no irq resource\n");
3889         *initcode = PPC_ADMA_INIT_IRQ1;
3890         ret = -ENXIO;
3891         goto err_irq_map;
3892     }
3893     dev_dbg(adev->dev, "irq %d, err irq %d\n",
3894         adev->irq, adev->err_irq);
3895
3896     ret = request_irq(adev->irq, ppc440spe_adma_eot_handler,
3897               0, dev_driver_string(adev->dev), chan);
3898     if (ret) {
3899         dev_err(adev->dev, "can't request irq %d\n",
3900             adev->irq);
3901         *initcode = PPC_ADMA_INIT_IRQ1;
3902         ret = -EIO;
3903         goto err_req1;
3904     }
3905
3906     /* only DMA engines have a separate error IRQ
3907      * so it's Ok if err_irq < 0 in XOR engine case.
3908      */
3909     if (adev->err_irq > 0) {
3910         /* both DMA engines share common error IRQ */
3911         ret = request_irq(adev->err_irq,
3912                   ppc440spe_adma_err_handler,
3913                   IRQF_SHARED,
3914                   dev_driver_string(adev->dev),
3915                   chan);
3916         if (ret) {
3917             dev_err(adev->dev, "can't request irq %d\n",
3918                 adev->err_irq);
3919             *initcode = PPC_ADMA_INIT_IRQ2;
3920             ret = -EIO;
3921             goto err_req2;
3922         }
3923     }
3924
3925     if (adev->id == PPC440SPE_XOR_ID) {
3926         /* enable XOR engine interrupts */
3927         iowrite32be(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT |
3928                 XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT,
3929                 &adev->xor_reg->ier);
3930     } else {
3931         u32 mask, enable;
3932
3933         np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe");
3934         if (!np) {
3935             pr_err("%s: can't find I2O device tree node\n",
3936                 __func__);
3937             ret = -ENODEV;
3938             goto err_req2;
3939         }
3940         adev->i2o_reg = of_iomap(np, 0);
3941         if (!adev->i2o_reg) {
3942             pr_err("%s: failed to map I2O registers\n", __func__);
3943             of_node_put(np);
3944             ret = -EINVAL;
3945             goto err_req2;
3946         }
3947         of_node_put(np);
3948         /* Unmask 'CS FIFO Attention' interrupts and
3949          * enable generating interrupts on errors
3950          */
3951         enable = (adev->id == PPC440SPE_DMA0_ID) ?
3952              ~(I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) :
3953              ~(I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM);
3954         mask = ioread32(&adev->i2o_reg->iopim) & enable;
3955         iowrite32(mask, &adev->i2o_reg->iopim);
3956     }
3957     return 0;
3958
3959 err_req2:
3960     free_irq(adev->irq, chan);
3961 err_req1:
3962     irq_dispose_mapping(adev->irq);
3963 err_irq_map:
3964     if (adev->err_irq > 0) {
3965         if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref))
3966             irq_dispose_mapping(adev->err_irq);
3967     }
3968     return ret;
3969 }
3970
3971 static void ppc440spe_adma_release_irqs(struct ppc440spe_adma_device *adev,
3972                     struct ppc440spe_adma_chan *chan)
3973 {
3974     u32 mask, disable;
3975
3976     if (adev->id == PPC440SPE_XOR_ID) {
3977         /* disable XOR engine interrupts */
3978         mask = ioread32be(&adev->xor_reg->ier);
3979         mask &= ~(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT |
3980               XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT);
3981         iowrite32be(mask, &adev->xor_reg->ier);
3982     } else {
3983         /* disable DMAx engine interrupts */
3984         disable = (adev->id == PPC440SPE_DMA0_ID) ?
3985               (I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) :
3986               (I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM);
3987         mask = ioread32(&adev->i2o_reg->iopim) | disable;
3988         iowrite32(mask, &adev->i2o_reg->iopim);
3989     }
3990     free_irq(adev->irq, chan);
3991     irq_dispose_mapping(adev->irq);
3992     if (adev->err_irq > 0) {
3993         free_irq(adev->err_irq, chan);
3994         if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref)) {
3995             irq_dispose_mapping(adev->err_irq);
3996             iounmap(adev->i2o_reg);
3997         }
3998     }
3999 }
4000
4001 /**
4002  * ppc440spe_adma_probe - probe the asynch device
4003  */
4004 static int ppc440spe_adma_probe(struct platform_device *ofdev)
4005 {
4006     struct device_node *np = ofdev->dev.of_node;
4007     struct resource res;
4008     struct ppc440spe_adma_device *adev;
4009     struct ppc440spe_adma_chan *chan;
4010     struct ppc_dma_chan_ref *ref, *_ref;
4011     int ret = 0, initcode = PPC_ADMA_INIT_OK;
4012     const u32 *idx;
4013     int len;
4014     void *regs;
4015     u32 id, pool_size;
4016
4017     if (of_device_is_compatible(np, "amcc,xor-accelerator")) {
4018         id = PPC440SPE_XOR_ID;
4019         /* As far as the XOR engine is concerned, it does not
4020          * use FIFOs but uses linked list. So there is no dependency
4021          * between pool size to allocate and the engine configuration.
4022          */
4023         pool_size = PAGE_SIZE << 1;
4024     } else {
4025         /* it is DMA0 or DMA1 */
4026         idx = of_get_property(np, "cell-index", &len);
4027         if (!idx || (len != sizeof(u32))) {
4028             dev_err(&ofdev->dev, "Device node %pOF has missing "
4029                 "or invalid cell-index property\n",
4030                 np);
4031             return -EINVAL;
4032         }
4033         id = *idx;
4034         /* DMA0,1 engines use FIFO to maintain CDBs, so we
4035          * should allocate the pool accordingly to size of this
4036          * FIFO. Thus, the pool size depends on the FIFO depth:
4037          * how much CDBs pointers the FIFO may contain then so
4038          * much CDBs we should provide in the pool.
4039          * That is
4040          *   CDB size = 32B;
4041          *   CDBs number = (DMA0_FIFO_SIZE >> 3);
4042          *   Pool size = CDBs number * CDB size =
4043          *      = (DMA0_FIFO_SIZE >> 3) << 5 = DMA0_FIFO_SIZE << 2.
4044          */
4045         pool_size = (id == PPC440SPE_DMA0_ID) ?
4046                 DMA0_FIFO_SIZE : DMA1_FIFO_SIZE;
4047         pool_size <<= 2;
4048     }
4049
4050     if (of_address_to_resource(np, 0, &res)) {
4051         dev_err(&ofdev->dev, "failed to get memory resource\n");
4052         initcode = PPC_ADMA_INIT_MEMRES;
4053         ret = -ENODEV;
4054         goto out;
4055     }
4056
4057     if (!request_mem_region(res.start, resource_size(&res),
4058                 dev_driver_string(&ofdev->dev))) {
4059         dev_err(&ofdev->dev, "failed to request memory region %pR\n",
4060             &res);
4061         initcode = PPC_ADMA_INIT_MEMREG;
4062         ret = -EBUSY;
4063         goto out;
4064     }
4065
4066     /* create a device */
4067     adev = kzalloc(sizeof(*adev), GFP_KERNEL);
4068     if (!adev) {
4069         initcode = PPC_ADMA_INIT_ALLOC;
4070         ret = -ENOMEM;
4071         goto err_adev_alloc;
4072     }
4073
4074     adev->id = id;
4075     adev->pool_size = pool_size;
4076     /* allocate coherent memory for hardware descriptors */
4077     adev->dma_desc_pool_virt = dma_alloc_coherent(&ofdev->dev,
4078                     adev->pool_size, &adev->dma_desc_pool,
4079                     GFP_KERNEL);
4080     if (adev->dma_desc_pool_virt == NULL) {
4081         dev_err(&ofdev->dev, "failed to allocate %d bytes of coherent "
4082             "memory for hardware descriptors\n",
4083             adev->pool_size);
4084         initcode = PPC_ADMA_INIT_COHERENT;
4085         ret = -ENOMEM;
4086         goto err_dma_alloc;
4087     }
4088     dev_dbg(&ofdev->dev, "allocated descriptor pool virt 0x%p phys 0x%llx\n",
4089         adev->dma_desc_pool_virt, (u64)adev->dma_desc_pool);
4090
4091     regs = ioremap(res.start, resource_size(&res));
4092     if (!regs) {
4093         dev_err(&ofdev->dev, "failed to ioremap regs!\n");
4094         ret = -ENOMEM;
4095         goto err_regs_alloc;
4096     }
4097
4098     if (adev->id == PPC440SPE_XOR_ID) {
4099         adev->xor_reg = regs;
4100         /* Reset XOR */
4101         iowrite32be(XOR_CRSR_XASR_BIT, &adev->xor_reg->crsr);
4102         iowrite32be(XOR_CRSR_64BA_BIT, &adev->xor_reg->crrr);
4103     } else {
4104         size_t fifo_size = (adev->id == PPC440SPE_DMA0_ID) ?
4105                    DMA0_FIFO_SIZE : DMA1_FIFO_SIZE;
4106         adev->dma_reg = regs;
4107         /* DMAx_FIFO_SIZE is defined in bytes,
4108          * <fsiz> - is defined in number of CDB pointers (8byte).
4109          * DMA FIFO Length = CSlength + CPlength, where
4110          * CSlength = CPlength = (fsiz + 1) * 8.
4111          */
4112         iowrite32(DMA_FIFO_ENABLE | ((fifo_size >> 3) - 2),
4113               &adev->dma_reg->fsiz);
4114         /* Configure DMA engine */
4115         iowrite32(DMA_CFG_DXEPR_HP | DMA_CFG_DFMPP_HP | DMA_CFG_FALGN,
4116               &adev->dma_reg->cfg);
4117         /* Clear Status */
4118         iowrite32(~0, &adev->dma_reg->dsts);
4119     }
4120
4121     adev->dev = &ofdev->dev;
4122     adev->common.dev = &ofdev->dev;
4123     INIT_LIST_HEAD(&adev->common.channels);
4124     platform_set_drvdata(ofdev, adev);
4125
4126     /* create a channel */
4127     chan = kzalloc(sizeof(*chan), GFP_KERNEL);
4128     if (!chan) {
4129         initcode = PPC_ADMA_INIT_CHANNEL;
4130         ret = -ENOMEM;
4131         goto err_chan_alloc;
4132     }
4133
4134     spin_lock_init(&chan->lock);
4135     INIT_LIST_HEAD(&chan->chain);
4136     INIT_LIST_HEAD(&chan->all_slots);
4137     chan->device = adev;
4138     chan->common.device = &adev->common;
4139     dma_cookie_init(&chan->common);
4140     list_add_tail(&chan->common.device_node, &adev->common.channels);
4141     tasklet_setup(&chan->irq_tasklet, ppc440spe_adma_tasklet);
4142
4143     /* allocate and map helper pages for async validation or
4144      * async_mult/async_sum_product operations on DMA0/1.
4145      */
4146     if (adev->id != PPC440SPE_XOR_ID) {
4147         chan->pdest_page = alloc_page(GFP_KERNEL);
4148         chan->qdest_page = alloc_page(GFP_KERNEL);
4149         if (!chan->pdest_page ||
4150             !chan->qdest_page) {
4151             if (chan->pdest_page)
4152                 __free_page(chan->pdest_page);
4153             if (chan->qdest_page)
4154                 __free_page(chan->qdest_page);
4155             ret = -ENOMEM;
4156             goto err_page_alloc;
4157         }
4158         chan->pdest = dma_map_page(&ofdev->dev, chan->pdest_page, 0,
4159                        PAGE_SIZE, DMA_BIDIRECTIONAL);
4160         chan->qdest = dma_map_page(&ofdev->dev, chan->qdest_page, 0,
4161                        PAGE_SIZE, DMA_BIDIRECTIONAL);
4162     }
4163
4164     ref = kmalloc(sizeof(*ref), GFP_KERNEL);
4165     if (ref) {
4166         ref->chan = &chan->common;
4167         INIT_LIST_HEAD(&ref->node);
4168         list_add_tail(&ref->node, &ppc440spe_adma_chan_list);
4169     } else {
4170         dev_err(&ofdev->dev, "failed to allocate channel reference!\n");
4171         ret = -ENOMEM;
4172         goto err_ref_alloc;
4173     }
4174
4175     ret = ppc440spe_adma_setup_irqs(adev, chan, &initcode);
4176     if (ret)
4177         goto err_irq;
4178
4179     ppc440spe_adma_init_capabilities(adev);
4180
4181     ret = dma_async_device_register(&adev->common);
4182     if (ret) {
4183         initcode = PPC_ADMA_INIT_REGISTER;
4184         dev_err(&ofdev->dev, "failed to register dma device\n");
4185         goto err_dev_reg;
4186     }
4187
4188     goto out;
4189
4190 err_dev_reg:
4191     ppc440spe_adma_release_irqs(adev, chan);
4192 err_irq:
4193     list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list, node) {
4194         if (chan == to_ppc440spe_adma_chan(ref->chan)) {
4195             list_del(&ref->node);
4196             kfree(ref);
4197         }
4198     }
4199 err_ref_alloc:
4200     if (adev->id != PPC440SPE_XOR_ID) {
4201         dma_unmap_page(&ofdev->dev, chan->pdest,
4202                    PAGE_SIZE, DMA_BIDIRECTIONAL);
4203         dma_unmap_page(&ofdev->dev, chan->qdest,
4204                    PAGE_SIZE, DMA_BIDIRECTIONAL);
4205         __free_page(chan->pdest_page);
4206         __free_page(chan->qdest_page);
4207     }
4208 err_page_alloc:
4209     kfree(chan);
4210 err_chan_alloc:
4211     if (adev->id == PPC440SPE_XOR_ID)
4212         iounmap(adev->xor_reg);
4213     else
4214         iounmap(adev->dma_reg);
4215 err_regs_alloc:
4216     dma_free_coherent(adev->dev, adev->pool_size,
4217               adev->dma_desc_pool_virt,
4218               adev->dma_desc_pool);
4219 err_dma_alloc:
4220     kfree(adev);
4221 err_adev_alloc:
4222     release_mem_region(res.start, resource_size(&res));
4223 out:
4224     if (id < PPC440SPE_ADMA_ENGINES_NUM)
4225         ppc440spe_adma_devices[id] = initcode;
4226
4227     return ret;
4228 }
4229
4230 /**
4231  * ppc440spe_adma_remove - remove the asynch device
4232  */
4233 static int ppc440spe_adma_remove(struct platform_device *ofdev)
4234 {
4235     struct ppc440spe_adma_device *adev = platform_get_drvdata(ofdev);
4236     struct device_node *np = ofdev->dev.of_node;
4237     struct resource res;
4238     struct dma_chan *chan, *_chan;
4239     struct ppc_dma_chan_ref *ref, *_ref;
4240     struct ppc440spe_adma_chan *ppc440spe_chan;
4241
4242     if (adev->id < PPC440SPE_ADMA_ENGINES_NUM)
4243         ppc440spe_adma_devices[adev->id] = -1;
4244
4245     dma_async_device_unregister(&adev->common);
4246
4247     list_for_each_entry_safe(chan, _chan, &adev->common.channels,
4248                  device_node) {
4249         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
4250         ppc440spe_adma_release_irqs(adev, ppc440spe_chan);
4251         tasklet_kill(&ppc440spe_chan->irq_tasklet);
4252         if (adev->id != PPC440SPE_XOR_ID) {
4253             dma_unmap_page(&ofdev->dev, ppc440spe_chan->pdest,
4254                     PAGE_SIZE, DMA_BIDIRECTIONAL);
4255             dma_unmap_page(&ofdev->dev, ppc440spe_chan->qdest,
4256                     PAGE_SIZE, DMA_BIDIRECTIONAL);
4257             __free_page(ppc440spe_chan->pdest_page);
4258             __free_page(ppc440spe_chan->qdest_page);
4259         }
4260         list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list,
4261                      node) {
4262             if (ppc440spe_chan ==
4263                 to_ppc440spe_adma_chan(ref->chan)) {
4264                 list_del(&ref->node);
4265                 kfree(ref);
4266             }
4267         }
4268         list_del(&chan->device_node);
4269         kfree(ppc440spe_chan);
4270     }
4271
4272     dma_free_coherent(adev->dev, adev->pool_size,
4273               adev->dma_desc_pool_virt, adev->dma_desc_pool);
4274     if (adev->id == PPC440SPE_XOR_ID)
4275         iounmap(adev->xor_reg);
4276     else
4277         iounmap(adev->dma_reg);
4278     of_address_to_resource(np, 0, &res);
4279     release_mem_region(res.start, resource_size(&res));
4280     kfree(adev);
4281     return 0;
4282 }
4283
4284 /*
4285  * /sys driver interface to enable h/w RAID-6 capabilities
4286  * Files created in e.g. /sys/devices/plb.0/400100100.dma0/driver/
4287  * directory are "devices", "enable" and "poly".
4288  * "devices" shows available engines.
4289  * "enable" is used to enable RAID-6 capabilities or to check
4290  * whether these has been activated.
4291  * "poly" allows setting/checking used polynomial (for PPC440SPe only).
4292  */
4293
4294 static ssize_t devices_show(struct device_driver *dev, char *buf)
4295 {
4296     ssize_t size = 0;
4297     int i;
4298
4299     for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++) {
4300         if (ppc440spe_adma_devices[i] == -1)
4301             continue;
4302         size += scnprintf(buf + size, PAGE_SIZE - size,
4303                  "PPC440SP(E)-ADMA.%d: %s\n", i,
4304                  ppc_adma_errors[ppc440spe_adma_devices[i]]);
4305     }
4306     return size;
4307 }
4308 static DRIVER_ATTR_RO(devices);
4309
4310 static ssize_t enable_show(struct device_driver *dev, char *buf)
4311 {
4312     return snprintf(buf, PAGE_SIZE,
4313             "PPC440SP(e) RAID-6 capabilities are %sABLED.\n",
4314             ppc440spe_r6_enabled ? "EN" : "DIS");
4315 }
4316
4317 static ssize_t enable_store(struct device_driver *dev, const char *buf,
4318                 size_t count)
4319 {
4320     unsigned long val;
4321     int err;
4322
4323     if (!count || count > 11)
4324         return -EINVAL;
4325
4326     if (!ppc440spe_r6_tchan)
4327         return -EFAULT;
4328
4329     /* Write a key */
4330     err = kstrtoul(buf, 16, &val);
4331     if (err)
4332         return err;
4333
4334     dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_XORBA, val);
4335     isync();
4336
4337     /* Verify whether it really works now */
4338     if (ppc440spe_test_raid6(ppc440spe_r6_tchan) == 0) {
4339         pr_info("PPC440SP(e) RAID-6 has been activated "
4340             "successfully\n");
4341         ppc440spe_r6_enabled = 1;
4342     } else {
4343         pr_info("PPC440SP(e) RAID-6 hasn't been activated!"
4344             " Error key ?\n");
4345         ppc440spe_r6_enabled = 0;
4346     }
4347     return count;
4348 }
4349 static DRIVER_ATTR_RW(enable);
4350
4351 static ssize_t poly_show(struct device_driver *dev, char *buf)
4352 {
4353     ssize_t size = 0;
4354     u32 reg;
4355
4356 #ifdef CONFIG_440SP
4357     /* 440SP has fixed polynomial */
4358     reg = 0x4d;
4359 #else
4360     reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL);
4361     reg >>= MQ0_CFBHL_POLY;
4362     reg &= 0xFF;
4363 #endif
4364
4365     size = snprintf(buf, PAGE_SIZE, "PPC440SP(e) RAID-6 driver "
4366             "uses 0x1%02x polynomial.\n", reg);
4367     return size;
4368 }
4369
4370 static ssize_t poly_store(struct device_driver *dev, const char *buf,
4371               size_t count)
4372 {
4373     unsigned long reg, val;
4374     int err;
4375 #ifdef CONFIG_440SP
4376     /* 440SP uses default 0x14D polynomial only */
4377     return -EINVAL;
4378 #endif
4379
4380     if (!count || count > 6)
4381         return -EINVAL;
4382
4383     /* e.g., 0x14D or 0x11D */
4384     err = kstrtoul(buf, 16, &val);
4385     if (err)
4386         return err;
4387
4388     if (val & ~0x1FF)
4389         return -EINVAL;
4390
4391     val &= 0xFF;
4392     reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL);
4393     reg &= ~(0xFF << MQ0_CFBHL_POLY);
4394     reg |= val << MQ0_CFBHL_POLY;
4395     dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL, reg);
4396
4397     return count;
4398 }
4399 static DRIVER_ATTR_RW(poly);
4400
4401 /*
4402  * Common initialisation for RAID engines; allocate memory for
4403  * DMAx FIFOs, perform configuration common for all DMA engines.
4404  * Further DMA engine specific configuration is done at probe time.
4405  */
4406 static int ppc440spe_configure_raid_devices(void)
4407 {
4408     struct device_node *np;
4409     struct resource i2o_res;
4410     struct i2o_regs __iomem *i2o_reg;
4411     dcr_host_t i2o_dcr_host;
4412     unsigned int dcr_base, dcr_len;
4413     int i, ret;
4414
4415     np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe");
4416     if (!np) {
4417         pr_err("%s: can't find I2O device tree node\n",
4418             __func__);
4419         return -ENODEV;
4420     }
4421
4422     if (of_address_to_resource(np, 0, &i2o_res)) {
4423         of_node_put(np);
4424         return -EINVAL;
4425     }
4426
4427     i2o_reg = of_iomap(np, 0);
4428     if (!i2o_reg) {
4429         pr_err("%s: failed to map I2O registers\n", __func__);
4430         of_node_put(np);
4431         return -EINVAL;
4432     }
4433
4434     /* Get I2O DCRs base */
4435     dcr_base = dcr_resource_start(np, 0);
4436     dcr_len = dcr_resource_len(np, 0);
4437     if (!dcr_base && !dcr_len) {
4438         pr_err("%pOF: can't get DCR registers base/len!\n", np);
4439         of_node_put(np);
4440         iounmap(i2o_reg);
4441         return -ENODEV;
4442     }
4443
4444     i2o_dcr_host = dcr_map(np, dcr_base, dcr_len);
4445     if (!DCR_MAP_OK(i2o_dcr_host)) {
4446         pr_err("%pOF: failed to map DCRs!\n", np);
4447         of_node_put(np);
4448         iounmap(i2o_reg);
4449         return -ENODEV;
4450     }
4451     of_node_put(np);
4452
4453     /* Provide memory regions for DMA's FIFOs: I2O, DMA0 and DMA1 share
4454      * the base address of FIFO memory space.
4455      * Actually we need twice more physical memory than programmed in the
4456      * <fsiz> register (because there are two FIFOs for each DMA: CP and CS)
4457      */
4458     ppc440spe_dma_fifo_buf = kmalloc((DMA0_FIFO_SIZE + DMA1_FIFO_SIZE) << 1,
4459                      GFP_KERNEL);
4460     if (!ppc440spe_dma_fifo_buf) {
4461         pr_err("%s: DMA FIFO buffer allocation failed.\n", __func__);
4462         iounmap(i2o_reg);
4463         dcr_unmap(i2o_dcr_host, dcr_len);
4464         return -ENOMEM;
4465     }
4466
4467     /*
4468      * Configure h/w
4469      */
4470     /* Reset I2O/DMA */
4471     mtdcri(SDR0, DCRN_SDR0_SRST, DCRN_SDR0_SRST_I2ODMA);
4472     mtdcri(SDR0, DCRN_SDR0_SRST, 0);
4473
4474     /* Setup the base address of mmaped registers */
4475     dcr_write(i2o_dcr_host, DCRN_I2O0_IBAH, (u32)(i2o_res.start >> 32));
4476     dcr_write(i2o_dcr_host, DCRN_I2O0_IBAL, (u32)(i2o_res.start) |
4477                         I2O_REG_ENABLE);
4478     dcr_unmap(i2o_dcr_host, dcr_len);
4479
4480     /* Setup FIFO memory space base address */
4481     iowrite32(0, &i2o_reg->ifbah);
4482     iowrite32(((u32)__pa(ppc440spe_dma_fifo_buf)), &i2o_reg->ifbal);
4483
4484     /* set zero FIFO size for I2O, so the whole
4485      * ppc440spe_dma_fifo_buf is used by DMAs.
4486      * DMAx_FIFOs will be configured while probe.
4487      */
4488     iowrite32(0, &i2o_reg->ifsiz);
4489     iounmap(i2o_reg);
4490
4491     /* To prepare WXOR/RXOR functionality we need access to
4492      * Memory Queue Module DCRs (finally it will be enabled
4493      * via /sys interface of the ppc440spe ADMA driver).
4494      */
4495     np = of_find_compatible_node(NULL, NULL, "ibm,mq-440spe");
4496     if (!np) {
4497         pr_err("%s: can't find MQ device tree node\n",
4498             __func__);
4499         ret = -ENODEV;
4500         goto out_free;
4501     }
4502
4503     /* Get MQ DCRs base */
4504     dcr_base = dcr_resource_start(np, 0);
4505     dcr_len = dcr_resource_len(np, 0);
4506     if (!dcr_base && !dcr_len) {
4507         pr_err("%pOF: can't get DCR registers base/len!\n", np);
4508         ret = -ENODEV;
4509         goto out_mq;
4510     }
4511
4512     ppc440spe_mq_dcr_host = dcr_map(np, dcr_base, dcr_len);
4513     if (!DCR_MAP_OK(ppc440spe_mq_dcr_host)) {
4514         pr_err("%pOF: failed to map DCRs!\n", np);
4515         ret = -ENODEV;
4516         goto out_mq;
4517     }
4518     of_node_put(np);
4519     ppc440spe_mq_dcr_len = dcr_len;
4520
4521     /* Set HB alias */
4522     dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_BAUH, DMA_CUED_XOR_HB);
4523
4524     /* Set:
4525      * - LL transaction passing limit to 1;
4526      * - Memory controller cycle limit to 1;
4527      * - Galois Polynomial to 0x14d (default)
4528      */
4529     dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL,
4530           (1 << MQ0_CFBHL_TPLM) | (1 << MQ0_CFBHL_HBCL) |
4531           (PPC440SPE_DEFAULT_POLY << MQ0_CFBHL_POLY));
4532
4533     atomic_set(&ppc440spe_adma_err_irq_ref, 0);
4534     for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++)
4535         ppc440spe_adma_devices[i] = -1;
4536
4537     return 0;
4538
4539 out_mq:
4540     of_node_put(np);
4541 out_free:
4542     kfree(ppc440spe_dma_fifo_buf);
4543     return ret;
4544 }
4545
4546 static const struct of_device_id ppc440spe_adma_of_match[] = {
4547     { .compatible   = "ibm,dma-440spe", },
4548     { .compatible   = "amcc,xor-accelerator", },
4549     {},
4550 };
4551 MODULE_DEVICE_TABLE(of, ppc440spe_adma_of_match);
4552
4553 static struct platform_driver ppc440spe_adma_driver = {
4554     .probe = ppc440spe_adma_probe,
4555     .remove = ppc440spe_adma_remove,
4556     .driver = {
4557         .name = "PPC440SP(E)-ADMA",
4558         .of_match_table = ppc440spe_adma_of_match,
4559     },
4560 };
4561
4562 static __init int ppc440spe_adma_init(void)
4563 {
4564     int ret;
4565
4566     ret = ppc440spe_configure_raid_devices();
4567     if (ret)
4568         return ret;
4569
4570     ret = platform_driver_register(&ppc440spe_adma_driver);
4571     if (ret) {
4572         pr_err("%s: failed to register platform driver\n",
4573             __func__);
4574         goto out_reg;
4575     }
4576
4577     /* Initialization status */
4578     ret = driver_create_file(&ppc440spe_adma_driver.driver,
4579                  &driver_attr_devices);
4580     if (ret)
4581         goto out_dev;
4582
4583     /* RAID-6 h/w enable entry */
4584     ret = driver_create_file(&ppc440spe_adma_driver.driver,
4585                  &driver_attr_enable);
4586     if (ret)
4587         goto out_en;
4588
4589     /* GF polynomial to use */
4590     ret = driver_create_file(&ppc440spe_adma_driver.driver,
4591                  &driver_attr_poly);
4592     if (!ret)
4593         return ret;
4594
4595     driver_remove_file(&ppc440spe_adma_driver.driver,
4596                &driver_attr_enable);
4597 out_en:
4598     driver_remove_file(&ppc440spe_adma_driver.driver,
4599                &driver_attr_devices);
4600 out_dev:
4601     /* User will not be able to enable h/w RAID-6 */
4602     pr_err("%s: failed to create RAID-6 driver interface\n",
4603         __func__);
4604     platform_driver_unregister(&ppc440spe_adma_driver);
4605 out_reg:
4606     dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len);
4607     kfree(ppc440spe_dma_fifo_buf);
4608     return ret;
4609 }
4610
4611 static void __exit ppc440spe_adma_exit(void)
4612 {
4613     driver_remove_file(&ppc440spe_adma_driver.driver,
4614                &driver_attr_poly);
4615     driver_remove_file(&ppc440spe_adma_driver.driver,
4616                &driver_attr_enable);
4617     driver_remove_file(&ppc440spe_adma_driver.driver,
4618                &driver_attr_devices);
4619     platform_driver_unregister(&ppc440spe_adma_driver);
4620     dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len);
4621     kfree(ppc440spe_dma_fifo_buf);
4622 }
4623
4624 arch_initcall(ppc440spe_adma_init);
4625 module_exit(ppc440spe_adma_exit);
4626
4627 MODULE_AUTHOR("Yuri Tikhonov <yur@emcraft.com>");
4628 MODULE_DESCRIPTION("PPC440SPE ADMA Engine Driver");
4629 MODULE_LICENSE("GPL");