Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * offload engine driver for the Intel Xscale series of i/o processors
0004  * Copyright © 2006, Intel Corporation.
0005  */
0006 
0007 /*
0008  * This driver supports the asynchrounous DMA copy and RAID engines available
0009  * on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x)
0010  */
0011 
0012 #include <linux/init.h>
0013 #include <linux/module.h>
0014 #include <linux/delay.h>
0015 #include <linux/dma-mapping.h>
0016 #include <linux/spinlock.h>
0017 #include <linux/interrupt.h>
0018 #include <linux/platform_device.h>
0019 #include <linux/prefetch.h>
0020 #include <linux/memory.h>
0021 #include <linux/ioport.h>
0022 #include <linux/raid/pq.h>
0023 #include <linux/slab.h>
0024 
0025 #include "iop-adma.h"
0026 #include "dmaengine.h"
0027 
0028 #define to_iop_adma_chan(chan) container_of(chan, struct iop_adma_chan, common)
0029 #define to_iop_adma_device(dev) \
0030     container_of(dev, struct iop_adma_device, common)
0031 #define tx_to_iop_adma_slot(tx) \
0032     container_of(tx, struct iop_adma_desc_slot, async_tx)
0033 
0034 /**
0035  * iop_adma_free_slots - flags descriptor slots for reuse
0036  * @slot: Slot to free
0037  * Caller must hold &iop_chan->lock while calling this function
0038  */
0039 static void iop_adma_free_slots(struct iop_adma_desc_slot *slot)
0040 {
0041     int stride = slot->slots_per_op;
0042 
0043     while (stride--) {
0044         slot->slots_per_op = 0;
0045         slot = list_entry(slot->slot_node.next,
0046                 struct iop_adma_desc_slot,
0047                 slot_node);
0048     }
0049 }
0050 
0051 static dma_cookie_t
0052 iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc,
0053     struct iop_adma_chan *iop_chan, dma_cookie_t cookie)
0054 {
0055     struct dma_async_tx_descriptor *tx = &desc->async_tx;
0056 
0057     BUG_ON(tx->cookie < 0);
0058     if (tx->cookie > 0) {
0059         cookie = tx->cookie;
0060         tx->cookie = 0;
0061 
0062         /* call the callback (must not sleep or submit new
0063          * operations to this channel)
0064          */
0065         dmaengine_desc_get_callback_invoke(tx, NULL);
0066 
0067         dma_descriptor_unmap(tx);
0068         if (desc->group_head)
0069             desc->group_head = NULL;
0070     }
0071 
0072     /* run dependent operations */
0073     dma_run_dependencies(tx);
0074 
0075     return cookie;
0076 }
0077 
0078 static int
0079 iop_adma_clean_slot(struct iop_adma_desc_slot *desc,
0080     struct iop_adma_chan *iop_chan)
0081 {
0082     /* the client is allowed to attach dependent operations
0083      * until 'ack' is set
0084      */
0085     if (!async_tx_test_ack(&desc->async_tx))
0086         return 0;
0087 
0088     /* leave the last descriptor in the chain
0089      * so we can append to it
0090      */
0091     if (desc->chain_node.next == &iop_chan->chain)
0092         return 1;
0093 
0094     dev_dbg(iop_chan->device->common.dev,
0095         "\tfree slot: %d slots_per_op: %d\n",
0096         desc->idx, desc->slots_per_op);
0097 
0098     list_del(&desc->chain_node);
0099     iop_adma_free_slots(desc);
0100 
0101     return 0;
0102 }
0103 
0104 static void __iop_adma_slot_cleanup(struct iop_adma_chan *iop_chan)
0105 {
0106     struct iop_adma_desc_slot *iter, *_iter, *grp_start = NULL;
0107     dma_cookie_t cookie = 0;
0108     u32 current_desc = iop_chan_get_current_descriptor(iop_chan);
0109     int busy = iop_chan_is_busy(iop_chan);
0110     int seen_current = 0, slot_cnt = 0, slots_per_op = 0;
0111 
0112     dev_dbg(iop_chan->device->common.dev, "%s\n", __func__);
0113     /* free completed slots from the chain starting with
0114      * the oldest descriptor
0115      */
0116     list_for_each_entry_safe(iter, _iter, &iop_chan->chain,
0117                     chain_node) {
0118         pr_debug("\tcookie: %d slot: %d busy: %d "
0119             "this_desc: %pad next_desc: %#llx ack: %d\n",
0120             iter->async_tx.cookie, iter->idx, busy,
0121             &iter->async_tx.phys, (u64)iop_desc_get_next_desc(iter),
0122             async_tx_test_ack(&iter->async_tx));
0123         prefetch(_iter);
0124         prefetch(&_iter->async_tx);
0125 
0126         /* do not advance past the current descriptor loaded into the
0127          * hardware channel, subsequent descriptors are either in
0128          * process or have not been submitted
0129          */
0130         if (seen_current)
0131             break;
0132 
0133         /* stop the search if we reach the current descriptor and the
0134          * channel is busy, or if it appears that the current descriptor
0135          * needs to be re-read (i.e. has been appended to)
0136          */
0137         if (iter->async_tx.phys == current_desc) {
0138             BUG_ON(seen_current++);
0139             if (busy || iop_desc_get_next_desc(iter))
0140                 break;
0141         }
0142 
0143         /* detect the start of a group transaction */
0144         if (!slot_cnt && !slots_per_op) {
0145             slot_cnt = iter->slot_cnt;
0146             slots_per_op = iter->slots_per_op;
0147             if (slot_cnt <= slots_per_op) {
0148                 slot_cnt = 0;
0149                 slots_per_op = 0;
0150             }
0151         }
0152 
0153         if (slot_cnt) {
0154             pr_debug("\tgroup++\n");
0155             if (!grp_start)
0156                 grp_start = iter;
0157             slot_cnt -= slots_per_op;
0158         }
0159 
0160         /* all the members of a group are complete */
0161         if (slots_per_op != 0 && slot_cnt == 0) {
0162             struct iop_adma_desc_slot *grp_iter, *_grp_iter;
0163             int end_of_chain = 0;
0164             pr_debug("\tgroup end\n");
0165 
0166             /* collect the total results */
0167             if (grp_start->xor_check_result) {
0168                 u32 zero_sum_result = 0;
0169                 slot_cnt = grp_start->slot_cnt;
0170                 grp_iter = grp_start;
0171 
0172                 list_for_each_entry_from(grp_iter,
0173                     &iop_chan->chain, chain_node) {
0174                     zero_sum_result |=
0175                         iop_desc_get_zero_result(grp_iter);
0176                     pr_debug("\titer%d result: %d\n",
0177                         grp_iter->idx, zero_sum_result);
0178                     slot_cnt -= slots_per_op;
0179                     if (slot_cnt == 0)
0180                         break;
0181                 }
0182                 pr_debug("\tgrp_start->xor_check_result: %p\n",
0183                     grp_start->xor_check_result);
0184                 *grp_start->xor_check_result = zero_sum_result;
0185             }
0186 
0187             /* clean up the group */
0188             slot_cnt = grp_start->slot_cnt;
0189             grp_iter = grp_start;
0190             list_for_each_entry_safe_from(grp_iter, _grp_iter,
0191                 &iop_chan->chain, chain_node) {
0192                 cookie = iop_adma_run_tx_complete_actions(
0193                     grp_iter, iop_chan, cookie);
0194 
0195                 slot_cnt -= slots_per_op;
0196                 end_of_chain = iop_adma_clean_slot(grp_iter,
0197                     iop_chan);
0198 
0199                 if (slot_cnt == 0 || end_of_chain)
0200                     break;
0201             }
0202 
0203             /* the group should be complete at this point */
0204             BUG_ON(slot_cnt);
0205 
0206             slots_per_op = 0;
0207             grp_start = NULL;
0208             if (end_of_chain)
0209                 break;
0210             else
0211                 continue;
0212         } else if (slots_per_op) /* wait for group completion */
0213             continue;
0214 
0215         /* write back zero sum results (single descriptor case) */
0216         if (iter->xor_check_result && iter->async_tx.cookie)
0217             *iter->xor_check_result =
0218                 iop_desc_get_zero_result(iter);
0219 
0220         cookie = iop_adma_run_tx_complete_actions(
0221                     iter, iop_chan, cookie);
0222 
0223         if (iop_adma_clean_slot(iter, iop_chan))
0224             break;
0225     }
0226 
0227     if (cookie > 0) {
0228         iop_chan->common.completed_cookie = cookie;
0229         pr_debug("\tcompleted cookie %d\n", cookie);
0230     }
0231 }
0232 
0233 static void
0234 iop_adma_slot_cleanup(struct iop_adma_chan *iop_chan)
0235 {
0236     spin_lock_bh(&iop_chan->lock);
0237     __iop_adma_slot_cleanup(iop_chan);
0238     spin_unlock_bh(&iop_chan->lock);
0239 }
0240 
0241 static void iop_adma_tasklet(struct tasklet_struct *t)
0242 {
0243     struct iop_adma_chan *iop_chan = from_tasklet(iop_chan, t,
0244                               irq_tasklet);
0245 
0246     /* lockdep will flag depedency submissions as potentially
0247      * recursive locking, this is not the case as a dependency
0248      * submission will never recurse a channels submit routine.
0249      * There are checks in async_tx.c to prevent this.
0250      */
0251     spin_lock_nested(&iop_chan->lock, SINGLE_DEPTH_NESTING);
0252     __iop_adma_slot_cleanup(iop_chan);
0253     spin_unlock(&iop_chan->lock);
0254 }
0255 
0256 static struct iop_adma_desc_slot *
0257 iop_adma_alloc_slots(struct iop_adma_chan *iop_chan, int num_slots,
0258             int slots_per_op)
0259 {
0260     struct iop_adma_desc_slot *iter, *_iter, *alloc_start = NULL;
0261     LIST_HEAD(chain);
0262     int slots_found, retry = 0;
0263 
0264     /* start search from the last allocated descrtiptor
0265      * if a contiguous allocation can not be found start searching
0266      * from the beginning of the list
0267      */
0268 retry:
0269     slots_found = 0;
0270     if (retry == 0)
0271         iter = iop_chan->last_used;
0272     else
0273         iter = list_entry(&iop_chan->all_slots,
0274             struct iop_adma_desc_slot,
0275             slot_node);
0276 
0277     list_for_each_entry_safe_continue(
0278         iter, _iter, &iop_chan->all_slots, slot_node) {
0279         prefetch(_iter);
0280         prefetch(&_iter->async_tx);
0281         if (iter->slots_per_op) {
0282             /* give up after finding the first busy slot
0283              * on the second pass through the list
0284              */
0285             if (retry)
0286                 break;
0287 
0288             slots_found = 0;
0289             continue;
0290         }
0291 
0292         /* start the allocation if the slot is correctly aligned */
0293         if (!slots_found++) {
0294             if (iop_desc_is_aligned(iter, slots_per_op))
0295                 alloc_start = iter;
0296             else {
0297                 slots_found = 0;
0298                 continue;
0299             }
0300         }
0301 
0302         if (slots_found == num_slots) {
0303             struct iop_adma_desc_slot *alloc_tail = NULL;
0304             struct iop_adma_desc_slot *last_used = NULL;
0305             iter = alloc_start;
0306             while (num_slots) {
0307                 int i;
0308                 dev_dbg(iop_chan->device->common.dev,
0309                     "allocated slot: %d "
0310                     "(desc %p phys: %#llx) slots_per_op %d\n",
0311                     iter->idx, iter->hw_desc,
0312                     (u64)iter->async_tx.phys, slots_per_op);
0313 
0314                 /* pre-ack all but the last descriptor */
0315                 if (num_slots != slots_per_op)
0316                     async_tx_ack(&iter->async_tx);
0317 
0318                 list_add_tail(&iter->chain_node, &chain);
0319                 alloc_tail = iter;
0320                 iter->async_tx.cookie = 0;
0321                 iter->slot_cnt = num_slots;
0322                 iter->xor_check_result = NULL;
0323                 for (i = 0; i < slots_per_op; i++) {
0324                     iter->slots_per_op = slots_per_op - i;
0325                     last_used = iter;
0326                     iter = list_entry(iter->slot_node.next,
0327                         struct iop_adma_desc_slot,
0328                         slot_node);
0329                 }
0330                 num_slots -= slots_per_op;
0331             }
0332             alloc_tail->group_head = alloc_start;
0333             alloc_tail->async_tx.cookie = -EBUSY;
0334             list_splice(&chain, &alloc_tail->tx_list);
0335             iop_chan->last_used = last_used;
0336             iop_desc_clear_next_desc(alloc_start);
0337             iop_desc_clear_next_desc(alloc_tail);
0338             return alloc_tail;
0339         }
0340     }
0341     if (!retry++)
0342         goto retry;
0343 
0344     /* perform direct reclaim if the allocation fails */
0345     __iop_adma_slot_cleanup(iop_chan);
0346 
0347     return NULL;
0348 }
0349 
0350 static void iop_adma_check_threshold(struct iop_adma_chan *iop_chan)
0351 {
0352     dev_dbg(iop_chan->device->common.dev, "pending: %d\n",
0353         iop_chan->pending);
0354 
0355     if (iop_chan->pending >= IOP_ADMA_THRESHOLD) {
0356         iop_chan->pending = 0;
0357         iop_chan_append(iop_chan);
0358     }
0359 }
0360 
0361 static dma_cookie_t
0362 iop_adma_tx_submit(struct dma_async_tx_descriptor *tx)
0363 {
0364     struct iop_adma_desc_slot *sw_desc = tx_to_iop_adma_slot(tx);
0365     struct iop_adma_chan *iop_chan = to_iop_adma_chan(tx->chan);
0366     struct iop_adma_desc_slot *grp_start, *old_chain_tail;
0367     int slot_cnt;
0368     dma_cookie_t cookie;
0369     dma_addr_t next_dma;
0370 
0371     grp_start = sw_desc->group_head;
0372     slot_cnt = grp_start->slot_cnt;
0373 
0374     spin_lock_bh(&iop_chan->lock);
0375     cookie = dma_cookie_assign(tx);
0376 
0377     old_chain_tail = list_entry(iop_chan->chain.prev,
0378         struct iop_adma_desc_slot, chain_node);
0379     list_splice_init(&sw_desc->tx_list,
0380              &old_chain_tail->chain_node);
0381 
0382     /* fix up the hardware chain */
0383     next_dma = grp_start->async_tx.phys;
0384     iop_desc_set_next_desc(old_chain_tail, next_dma);
0385     BUG_ON(iop_desc_get_next_desc(old_chain_tail) != next_dma); /* flush */
0386 
0387     /* check for pre-chained descriptors */
0388     iop_paranoia(iop_desc_get_next_desc(sw_desc));
0389 
0390     /* increment the pending count by the number of slots
0391      * memcpy operations have a 1:1 (slot:operation) relation
0392      * other operations are heavier and will pop the threshold
0393      * more often.
0394      */
0395     iop_chan->pending += slot_cnt;
0396     iop_adma_check_threshold(iop_chan);
0397     spin_unlock_bh(&iop_chan->lock);
0398 
0399     dev_dbg(iop_chan->device->common.dev, "%s cookie: %d slot: %d\n",
0400         __func__, sw_desc->async_tx.cookie, sw_desc->idx);
0401 
0402     return cookie;
0403 }
0404 
0405 static void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan);
0406 static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan);
0407 
0408 /**
0409  * iop_adma_alloc_chan_resources -  returns the number of allocated descriptors
0410  * @chan: allocate descriptor resources for this channel
0411  *
0412  * Note: We keep the slots for 1 operation on iop_chan->chain at all times.  To
0413  * avoid deadlock, via async_xor, num_descs_in_pool must at a minimum be
0414  * greater than 2x the number slots needed to satisfy a device->max_xor
0415  * request.
0416  * */
0417 static int iop_adma_alloc_chan_resources(struct dma_chan *chan)
0418 {
0419     char *hw_desc;
0420     dma_addr_t dma_desc;
0421     int idx;
0422     struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
0423     struct iop_adma_desc_slot *slot = NULL;
0424     int init = iop_chan->slots_allocated ? 0 : 1;
0425     struct iop_adma_platform_data *plat_data =
0426         dev_get_platdata(&iop_chan->device->pdev->dev);
0427     int num_descs_in_pool = plat_data->pool_size/IOP_ADMA_SLOT_SIZE;
0428 
0429     /* Allocate descriptor slots */
0430     do {
0431         idx = iop_chan->slots_allocated;
0432         if (idx == num_descs_in_pool)
0433             break;
0434 
0435         slot = kzalloc(sizeof(*slot), GFP_KERNEL);
0436         if (!slot) {
0437             printk(KERN_INFO "IOP ADMA Channel only initialized"
0438                 " %d descriptor slots", idx);
0439             break;
0440         }
0441         hw_desc = (char *) iop_chan->device->dma_desc_pool_virt;
0442         slot->hw_desc = (void *) &hw_desc[idx * IOP_ADMA_SLOT_SIZE];
0443 
0444         dma_async_tx_descriptor_init(&slot->async_tx, chan);
0445         slot->async_tx.tx_submit = iop_adma_tx_submit;
0446         INIT_LIST_HEAD(&slot->tx_list);
0447         INIT_LIST_HEAD(&slot->chain_node);
0448         INIT_LIST_HEAD(&slot->slot_node);
0449         dma_desc = iop_chan->device->dma_desc_pool;
0450         slot->async_tx.phys = dma_desc + idx * IOP_ADMA_SLOT_SIZE;
0451         slot->idx = idx;
0452 
0453         spin_lock_bh(&iop_chan->lock);
0454         iop_chan->slots_allocated++;
0455         list_add_tail(&slot->slot_node, &iop_chan->all_slots);
0456         spin_unlock_bh(&iop_chan->lock);
0457     } while (iop_chan->slots_allocated < num_descs_in_pool);
0458 
0459     if (idx && !iop_chan->last_used)
0460         iop_chan->last_used = list_entry(iop_chan->all_slots.next,
0461                     struct iop_adma_desc_slot,
0462                     slot_node);
0463 
0464     dev_dbg(iop_chan->device->common.dev,
0465         "allocated %d descriptor slots last_used: %p\n",
0466         iop_chan->slots_allocated, iop_chan->last_used);
0467 
0468     /* initialize the channel and the chain with a null operation */
0469     if (init) {
0470         if (dma_has_cap(DMA_MEMCPY,
0471             iop_chan->device->common.cap_mask))
0472             iop_chan_start_null_memcpy(iop_chan);
0473         else if (dma_has_cap(DMA_XOR,
0474             iop_chan->device->common.cap_mask))
0475             iop_chan_start_null_xor(iop_chan);
0476         else
0477             BUG();
0478     }
0479 
0480     return (idx > 0) ? idx : -ENOMEM;
0481 }
0482 
0483 static struct dma_async_tx_descriptor *
0484 iop_adma_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags)
0485 {
0486     struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
0487     struct iop_adma_desc_slot *sw_desc, *grp_start;
0488     int slot_cnt, slots_per_op;
0489 
0490     dev_dbg(iop_chan->device->common.dev, "%s\n", __func__);
0491 
0492     spin_lock_bh(&iop_chan->lock);
0493     slot_cnt = iop_chan_interrupt_slot_count(&slots_per_op, iop_chan);
0494     sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
0495     if (sw_desc) {
0496         grp_start = sw_desc->group_head;
0497         iop_desc_init_interrupt(grp_start, iop_chan);
0498         sw_desc->async_tx.flags = flags;
0499     }
0500     spin_unlock_bh(&iop_chan->lock);
0501 
0502     return sw_desc ? &sw_desc->async_tx : NULL;
0503 }
0504 
0505 static struct dma_async_tx_descriptor *
0506 iop_adma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dma_dest,
0507              dma_addr_t dma_src, size_t len, unsigned long flags)
0508 {
0509     struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
0510     struct iop_adma_desc_slot *sw_desc, *grp_start;
0511     int slot_cnt, slots_per_op;
0512 
0513     if (unlikely(!len))
0514         return NULL;
0515     BUG_ON(len > IOP_ADMA_MAX_BYTE_COUNT);
0516 
0517     dev_dbg(iop_chan->device->common.dev, "%s len: %zu\n",
0518         __func__, len);
0519 
0520     spin_lock_bh(&iop_chan->lock);
0521     slot_cnt = iop_chan_memcpy_slot_count(len, &slots_per_op);
0522     sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
0523     if (sw_desc) {
0524         grp_start = sw_desc->group_head;
0525         iop_desc_init_memcpy(grp_start, flags);
0526         iop_desc_set_byte_count(grp_start, iop_chan, len);
0527         iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest);
0528         iop_desc_set_memcpy_src_addr(grp_start, dma_src);
0529         sw_desc->async_tx.flags = flags;
0530     }
0531     spin_unlock_bh(&iop_chan->lock);
0532 
0533     return sw_desc ? &sw_desc->async_tx : NULL;
0534 }
0535 
0536 static struct dma_async_tx_descriptor *
0537 iop_adma_prep_dma_xor(struct dma_chan *chan, dma_addr_t dma_dest,
0538               dma_addr_t *dma_src, unsigned int src_cnt, size_t len,
0539               unsigned long flags)
0540 {
0541     struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
0542     struct iop_adma_desc_slot *sw_desc, *grp_start;
0543     int slot_cnt, slots_per_op;
0544 
0545     if (unlikely(!len))
0546         return NULL;
0547     BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT);
0548 
0549     dev_dbg(iop_chan->device->common.dev,
0550         "%s src_cnt: %d len: %zu flags: %lx\n",
0551         __func__, src_cnt, len, flags);
0552 
0553     spin_lock_bh(&iop_chan->lock);
0554     slot_cnt = iop_chan_xor_slot_count(len, src_cnt, &slots_per_op);
0555     sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
0556     if (sw_desc) {
0557         grp_start = sw_desc->group_head;
0558         iop_desc_init_xor(grp_start, src_cnt, flags);
0559         iop_desc_set_byte_count(grp_start, iop_chan, len);
0560         iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest);
0561         sw_desc->async_tx.flags = flags;
0562         while (src_cnt--)
0563             iop_desc_set_xor_src_addr(grp_start, src_cnt,
0564                           dma_src[src_cnt]);
0565     }
0566     spin_unlock_bh(&iop_chan->lock);
0567 
0568     return sw_desc ? &sw_desc->async_tx : NULL;
0569 }
0570 
0571 static struct dma_async_tx_descriptor *
0572 iop_adma_prep_dma_xor_val(struct dma_chan *chan, dma_addr_t *dma_src,
0573               unsigned int src_cnt, size_t len, u32 *result,
0574               unsigned long flags)
0575 {
0576     struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
0577     struct iop_adma_desc_slot *sw_desc, *grp_start;
0578     int slot_cnt, slots_per_op;
0579 
0580     if (unlikely(!len))
0581         return NULL;
0582 
0583     dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %zu\n",
0584         __func__, src_cnt, len);
0585 
0586     spin_lock_bh(&iop_chan->lock);
0587     slot_cnt = iop_chan_zero_sum_slot_count(len, src_cnt, &slots_per_op);
0588     sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
0589     if (sw_desc) {
0590         grp_start = sw_desc->group_head;
0591         iop_desc_init_zero_sum(grp_start, src_cnt, flags);
0592         iop_desc_set_zero_sum_byte_count(grp_start, len);
0593         grp_start->xor_check_result = result;
0594         pr_debug("\t%s: grp_start->xor_check_result: %p\n",
0595             __func__, grp_start->xor_check_result);
0596         sw_desc->async_tx.flags = flags;
0597         while (src_cnt--)
0598             iop_desc_set_zero_sum_src_addr(grp_start, src_cnt,
0599                                dma_src[src_cnt]);
0600     }
0601     spin_unlock_bh(&iop_chan->lock);
0602 
0603     return sw_desc ? &sw_desc->async_tx : NULL;
0604 }
0605 
0606 static struct dma_async_tx_descriptor *
0607 iop_adma_prep_dma_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
0608              unsigned int src_cnt, const unsigned char *scf, size_t len,
0609              unsigned long flags)
0610 {
0611     struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
0612     struct iop_adma_desc_slot *sw_desc, *g;
0613     int slot_cnt, slots_per_op;
0614     int continue_srcs;
0615 
0616     if (unlikely(!len))
0617         return NULL;
0618     BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT);
0619 
0620     dev_dbg(iop_chan->device->common.dev,
0621         "%s src_cnt: %d len: %zu flags: %lx\n",
0622         __func__, src_cnt, len, flags);
0623 
0624     if (dmaf_p_disabled_continue(flags))
0625         continue_srcs = 1+src_cnt;
0626     else if (dmaf_continue(flags))
0627         continue_srcs = 3+src_cnt;
0628     else
0629         continue_srcs = 0+src_cnt;
0630 
0631     spin_lock_bh(&iop_chan->lock);
0632     slot_cnt = iop_chan_pq_slot_count(len, continue_srcs, &slots_per_op);
0633     sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
0634     if (sw_desc) {
0635         int i;
0636 
0637         g = sw_desc->group_head;
0638         iop_desc_set_byte_count(g, iop_chan, len);
0639 
0640         /* even if P is disabled its destination address (bits
0641          * [3:0]) must match Q.  It is ok if P points to an
0642          * invalid address, it won't be written.
0643          */
0644         if (flags & DMA_PREP_PQ_DISABLE_P)
0645             dst[0] = dst[1] & 0x7;
0646 
0647         iop_desc_set_pq_addr(g, dst);
0648         sw_desc->async_tx.flags = flags;
0649         for (i = 0; i < src_cnt; i++)
0650             iop_desc_set_pq_src_addr(g, i, src[i], scf[i]);
0651 
0652         /* if we are continuing a previous operation factor in
0653          * the old p and q values, see the comment for dma_maxpq
0654          * in include/linux/dmaengine.h
0655          */
0656         if (dmaf_p_disabled_continue(flags))
0657             iop_desc_set_pq_src_addr(g, i++, dst[1], 1);
0658         else if (dmaf_continue(flags)) {
0659             iop_desc_set_pq_src_addr(g, i++, dst[0], 0);
0660             iop_desc_set_pq_src_addr(g, i++, dst[1], 1);
0661             iop_desc_set_pq_src_addr(g, i++, dst[1], 0);
0662         }
0663         iop_desc_init_pq(g, i, flags);
0664     }
0665     spin_unlock_bh(&iop_chan->lock);
0666 
0667     return sw_desc ? &sw_desc->async_tx : NULL;
0668 }
0669 
0670 static struct dma_async_tx_descriptor *
0671 iop_adma_prep_dma_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
0672              unsigned int src_cnt, const unsigned char *scf,
0673              size_t len, enum sum_check_flags *pqres,
0674              unsigned long flags)
0675 {
0676     struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
0677     struct iop_adma_desc_slot *sw_desc, *g;
0678     int slot_cnt, slots_per_op;
0679 
0680     if (unlikely(!len))
0681         return NULL;
0682     BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT);
0683 
0684     dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %zu\n",
0685         __func__, src_cnt, len);
0686 
0687     spin_lock_bh(&iop_chan->lock);
0688     slot_cnt = iop_chan_pq_zero_sum_slot_count(len, src_cnt + 2, &slots_per_op);
0689     sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
0690     if (sw_desc) {
0691         /* for validate operations p and q are tagged onto the
0692          * end of the source list
0693          */
0694         int pq_idx = src_cnt;
0695 
0696         g = sw_desc->group_head;
0697         iop_desc_init_pq_zero_sum(g, src_cnt+2, flags);
0698         iop_desc_set_pq_zero_sum_byte_count(g, len);
0699         g->pq_check_result = pqres;
0700         pr_debug("\t%s: g->pq_check_result: %p\n",
0701             __func__, g->pq_check_result);
0702         sw_desc->async_tx.flags = flags;
0703         while (src_cnt--)
0704             iop_desc_set_pq_zero_sum_src_addr(g, src_cnt,
0705                               src[src_cnt],
0706                               scf[src_cnt]);
0707         iop_desc_set_pq_zero_sum_addr(g, pq_idx, src);
0708     }
0709     spin_unlock_bh(&iop_chan->lock);
0710 
0711     return sw_desc ? &sw_desc->async_tx : NULL;
0712 }
0713 
0714 static void iop_adma_free_chan_resources(struct dma_chan *chan)
0715 {
0716     struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
0717     struct iop_adma_desc_slot *iter, *_iter;
0718     int in_use_descs = 0;
0719 
0720     iop_adma_slot_cleanup(iop_chan);
0721 
0722     spin_lock_bh(&iop_chan->lock);
0723     list_for_each_entry_safe(iter, _iter, &iop_chan->chain,
0724                     chain_node) {
0725         in_use_descs++;
0726         list_del(&iter->chain_node);
0727     }
0728     list_for_each_entry_safe_reverse(
0729         iter, _iter, &iop_chan->all_slots, slot_node) {
0730         list_del(&iter->slot_node);
0731         kfree(iter);
0732         iop_chan->slots_allocated--;
0733     }
0734     iop_chan->last_used = NULL;
0735 
0736     dev_dbg(iop_chan->device->common.dev, "%s slots_allocated %d\n",
0737         __func__, iop_chan->slots_allocated);
0738     spin_unlock_bh(&iop_chan->lock);
0739 
0740     /* one is ok since we left it on there on purpose */
0741     if (in_use_descs > 1)
0742         printk(KERN_ERR "IOP: Freeing %d in use descriptors!\n",
0743             in_use_descs - 1);
0744 }
0745 
0746 /**
0747  * iop_adma_status - poll the status of an ADMA transaction
0748  * @chan: ADMA channel handle
0749  * @cookie: ADMA transaction identifier
0750  * @txstate: a holder for the current state of the channel or NULL
0751  */
0752 static enum dma_status iop_adma_status(struct dma_chan *chan,
0753                     dma_cookie_t cookie,
0754                     struct dma_tx_state *txstate)
0755 {
0756     struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
0757     int ret;
0758 
0759     ret = dma_cookie_status(chan, cookie, txstate);
0760     if (ret == DMA_COMPLETE)
0761         return ret;
0762 
0763     iop_adma_slot_cleanup(iop_chan);
0764 
0765     return dma_cookie_status(chan, cookie, txstate);
0766 }
0767 
0768 static irqreturn_t iop_adma_eot_handler(int irq, void *data)
0769 {
0770     struct iop_adma_chan *chan = data;
0771 
0772     dev_dbg(chan->device->common.dev, "%s\n", __func__);
0773 
0774     tasklet_schedule(&chan->irq_tasklet);
0775 
0776     iop_adma_device_clear_eot_status(chan);
0777 
0778     return IRQ_HANDLED;
0779 }
0780 
0781 static irqreturn_t iop_adma_eoc_handler(int irq, void *data)
0782 {
0783     struct iop_adma_chan *chan = data;
0784 
0785     dev_dbg(chan->device->common.dev, "%s\n", __func__);
0786 
0787     tasklet_schedule(&chan->irq_tasklet);
0788 
0789     iop_adma_device_clear_eoc_status(chan);
0790 
0791     return IRQ_HANDLED;
0792 }
0793 
0794 static irqreturn_t iop_adma_err_handler(int irq, void *data)
0795 {
0796     struct iop_adma_chan *chan = data;
0797     unsigned long status = iop_chan_get_status(chan);
0798 
0799     dev_err(chan->device->common.dev,
0800         "error ( %s%s%s%s%s%s%s)\n",
0801         iop_is_err_int_parity(status, chan) ? "int_parity " : "",
0802         iop_is_err_mcu_abort(status, chan) ? "mcu_abort " : "",
0803         iop_is_err_int_tabort(status, chan) ? "int_tabort " : "",
0804         iop_is_err_int_mabort(status, chan) ? "int_mabort " : "",
0805         iop_is_err_pci_tabort(status, chan) ? "pci_tabort " : "",
0806         iop_is_err_pci_mabort(status, chan) ? "pci_mabort " : "",
0807         iop_is_err_split_tx(status, chan) ? "split_tx " : "");
0808 
0809     iop_adma_device_clear_err_status(chan);
0810 
0811     BUG();
0812 
0813     return IRQ_HANDLED;
0814 }
0815 
0816 static void iop_adma_issue_pending(struct dma_chan *chan)
0817 {
0818     struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
0819 
0820     if (iop_chan->pending) {
0821         iop_chan->pending = 0;
0822         iop_chan_append(iop_chan);
0823     }
0824 }
0825 
0826 /*
0827  * Perform a transaction to verify the HW works.
0828  */
0829 #define IOP_ADMA_TEST_SIZE 2000
0830 
0831 static int iop_adma_memcpy_self_test(struct iop_adma_device *device)
0832 {
0833     int i;
0834     void *src, *dest;
0835     dma_addr_t src_dma, dest_dma;
0836     struct dma_chan *dma_chan;
0837     dma_cookie_t cookie;
0838     struct dma_async_tx_descriptor *tx;
0839     int err = 0;
0840     struct iop_adma_chan *iop_chan;
0841 
0842     dev_dbg(device->common.dev, "%s\n", __func__);
0843 
0844     src = kmalloc(IOP_ADMA_TEST_SIZE, GFP_KERNEL);
0845     if (!src)
0846         return -ENOMEM;
0847     dest = kzalloc(IOP_ADMA_TEST_SIZE, GFP_KERNEL);
0848     if (!dest) {
0849         kfree(src);
0850         return -ENOMEM;
0851     }
0852 
0853     /* Fill in src buffer */
0854     for (i = 0; i < IOP_ADMA_TEST_SIZE; i++)
0855         ((u8 *) src)[i] = (u8)i;
0856 
0857     /* Start copy, using first DMA channel */
0858     dma_chan = container_of(device->common.channels.next,
0859                 struct dma_chan,
0860                 device_node);
0861     if (iop_adma_alloc_chan_resources(dma_chan) < 1) {
0862         err = -ENODEV;
0863         goto out;
0864     }
0865 
0866     dest_dma = dma_map_single(dma_chan->device->dev, dest,
0867                 IOP_ADMA_TEST_SIZE, DMA_FROM_DEVICE);
0868     src_dma = dma_map_single(dma_chan->device->dev, src,
0869                 IOP_ADMA_TEST_SIZE, DMA_TO_DEVICE);
0870     tx = iop_adma_prep_dma_memcpy(dma_chan, dest_dma, src_dma,
0871                       IOP_ADMA_TEST_SIZE,
0872                       DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
0873 
0874     cookie = iop_adma_tx_submit(tx);
0875     iop_adma_issue_pending(dma_chan);
0876     msleep(1);
0877 
0878     if (iop_adma_status(dma_chan, cookie, NULL) !=
0879             DMA_COMPLETE) {
0880         dev_err(dma_chan->device->dev,
0881             "Self-test copy timed out, disabling\n");
0882         err = -ENODEV;
0883         goto free_resources;
0884     }
0885 
0886     iop_chan = to_iop_adma_chan(dma_chan);
0887     dma_sync_single_for_cpu(&iop_chan->device->pdev->dev, dest_dma,
0888         IOP_ADMA_TEST_SIZE, DMA_FROM_DEVICE);
0889     if (memcmp(src, dest, IOP_ADMA_TEST_SIZE)) {
0890         dev_err(dma_chan->device->dev,
0891             "Self-test copy failed compare, disabling\n");
0892         err = -ENODEV;
0893         goto free_resources;
0894     }
0895 
0896 free_resources:
0897     iop_adma_free_chan_resources(dma_chan);
0898 out:
0899     kfree(src);
0900     kfree(dest);
0901     return err;
0902 }
0903 
0904 #define IOP_ADMA_NUM_SRC_TEST 4 /* must be <= 15 */
0905 static int
0906 iop_adma_xor_val_self_test(struct iop_adma_device *device)
0907 {
0908     int i, src_idx;
0909     struct page *dest;
0910     struct page *xor_srcs[IOP_ADMA_NUM_SRC_TEST];
0911     struct page *zero_sum_srcs[IOP_ADMA_NUM_SRC_TEST + 1];
0912     dma_addr_t dma_srcs[IOP_ADMA_NUM_SRC_TEST + 1];
0913     dma_addr_t dest_dma;
0914     struct dma_async_tx_descriptor *tx;
0915     struct dma_chan *dma_chan;
0916     dma_cookie_t cookie;
0917     u8 cmp_byte = 0;
0918     u32 cmp_word;
0919     u32 zero_sum_result;
0920     int err = 0;
0921     struct iop_adma_chan *iop_chan;
0922 
0923     dev_dbg(device->common.dev, "%s\n", __func__);
0924 
0925     for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) {
0926         xor_srcs[src_idx] = alloc_page(GFP_KERNEL);
0927         if (!xor_srcs[src_idx]) {
0928             while (src_idx--)
0929                 __free_page(xor_srcs[src_idx]);
0930             return -ENOMEM;
0931         }
0932     }
0933 
0934     dest = alloc_page(GFP_KERNEL);
0935     if (!dest) {
0936         while (src_idx--)
0937             __free_page(xor_srcs[src_idx]);
0938         return -ENOMEM;
0939     }
0940 
0941     /* Fill in src buffers */
0942     for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) {
0943         u8 *ptr = page_address(xor_srcs[src_idx]);
0944         for (i = 0; i < PAGE_SIZE; i++)
0945             ptr[i] = (1 << src_idx);
0946     }
0947 
0948     for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++)
0949         cmp_byte ^= (u8) (1 << src_idx);
0950 
0951     cmp_word = (cmp_byte << 24) | (cmp_byte << 16) |
0952             (cmp_byte << 8) | cmp_byte;
0953 
0954     memset(page_address(dest), 0, PAGE_SIZE);
0955 
0956     dma_chan = container_of(device->common.channels.next,
0957                 struct dma_chan,
0958                 device_node);
0959     if (iop_adma_alloc_chan_resources(dma_chan) < 1) {
0960         err = -ENODEV;
0961         goto out;
0962     }
0963 
0964     /* test xor */
0965     dest_dma = dma_map_page(dma_chan->device->dev, dest, 0,
0966                 PAGE_SIZE, DMA_FROM_DEVICE);
0967     for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++)
0968         dma_srcs[i] = dma_map_page(dma_chan->device->dev, xor_srcs[i],
0969                        0, PAGE_SIZE, DMA_TO_DEVICE);
0970     tx = iop_adma_prep_dma_xor(dma_chan, dest_dma, dma_srcs,
0971                    IOP_ADMA_NUM_SRC_TEST, PAGE_SIZE,
0972                    DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
0973 
0974     cookie = iop_adma_tx_submit(tx);
0975     iop_adma_issue_pending(dma_chan);
0976     msleep(8);
0977 
0978     if (iop_adma_status(dma_chan, cookie, NULL) !=
0979         DMA_COMPLETE) {
0980         dev_err(dma_chan->device->dev,
0981             "Self-test xor timed out, disabling\n");
0982         err = -ENODEV;
0983         goto free_resources;
0984     }
0985 
0986     iop_chan = to_iop_adma_chan(dma_chan);
0987     dma_sync_single_for_cpu(&iop_chan->device->pdev->dev, dest_dma,
0988         PAGE_SIZE, DMA_FROM_DEVICE);
0989     for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) {
0990         u32 *ptr = page_address(dest);
0991         if (ptr[i] != cmp_word) {
0992             dev_err(dma_chan->device->dev,
0993                 "Self-test xor failed compare, disabling\n");
0994             err = -ENODEV;
0995             goto free_resources;
0996         }
0997     }
0998     dma_sync_single_for_device(&iop_chan->device->pdev->dev, dest_dma,
0999         PAGE_SIZE, DMA_TO_DEVICE);
1000 
1001     /* skip zero sum if the capability is not present */
1002     if (!dma_has_cap(DMA_XOR_VAL, dma_chan->device->cap_mask))
1003         goto free_resources;
1004 
1005     /* zero sum the sources with the destintation page */
1006     for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++)
1007         zero_sum_srcs[i] = xor_srcs[i];
1008     zero_sum_srcs[i] = dest;
1009 
1010     zero_sum_result = 1;
1011 
1012     for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 1; i++)
1013         dma_srcs[i] = dma_map_page(dma_chan->device->dev,
1014                        zero_sum_srcs[i], 0, PAGE_SIZE,
1015                        DMA_TO_DEVICE);
1016     tx = iop_adma_prep_dma_xor_val(dma_chan, dma_srcs,
1017                        IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE,
1018                        &zero_sum_result,
1019                        DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1020 
1021     cookie = iop_adma_tx_submit(tx);
1022     iop_adma_issue_pending(dma_chan);
1023     msleep(8);
1024 
1025     if (iop_adma_status(dma_chan, cookie, NULL) != DMA_COMPLETE) {
1026         dev_err(dma_chan->device->dev,
1027             "Self-test zero sum timed out, disabling\n");
1028         err = -ENODEV;
1029         goto free_resources;
1030     }
1031 
1032     if (zero_sum_result != 0) {
1033         dev_err(dma_chan->device->dev,
1034             "Self-test zero sum failed compare, disabling\n");
1035         err = -ENODEV;
1036         goto free_resources;
1037     }
1038 
1039     /* test for non-zero parity sum */
1040     zero_sum_result = 0;
1041     for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 1; i++)
1042         dma_srcs[i] = dma_map_page(dma_chan->device->dev,
1043                        zero_sum_srcs[i], 0, PAGE_SIZE,
1044                        DMA_TO_DEVICE);
1045     tx = iop_adma_prep_dma_xor_val(dma_chan, dma_srcs,
1046                        IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE,
1047                        &zero_sum_result,
1048                        DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1049 
1050     cookie = iop_adma_tx_submit(tx);
1051     iop_adma_issue_pending(dma_chan);
1052     msleep(8);
1053 
1054     if (iop_adma_status(dma_chan, cookie, NULL) != DMA_COMPLETE) {
1055         dev_err(dma_chan->device->dev,
1056             "Self-test non-zero sum timed out, disabling\n");
1057         err = -ENODEV;
1058         goto free_resources;
1059     }
1060 
1061     if (zero_sum_result != 1) {
1062         dev_err(dma_chan->device->dev,
1063             "Self-test non-zero sum failed compare, disabling\n");
1064         err = -ENODEV;
1065         goto free_resources;
1066     }
1067 
1068 free_resources:
1069     iop_adma_free_chan_resources(dma_chan);
1070 out:
1071     src_idx = IOP_ADMA_NUM_SRC_TEST;
1072     while (src_idx--)
1073         __free_page(xor_srcs[src_idx]);
1074     __free_page(dest);
1075     return err;
1076 }
1077 
1078 #ifdef CONFIG_RAID6_PQ
1079 static int
1080 iop_adma_pq_zero_sum_self_test(struct iop_adma_device *device)
1081 {
1082     /* combined sources, software pq results, and extra hw pq results */
1083     struct page *pq[IOP_ADMA_NUM_SRC_TEST+2+2];
1084     /* ptr to the extra hw pq buffers defined above */
1085     struct page **pq_hw = &pq[IOP_ADMA_NUM_SRC_TEST+2];
1086     /* address conversion buffers (dma_map / page_address) */
1087     void *pq_sw[IOP_ADMA_NUM_SRC_TEST+2];
1088     dma_addr_t pq_src[IOP_ADMA_NUM_SRC_TEST+2];
1089     dma_addr_t *pq_dest = &pq_src[IOP_ADMA_NUM_SRC_TEST];
1090 
1091     int i;
1092     struct dma_async_tx_descriptor *tx;
1093     struct dma_chan *dma_chan;
1094     dma_cookie_t cookie;
1095     u32 zero_sum_result;
1096     int err = 0;
1097     struct device *dev;
1098 
1099     dev_dbg(device->common.dev, "%s\n", __func__);
1100 
1101     for (i = 0; i < ARRAY_SIZE(pq); i++) {
1102         pq[i] = alloc_page(GFP_KERNEL);
1103         if (!pq[i]) {
1104             while (i--)
1105                 __free_page(pq[i]);
1106             return -ENOMEM;
1107         }
1108     }
1109 
1110     /* Fill in src buffers */
1111     for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) {
1112         pq_sw[i] = page_address(pq[i]);
1113         memset(pq_sw[i], 0x11111111 * (1<<i), PAGE_SIZE);
1114     }
1115     pq_sw[i] = page_address(pq[i]);
1116     pq_sw[i+1] = page_address(pq[i+1]);
1117 
1118     dma_chan = container_of(device->common.channels.next,
1119                 struct dma_chan,
1120                 device_node);
1121     if (iop_adma_alloc_chan_resources(dma_chan) < 1) {
1122         err = -ENODEV;
1123         goto out;
1124     }
1125 
1126     dev = dma_chan->device->dev;
1127 
1128     /* initialize the dests */
1129     memset(page_address(pq_hw[0]), 0 , PAGE_SIZE);
1130     memset(page_address(pq_hw[1]), 0 , PAGE_SIZE);
1131 
1132     /* test pq */
1133     pq_dest[0] = dma_map_page(dev, pq_hw[0], 0, PAGE_SIZE, DMA_FROM_DEVICE);
1134     pq_dest[1] = dma_map_page(dev, pq_hw[1], 0, PAGE_SIZE, DMA_FROM_DEVICE);
1135     for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++)
1136         pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE,
1137                      DMA_TO_DEVICE);
1138 
1139     tx = iop_adma_prep_dma_pq(dma_chan, pq_dest, pq_src,
1140                   IOP_ADMA_NUM_SRC_TEST, (u8 *)raid6_gfexp,
1141                   PAGE_SIZE,
1142                   DMA_PREP_INTERRUPT |
1143                   DMA_CTRL_ACK);
1144 
1145     cookie = iop_adma_tx_submit(tx);
1146     iop_adma_issue_pending(dma_chan);
1147     msleep(8);
1148 
1149     if (iop_adma_status(dma_chan, cookie, NULL) !=
1150         DMA_COMPLETE) {
1151         dev_err(dev, "Self-test pq timed out, disabling\n");
1152         err = -ENODEV;
1153         goto free_resources;
1154     }
1155 
1156     raid6_call.gen_syndrome(IOP_ADMA_NUM_SRC_TEST+2, PAGE_SIZE, pq_sw);
1157 
1158     if (memcmp(pq_sw[IOP_ADMA_NUM_SRC_TEST],
1159            page_address(pq_hw[0]), PAGE_SIZE) != 0) {
1160         dev_err(dev, "Self-test p failed compare, disabling\n");
1161         err = -ENODEV;
1162         goto free_resources;
1163     }
1164     if (memcmp(pq_sw[IOP_ADMA_NUM_SRC_TEST+1],
1165            page_address(pq_hw[1]), PAGE_SIZE) != 0) {
1166         dev_err(dev, "Self-test q failed compare, disabling\n");
1167         err = -ENODEV;
1168         goto free_resources;
1169     }
1170 
1171     /* test correct zero sum using the software generated pq values */
1172     for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 2; i++)
1173         pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE,
1174                      DMA_TO_DEVICE);
1175 
1176     zero_sum_result = ~0;
1177     tx = iop_adma_prep_dma_pq_val(dma_chan, &pq_src[IOP_ADMA_NUM_SRC_TEST],
1178                       pq_src, IOP_ADMA_NUM_SRC_TEST,
1179                       raid6_gfexp, PAGE_SIZE, &zero_sum_result,
1180                       DMA_PREP_INTERRUPT|DMA_CTRL_ACK);
1181 
1182     cookie = iop_adma_tx_submit(tx);
1183     iop_adma_issue_pending(dma_chan);
1184     msleep(8);
1185 
1186     if (iop_adma_status(dma_chan, cookie, NULL) !=
1187         DMA_COMPLETE) {
1188         dev_err(dev, "Self-test pq-zero-sum timed out, disabling\n");
1189         err = -ENODEV;
1190         goto free_resources;
1191     }
1192 
1193     if (zero_sum_result != 0) {
1194         dev_err(dev, "Self-test pq-zero-sum failed to validate: %x\n",
1195             zero_sum_result);
1196         err = -ENODEV;
1197         goto free_resources;
1198     }
1199 
1200     /* test incorrect zero sum */
1201     i = IOP_ADMA_NUM_SRC_TEST;
1202     memset(pq_sw[i] + 100, 0, 100);
1203     memset(pq_sw[i+1] + 200, 0, 200);
1204     for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 2; i++)
1205         pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE,
1206                      DMA_TO_DEVICE);
1207 
1208     zero_sum_result = 0;
1209     tx = iop_adma_prep_dma_pq_val(dma_chan, &pq_src[IOP_ADMA_NUM_SRC_TEST],
1210                       pq_src, IOP_ADMA_NUM_SRC_TEST,
1211                       raid6_gfexp, PAGE_SIZE, &zero_sum_result,
1212                       DMA_PREP_INTERRUPT|DMA_CTRL_ACK);
1213 
1214     cookie = iop_adma_tx_submit(tx);
1215     iop_adma_issue_pending(dma_chan);
1216     msleep(8);
1217 
1218     if (iop_adma_status(dma_chan, cookie, NULL) !=
1219         DMA_COMPLETE) {
1220         dev_err(dev, "Self-test !pq-zero-sum timed out, disabling\n");
1221         err = -ENODEV;
1222         goto free_resources;
1223     }
1224 
1225     if (zero_sum_result != (SUM_CHECK_P_RESULT | SUM_CHECK_Q_RESULT)) {
1226         dev_err(dev, "Self-test !pq-zero-sum failed to validate: %x\n",
1227             zero_sum_result);
1228         err = -ENODEV;
1229         goto free_resources;
1230     }
1231 
1232 free_resources:
1233     iop_adma_free_chan_resources(dma_chan);
1234 out:
1235     i = ARRAY_SIZE(pq);
1236     while (i--)
1237         __free_page(pq[i]);
1238     return err;
1239 }
1240 #endif
1241 
1242 static int iop_adma_remove(struct platform_device *dev)
1243 {
1244     struct iop_adma_device *device = platform_get_drvdata(dev);
1245     struct dma_chan *chan, *_chan;
1246     struct iop_adma_chan *iop_chan;
1247     struct iop_adma_platform_data *plat_data = dev_get_platdata(&dev->dev);
1248 
1249     dma_async_device_unregister(&device->common);
1250 
1251     dma_free_coherent(&dev->dev, plat_data->pool_size,
1252             device->dma_desc_pool_virt, device->dma_desc_pool);
1253 
1254     list_for_each_entry_safe(chan, _chan, &device->common.channels,
1255                 device_node) {
1256         iop_chan = to_iop_adma_chan(chan);
1257         list_del(&chan->device_node);
1258         kfree(iop_chan);
1259     }
1260     kfree(device);
1261 
1262     return 0;
1263 }
1264 
1265 static int iop_adma_probe(struct platform_device *pdev)
1266 {
1267     struct resource *res;
1268     int ret = 0, i;
1269     struct iop_adma_device *adev;
1270     struct iop_adma_chan *iop_chan;
1271     struct dma_device *dma_dev;
1272     struct iop_adma_platform_data *plat_data = dev_get_platdata(&pdev->dev);
1273 
1274     res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1275     if (!res)
1276         return -ENODEV;
1277 
1278     if (!devm_request_mem_region(&pdev->dev, res->start,
1279                 resource_size(res), pdev->name))
1280         return -EBUSY;
1281 
1282     adev = kzalloc(sizeof(*adev), GFP_KERNEL);
1283     if (!adev)
1284         return -ENOMEM;
1285     dma_dev = &adev->common;
1286 
1287     /* allocate coherent memory for hardware descriptors
1288      * note: writecombine gives slightly better performance, but
1289      * requires that we explicitly flush the writes
1290      */
1291     adev->dma_desc_pool_virt = dma_alloc_wc(&pdev->dev,
1292                         plat_data->pool_size,
1293                         &adev->dma_desc_pool,
1294                         GFP_KERNEL);
1295     if (!adev->dma_desc_pool_virt) {
1296         ret = -ENOMEM;
1297         goto err_free_adev;
1298     }
1299 
1300     dev_dbg(&pdev->dev, "%s: allocated descriptor pool virt %p phys %pad\n",
1301         __func__, adev->dma_desc_pool_virt, &adev->dma_desc_pool);
1302 
1303     adev->id = plat_data->hw_id;
1304 
1305     /* discover transaction capabilites from the platform data */
1306     dma_dev->cap_mask = plat_data->cap_mask;
1307 
1308     adev->pdev = pdev;
1309     platform_set_drvdata(pdev, adev);
1310 
1311     INIT_LIST_HEAD(&dma_dev->channels);
1312 
1313     /* set base routines */
1314     dma_dev->device_alloc_chan_resources = iop_adma_alloc_chan_resources;
1315     dma_dev->device_free_chan_resources = iop_adma_free_chan_resources;
1316     dma_dev->device_tx_status = iop_adma_status;
1317     dma_dev->device_issue_pending = iop_adma_issue_pending;
1318     dma_dev->dev = &pdev->dev;
1319 
1320     /* set prep routines based on capability */
1321     if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask))
1322         dma_dev->device_prep_dma_memcpy = iop_adma_prep_dma_memcpy;
1323     if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1324         dma_dev->max_xor = iop_adma_get_max_xor();
1325         dma_dev->device_prep_dma_xor = iop_adma_prep_dma_xor;
1326     }
1327     if (dma_has_cap(DMA_XOR_VAL, dma_dev->cap_mask))
1328         dma_dev->device_prep_dma_xor_val =
1329             iop_adma_prep_dma_xor_val;
1330     if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
1331         dma_set_maxpq(dma_dev, iop_adma_get_max_pq(), 0);
1332         dma_dev->device_prep_dma_pq = iop_adma_prep_dma_pq;
1333     }
1334     if (dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask))
1335         dma_dev->device_prep_dma_pq_val =
1336             iop_adma_prep_dma_pq_val;
1337     if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask))
1338         dma_dev->device_prep_dma_interrupt =
1339             iop_adma_prep_dma_interrupt;
1340 
1341     iop_chan = kzalloc(sizeof(*iop_chan), GFP_KERNEL);
1342     if (!iop_chan) {
1343         ret = -ENOMEM;
1344         goto err_free_dma;
1345     }
1346     iop_chan->device = adev;
1347 
1348     iop_chan->mmr_base = devm_ioremap(&pdev->dev, res->start,
1349                     resource_size(res));
1350     if (!iop_chan->mmr_base) {
1351         ret = -ENOMEM;
1352         goto err_free_iop_chan;
1353     }
1354     tasklet_setup(&iop_chan->irq_tasklet, iop_adma_tasklet);
1355 
1356     /* clear errors before enabling interrupts */
1357     iop_adma_device_clear_err_status(iop_chan);
1358 
1359     for (i = 0; i < 3; i++) {
1360         static const irq_handler_t handler[] = {
1361             iop_adma_eot_handler,
1362             iop_adma_eoc_handler,
1363             iop_adma_err_handler
1364         };
1365         int irq = platform_get_irq(pdev, i);
1366         if (irq < 0) {
1367             ret = -ENXIO;
1368             goto err_free_iop_chan;
1369         } else {
1370             ret = devm_request_irq(&pdev->dev, irq,
1371                     handler[i], 0, pdev->name, iop_chan);
1372             if (ret)
1373                 goto err_free_iop_chan;
1374         }
1375     }
1376 
1377     spin_lock_init(&iop_chan->lock);
1378     INIT_LIST_HEAD(&iop_chan->chain);
1379     INIT_LIST_HEAD(&iop_chan->all_slots);
1380     iop_chan->common.device = dma_dev;
1381     dma_cookie_init(&iop_chan->common);
1382     list_add_tail(&iop_chan->common.device_node, &dma_dev->channels);
1383 
1384     if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
1385         ret = iop_adma_memcpy_self_test(adev);
1386         dev_dbg(&pdev->dev, "memcpy self test returned %d\n", ret);
1387         if (ret)
1388             goto err_free_iop_chan;
1389     }
1390 
1391     if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1392         ret = iop_adma_xor_val_self_test(adev);
1393         dev_dbg(&pdev->dev, "xor self test returned %d\n", ret);
1394         if (ret)
1395             goto err_free_iop_chan;
1396     }
1397 
1398     if (dma_has_cap(DMA_PQ, dma_dev->cap_mask) &&
1399         dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask)) {
1400         #ifdef CONFIG_RAID6_PQ
1401         ret = iop_adma_pq_zero_sum_self_test(adev);
1402         dev_dbg(&pdev->dev, "pq self test returned %d\n", ret);
1403         #else
1404         /* can not test raid6, so do not publish capability */
1405         dma_cap_clear(DMA_PQ, dma_dev->cap_mask);
1406         dma_cap_clear(DMA_PQ_VAL, dma_dev->cap_mask);
1407         ret = 0;
1408         #endif
1409         if (ret)
1410             goto err_free_iop_chan;
1411     }
1412 
1413     dev_info(&pdev->dev, "Intel(R) IOP: ( %s%s%s%s%s%s)\n",
1414          dma_has_cap(DMA_PQ, dma_dev->cap_mask) ? "pq " : "",
1415          dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask) ? "pq_val " : "",
1416          dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "",
1417          dma_has_cap(DMA_XOR_VAL, dma_dev->cap_mask) ? "xor_val " : "",
1418          dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "cpy " : "",
1419          dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "intr " : "");
1420 
1421     dma_async_device_register(dma_dev);
1422     goto out;
1423 
1424  err_free_iop_chan:
1425     kfree(iop_chan);
1426  err_free_dma:
1427     dma_free_coherent(&adev->pdev->dev, plat_data->pool_size,
1428             adev->dma_desc_pool_virt, adev->dma_desc_pool);
1429  err_free_adev:
1430     kfree(adev);
1431  out:
1432     return ret;
1433 }
1434 
1435 static void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan)
1436 {
1437     struct iop_adma_desc_slot *sw_desc, *grp_start;
1438     dma_cookie_t cookie;
1439     int slot_cnt, slots_per_op;
1440 
1441     dev_dbg(iop_chan->device->common.dev, "%s\n", __func__);
1442 
1443     spin_lock_bh(&iop_chan->lock);
1444     slot_cnt = iop_chan_memcpy_slot_count(0, &slots_per_op);
1445     sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
1446     if (sw_desc) {
1447         grp_start = sw_desc->group_head;
1448 
1449         list_splice_init(&sw_desc->tx_list, &iop_chan->chain);
1450         async_tx_ack(&sw_desc->async_tx);
1451         iop_desc_init_memcpy(grp_start, 0);
1452         iop_desc_set_byte_count(grp_start, iop_chan, 0);
1453         iop_desc_set_dest_addr(grp_start, iop_chan, 0);
1454         iop_desc_set_memcpy_src_addr(grp_start, 0);
1455 
1456         cookie = dma_cookie_assign(&sw_desc->async_tx);
1457 
1458         /* initialize the completed cookie to be less than
1459          * the most recently used cookie
1460          */
1461         iop_chan->common.completed_cookie = cookie - 1;
1462 
1463         /* channel should not be busy */
1464         BUG_ON(iop_chan_is_busy(iop_chan));
1465 
1466         /* clear any prior error-status bits */
1467         iop_adma_device_clear_err_status(iop_chan);
1468 
1469         /* disable operation */
1470         iop_chan_disable(iop_chan);
1471 
1472         /* set the descriptor address */
1473         iop_chan_set_next_descriptor(iop_chan, sw_desc->async_tx.phys);
1474 
1475         /* 1/ don't add pre-chained descriptors
1476          * 2/ dummy read to flush next_desc write
1477          */
1478         BUG_ON(iop_desc_get_next_desc(sw_desc));
1479 
1480         /* run the descriptor */
1481         iop_chan_enable(iop_chan);
1482     } else
1483         dev_err(iop_chan->device->common.dev,
1484             "failed to allocate null descriptor\n");
1485     spin_unlock_bh(&iop_chan->lock);
1486 }
1487 
1488 static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan)
1489 {
1490     struct iop_adma_desc_slot *sw_desc, *grp_start;
1491     dma_cookie_t cookie;
1492     int slot_cnt, slots_per_op;
1493 
1494     dev_dbg(iop_chan->device->common.dev, "%s\n", __func__);
1495 
1496     spin_lock_bh(&iop_chan->lock);
1497     slot_cnt = iop_chan_xor_slot_count(0, 2, &slots_per_op);
1498     sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
1499     if (sw_desc) {
1500         grp_start = sw_desc->group_head;
1501         list_splice_init(&sw_desc->tx_list, &iop_chan->chain);
1502         async_tx_ack(&sw_desc->async_tx);
1503         iop_desc_init_null_xor(grp_start, 2, 0);
1504         iop_desc_set_byte_count(grp_start, iop_chan, 0);
1505         iop_desc_set_dest_addr(grp_start, iop_chan, 0);
1506         iop_desc_set_xor_src_addr(grp_start, 0, 0);
1507         iop_desc_set_xor_src_addr(grp_start, 1, 0);
1508 
1509         cookie = dma_cookie_assign(&sw_desc->async_tx);
1510 
1511         /* initialize the completed cookie to be less than
1512          * the most recently used cookie
1513          */
1514         iop_chan->common.completed_cookie = cookie - 1;
1515 
1516         /* channel should not be busy */
1517         BUG_ON(iop_chan_is_busy(iop_chan));
1518 
1519         /* clear any prior error-status bits */
1520         iop_adma_device_clear_err_status(iop_chan);
1521 
1522         /* disable operation */
1523         iop_chan_disable(iop_chan);
1524 
1525         /* set the descriptor address */
1526         iop_chan_set_next_descriptor(iop_chan, sw_desc->async_tx.phys);
1527 
1528         /* 1/ don't add pre-chained descriptors
1529          * 2/ dummy read to flush next_desc write
1530          */
1531         BUG_ON(iop_desc_get_next_desc(sw_desc));
1532 
1533         /* run the descriptor */
1534         iop_chan_enable(iop_chan);
1535     } else
1536         dev_err(iop_chan->device->common.dev,
1537             "failed to allocate null descriptor\n");
1538     spin_unlock_bh(&iop_chan->lock);
1539 }
1540 
1541 static struct platform_driver iop_adma_driver = {
1542     .probe      = iop_adma_probe,
1543     .remove     = iop_adma_remove,
1544     .driver     = {
1545         .name   = "iop-adma",
1546     },
1547 };
1548 
1549 module_platform_driver(iop_adma_driver);
1550 
1551 MODULE_AUTHOR("Intel Corporation");
1552 MODULE_DESCRIPTION("IOP ADMA Engine Driver");
1553 MODULE_LICENSE("GPL");
1554 MODULE_ALIAS("platform:iop-adma");
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!