crypto/async_tx/async_xor.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * xor offload engine api
0004  *
0005  * Copyright © 2006, Intel Corporation.
0006  *
0007  *      Dan Williams <dan.j.williams@intel.com>
0008  *
0009  *      with architecture considerations by:
0010  *      Neil Brown <neilb@suse.de>
0011  *      Jeff Garzik <jeff@garzik.org>
0012  */
0013 #include <linux/kernel.h>
0014 #include <linux/interrupt.h>
0015 #include <linux/module.h>
0016 #include <linux/mm.h>
0017 #include <linux/dma-mapping.h>
0018 #include <linux/raid/xor.h>
0019 #include <linux/async_tx.h>
0020
0021 /* do_async_xor - dma map the pages and perform the xor with an engine */
0022 static __async_inline struct dma_async_tx_descriptor *
0023 do_async_xor(struct dma_chan *chan, struct dmaengine_unmap_data *unmap,
0024          struct async_submit_ctl *submit)
0025 {
0026     struct dma_device *dma = chan->device;
0027     struct dma_async_tx_descriptor *tx = NULL;
0028     dma_async_tx_callback cb_fn_orig = submit->cb_fn;
0029     void *cb_param_orig = submit->cb_param;
0030     enum async_tx_flags flags_orig = submit->flags;
0031     enum dma_ctrl_flags dma_flags = 0;
0032     int src_cnt = unmap->to_cnt;
0033     int xor_src_cnt;
0034     dma_addr_t dma_dest = unmap->addr[unmap->to_cnt];
0035     dma_addr_t *src_list = unmap->addr;
0036
0037     while (src_cnt) {
0038         dma_addr_t tmp;
0039
0040         submit->flags = flags_orig;
0041         xor_src_cnt = min(src_cnt, (int)dma->max_xor);
0042         /* if we are submitting additional xors, leave the chain open
0043          * and clear the callback parameters
0044          */
0045         if (src_cnt > xor_src_cnt) {
0046             submit->flags &= ~ASYNC_TX_ACK;
0047             submit->flags |= ASYNC_TX_FENCE;
0048             submit->cb_fn = NULL;
0049             submit->cb_param = NULL;
0050         } else {
0051             submit->cb_fn = cb_fn_orig;
0052             submit->cb_param = cb_param_orig;
0053         }
0054         if (submit->cb_fn)
0055             dma_flags |= DMA_PREP_INTERRUPT;
0056         if (submit->flags & ASYNC_TX_FENCE)
0057             dma_flags |= DMA_PREP_FENCE;
0058
0059         /* Drivers force forward progress in case they can not provide a
0060          * descriptor
0061          */
0062         tmp = src_list[0];
0063         if (src_list > unmap->addr)
0064             src_list[0] = dma_dest;
0065         tx = dma->device_prep_dma_xor(chan, dma_dest, src_list,
0066                           xor_src_cnt, unmap->len,
0067                           dma_flags);
0068
0069         if (unlikely(!tx))
0070             async_tx_quiesce(&submit->depend_tx);
0071
0072         /* spin wait for the preceding transactions to complete */
0073         while (unlikely(!tx)) {
0074             dma_async_issue_pending(chan);
0075             tx = dma->device_prep_dma_xor(chan, dma_dest,
0076                               src_list,
0077                               xor_src_cnt, unmap->len,
0078                               dma_flags);
0079         }
0080         src_list[0] = tmp;
0081
0082         dma_set_unmap(tx, unmap);
0083         async_tx_submit(chan, tx, submit);
0084         submit->depend_tx = tx;
0085
0086         if (src_cnt > xor_src_cnt) {
0087             /* drop completed sources */
0088             src_cnt -= xor_src_cnt;
0089             /* use the intermediate result a source */
0090             src_cnt++;
0091             src_list += xor_src_cnt - 1;
0092         } else
0093             break;
0094     }
0095
0096     return tx;
0097 }
0098
0099 static void
0100 do_sync_xor_offs(struct page *dest, unsigned int offset,
0101         struct page **src_list, unsigned int *src_offs,
0102         int src_cnt, size_t len, struct async_submit_ctl *submit)
0103 {
0104     int i;
0105     int xor_src_cnt = 0;
0106     int src_off = 0;
0107     void *dest_buf;
0108     void **srcs;
0109
0110     if (submit->scribble)
0111         srcs = submit->scribble;
0112     else
0113         srcs = (void **) src_list;
0114
0115     /* convert to buffer pointers */
0116     for (i = 0; i < src_cnt; i++)
0117         if (src_list[i])
0118             srcs[xor_src_cnt++] = page_address(src_list[i]) +
0119                 (src_offs ? src_offs[i] : offset);
0120     src_cnt = xor_src_cnt;
0121     /* set destination address */
0122     dest_buf = page_address(dest) + offset;
0123
0124     if (submit->flags & ASYNC_TX_XOR_ZERO_DST)
0125         memset(dest_buf, 0, len);
0126
0127     while (src_cnt > 0) {
0128         /* process up to 'MAX_XOR_BLOCKS' sources */
0129         xor_src_cnt = min(src_cnt, MAX_XOR_BLOCKS);
0130         xor_blocks(xor_src_cnt, len, dest_buf, &srcs[src_off]);
0131
0132         /* drop completed sources */
0133         src_cnt -= xor_src_cnt;
0134         src_off += xor_src_cnt;
0135     }
0136
0137     async_tx_sync_epilog(submit);
0138 }
0139
0140 static inline bool
0141 dma_xor_aligned_offsets(struct dma_device *device, unsigned int offset,
0142         unsigned int *src_offs, int src_cnt, int len)
0143 {
0144     int i;
0145
0146     if (!is_dma_xor_aligned(device, offset, 0, len))
0147         return false;
0148
0149     if (!src_offs)
0150         return true;
0151
0152     for (i = 0; i < src_cnt; i++) {
0153         if (!is_dma_xor_aligned(device, src_offs[i], 0, len))
0154             return false;
0155     }
0156     return true;
0157 }
0158
0159 /**
0160  * async_xor_offs - attempt to xor a set of blocks with a dma engine.
0161  * @dest: destination page
0162  * @offset: dst offset to start transaction
0163  * @src_list: array of source pages
0164  * @src_offs: array of source pages offset, NULL means common src/dst offset
0165  * @src_cnt: number of source pages
0166  * @len: length in bytes
0167  * @submit: submission / completion modifiers
0168  *
0169  * honored flags: ASYNC_TX_ACK, ASYNC_TX_XOR_ZERO_DST, ASYNC_TX_XOR_DROP_DST
0170  *
0171  * xor_blocks always uses the dest as a source so the
0172  * ASYNC_TX_XOR_ZERO_DST flag must be set to not include dest data in
0173  * the calculation.  The assumption with dma engines is that they only
0174  * use the destination buffer as a source when it is explicitly specified
0175  * in the source list.
0176  *
0177  * src_list note: if the dest is also a source it must be at index zero.
0178  * The contents of this array will be overwritten if a scribble region
0179  * is not specified.
0180  */
0181 struct dma_async_tx_descriptor *
0182 async_xor_offs(struct page *dest, unsigned int offset,
0183         struct page **src_list, unsigned int *src_offs,
0184         int src_cnt, size_t len, struct async_submit_ctl *submit)
0185 {
0186     struct dma_chan *chan = async_tx_find_channel(submit, DMA_XOR,
0187                               &dest, 1, src_list,
0188                               src_cnt, len);
0189     struct dma_device *device = chan ? chan->device : NULL;
0190     struct dmaengine_unmap_data *unmap = NULL;
0191
0192     BUG_ON(src_cnt <= 1);
0193
0194     if (device)
0195         unmap = dmaengine_get_unmap_data(device->dev, src_cnt+1, GFP_NOWAIT);
0196
0197     if (unmap && dma_xor_aligned_offsets(device, offset,
0198                 src_offs, src_cnt, len)) {
0199         struct dma_async_tx_descriptor *tx;
0200         int i, j;
0201
0202         /* run the xor asynchronously */
0203         pr_debug("%s (async): len: %zu\n", __func__, len);
0204
0205         unmap->len = len;
0206         for (i = 0, j = 0; i < src_cnt; i++) {
0207             if (!src_list[i])
0208                 continue;
0209             unmap->to_cnt++;
0210             unmap->addr[j++] = dma_map_page(device->dev, src_list[i],
0211                     src_offs ? src_offs[i] : offset,
0212                     len, DMA_TO_DEVICE);
0213         }
0214
0215         /* map it bidirectional as it may be re-used as a source */
0216         unmap->addr[j] = dma_map_page(device->dev, dest, offset, len,
0217                           DMA_BIDIRECTIONAL);
0218         unmap->bidi_cnt = 1;
0219
0220         tx = do_async_xor(chan, unmap, submit);
0221         dmaengine_unmap_put(unmap);
0222         return tx;
0223     } else {
0224         dmaengine_unmap_put(unmap);
0225         /* run the xor synchronously */
0226         pr_debug("%s (sync): len: %zu\n", __func__, len);
0227         WARN_ONCE(chan, "%s: no space for dma address conversion\n",
0228               __func__);
0229
0230         /* in the sync case the dest is an implied source
0231          * (assumes the dest is the first source)
0232          */
0233         if (submit->flags & ASYNC_TX_XOR_DROP_DST) {
0234             src_cnt--;
0235             src_list++;
0236             if (src_offs)
0237                 src_offs++;
0238         }
0239
0240         /* wait for any prerequisite operations */
0241         async_tx_quiesce(&submit->depend_tx);
0242
0243         do_sync_xor_offs(dest, offset, src_list, src_offs,
0244                 src_cnt, len, submit);
0245
0246         return NULL;
0247     }
0248 }
0249 EXPORT_SYMBOL_GPL(async_xor_offs);
0250
0251 /**
0252  * async_xor - attempt to xor a set of blocks with a dma engine.
0253  * @dest: destination page
0254  * @src_list: array of source pages
0255  * @offset: common src/dst offset to start transaction
0256  * @src_cnt: number of source pages
0257  * @len: length in bytes
0258  * @submit: submission / completion modifiers
0259  *
0260  * honored flags: ASYNC_TX_ACK, ASYNC_TX_XOR_ZERO_DST, ASYNC_TX_XOR_DROP_DST
0261  *
0262  * xor_blocks always uses the dest as a source so the
0263  * ASYNC_TX_XOR_ZERO_DST flag must be set to not include dest data in
0264  * the calculation.  The assumption with dma engines is that they only
0265  * use the destination buffer as a source when it is explicitly specified
0266  * in the source list.
0267  *
0268  * src_list note: if the dest is also a source it must be at index zero.
0269  * The contents of this array will be overwritten if a scribble region
0270  * is not specified.
0271  */
0272 struct dma_async_tx_descriptor *
0273 async_xor(struct page *dest, struct page **src_list, unsigned int offset,
0274       int src_cnt, size_t len, struct async_submit_ctl *submit)
0275 {
0276     return async_xor_offs(dest, offset, src_list, NULL,
0277             src_cnt, len, submit);
0278 }
0279 EXPORT_SYMBOL_GPL(async_xor);
0280
0281 static int page_is_zero(struct page *p, unsigned int offset, size_t len)
0282 {
0283     return !memchr_inv(page_address(p) + offset, 0, len);
0284 }
0285
0286 static inline struct dma_chan *
0287 xor_val_chan(struct async_submit_ctl *submit, struct page *dest,
0288          struct page **src_list, int src_cnt, size_t len)
0289 {
0290     #ifdef CONFIG_ASYNC_TX_DISABLE_XOR_VAL_DMA
0291     return NULL;
0292     #endif
0293     return async_tx_find_channel(submit, DMA_XOR_VAL, &dest, 1, src_list,
0294                      src_cnt, len);
0295 }
0296
0297 /**
0298  * async_xor_val_offs - attempt a xor parity check with a dma engine.
0299  * @dest: destination page used if the xor is performed synchronously
0300  * @offset: des offset in pages to start transaction
0301  * @src_list: array of source pages
0302  * @src_offs: array of source pages offset, NULL means common src/det offset
0303  * @src_cnt: number of source pages
0304  * @len: length in bytes
0305  * @result: 0 if sum == 0 else non-zero
0306  * @submit: submission / completion modifiers
0307  *
0308  * honored flags: ASYNC_TX_ACK
0309  *
0310  * src_list note: if the dest is also a source it must be at index zero.
0311  * The contents of this array will be overwritten if a scribble region
0312  * is not specified.
0313  */
0314 struct dma_async_tx_descriptor *
0315 async_xor_val_offs(struct page *dest, unsigned int offset,
0316         struct page **src_list, unsigned int *src_offs,
0317         int src_cnt, size_t len, enum sum_check_flags *result,
0318         struct async_submit_ctl *submit)
0319 {
0320     struct dma_chan *chan = xor_val_chan(submit, dest, src_list, src_cnt, len);
0321     struct dma_device *device = chan ? chan->device : NULL;
0322     struct dma_async_tx_descriptor *tx = NULL;
0323     struct dmaengine_unmap_data *unmap = NULL;
0324
0325     BUG_ON(src_cnt <= 1);
0326
0327     if (device)
0328         unmap = dmaengine_get_unmap_data(device->dev, src_cnt, GFP_NOWAIT);
0329
0330     if (unmap && src_cnt <= device->max_xor &&
0331         dma_xor_aligned_offsets(device, offset, src_offs, src_cnt, len)) {
0332         unsigned long dma_prep_flags = 0;
0333         int i;
0334
0335         pr_debug("%s: (async) len: %zu\n", __func__, len);
0336
0337         if (submit->cb_fn)
0338             dma_prep_flags |= DMA_PREP_INTERRUPT;
0339         if (submit->flags & ASYNC_TX_FENCE)
0340             dma_prep_flags |= DMA_PREP_FENCE;
0341
0342         for (i = 0; i < src_cnt; i++) {
0343             unmap->addr[i] = dma_map_page(device->dev, src_list[i],
0344                     src_offs ? src_offs[i] : offset,
0345                     len, DMA_TO_DEVICE);
0346             unmap->to_cnt++;
0347         }
0348         unmap->len = len;
0349
0350         tx = device->device_prep_dma_xor_val(chan, unmap->addr, src_cnt,
0351                              len, result,
0352                              dma_prep_flags);
0353         if (unlikely(!tx)) {
0354             async_tx_quiesce(&submit->depend_tx);
0355
0356             while (!tx) {
0357                 dma_async_issue_pending(chan);
0358                 tx = device->device_prep_dma_xor_val(chan,
0359                     unmap->addr, src_cnt, len, result,
0360                     dma_prep_flags);
0361             }
0362         }
0363         dma_set_unmap(tx, unmap);
0364         async_tx_submit(chan, tx, submit);
0365     } else {
0366         enum async_tx_flags flags_orig = submit->flags;
0367
0368         pr_debug("%s: (sync) len: %zu\n", __func__, len);
0369         WARN_ONCE(device && src_cnt <= device->max_xor,
0370               "%s: no space for dma address conversion\n",
0371               __func__);
0372
0373         submit->flags |= ASYNC_TX_XOR_DROP_DST;
0374         submit->flags &= ~ASYNC_TX_ACK;
0375
0376         tx = async_xor_offs(dest, offset, src_list, src_offs,
0377                 src_cnt, len, submit);
0378
0379         async_tx_quiesce(&tx);
0380
0381         *result = !page_is_zero(dest, offset, len) << SUM_CHECK_P;
0382
0383         async_tx_sync_epilog(submit);
0384         submit->flags = flags_orig;
0385     }
0386     dmaengine_unmap_put(unmap);
0387
0388     return tx;
0389 }
0390 EXPORT_SYMBOL_GPL(async_xor_val_offs);
0391
0392 /**
0393  * async_xor_val - attempt a xor parity check with a dma engine.
0394  * @dest: destination page used if the xor is performed synchronously
0395  * @src_list: array of source pages
0396  * @offset: offset in pages to start transaction
0397  * @src_cnt: number of source pages
0398  * @len: length in bytes
0399  * @result: 0 if sum == 0 else non-zero
0400  * @submit: submission / completion modifiers
0401  *
0402  * honored flags: ASYNC_TX_ACK
0403  *
0404  * src_list note: if the dest is also a source it must be at index zero.
0405  * The contents of this array will be overwritten if a scribble region
0406  * is not specified.
0407  */
0408 struct dma_async_tx_descriptor *
0409 async_xor_val(struct page *dest, struct page **src_list, unsigned int offset,
0410           int src_cnt, size_t len, enum sum_check_flags *result,
0411           struct async_submit_ctl *submit)
0412 {
0413     return async_xor_val_offs(dest, offset, src_list, NULL, src_cnt,
0414             len, result, submit);
0415 }
0416 EXPORT_SYMBOL_GPL(async_xor_val);
0417
0418 MODULE_AUTHOR("Intel Corporation");
0419 MODULE_DESCRIPTION("asynchronous xor/xor-zero-sum api");
0420 MODULE_LICENSE("GPL");