crypto/ccp/ccp-ops.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * AMD Cryptographic Coprocessor (CCP) driver
0004  *
0005  * Copyright (C) 2013-2019 Advanced Micro Devices, Inc.
0006  *
0007  * Author: Tom Lendacky <thomas.lendacky@amd.com>
0008  * Author: Gary R Hook <gary.hook@amd.com>
0009  */
0010
0011 #include <linux/dma-mapping.h>
0012 #include <linux/module.h>
0013 #include <linux/kernel.h>
0014 #include <linux/interrupt.h>
0015 #include <crypto/scatterwalk.h>
0016 #include <crypto/des.h>
0017 #include <linux/ccp.h>
0018
0019 #include "ccp-dev.h"
0020
0021 /* SHA initial context values */
0022 static const __be32 ccp_sha1_init[SHA1_DIGEST_SIZE / sizeof(__be32)] = {
0023     cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1),
0024     cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3),
0025     cpu_to_be32(SHA1_H4),
0026 };
0027
0028 static const __be32 ccp_sha224_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
0029     cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1),
0030     cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3),
0031     cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5),
0032     cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7),
0033 };
0034
0035 static const __be32 ccp_sha256_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
0036     cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1),
0037     cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3),
0038     cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5),
0039     cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7),
0040 };
0041
0042 static const __be64 ccp_sha384_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {
0043     cpu_to_be64(SHA384_H0), cpu_to_be64(SHA384_H1),
0044     cpu_to_be64(SHA384_H2), cpu_to_be64(SHA384_H3),
0045     cpu_to_be64(SHA384_H4), cpu_to_be64(SHA384_H5),
0046     cpu_to_be64(SHA384_H6), cpu_to_be64(SHA384_H7),
0047 };
0048
0049 static const __be64 ccp_sha512_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {
0050     cpu_to_be64(SHA512_H0), cpu_to_be64(SHA512_H1),
0051     cpu_to_be64(SHA512_H2), cpu_to_be64(SHA512_H3),
0052     cpu_to_be64(SHA512_H4), cpu_to_be64(SHA512_H5),
0053     cpu_to_be64(SHA512_H6), cpu_to_be64(SHA512_H7),
0054 };
0055
0056 #define CCP_NEW_JOBID(ccp)  ((ccp->vdata->version == CCP_VERSION(3, 0)) ? \
0057                     ccp_gen_jobid(ccp) : 0)
0058
0059 static u32 ccp_gen_jobid(struct ccp_device *ccp)
0060 {
0061     return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK;
0062 }
0063
0064 static void ccp_sg_free(struct ccp_sg_workarea *wa)
0065 {
0066     if (wa->dma_count)
0067         dma_unmap_sg(wa->dma_dev, wa->dma_sg_head, wa->nents, wa->dma_dir);
0068
0069     wa->dma_count = 0;
0070 }
0071
0072 static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev,
0073                 struct scatterlist *sg, u64 len,
0074                 enum dma_data_direction dma_dir)
0075 {
0076     memset(wa, 0, sizeof(*wa));
0077
0078     wa->sg = sg;
0079     if (!sg)
0080         return 0;
0081
0082     wa->nents = sg_nents_for_len(sg, len);
0083     if (wa->nents < 0)
0084         return wa->nents;
0085
0086     wa->bytes_left = len;
0087     wa->sg_used = 0;
0088
0089     if (len == 0)
0090         return 0;
0091
0092     if (dma_dir == DMA_NONE)
0093         return 0;
0094
0095     wa->dma_sg = sg;
0096     wa->dma_sg_head = sg;
0097     wa->dma_dev = dev;
0098     wa->dma_dir = dma_dir;
0099     wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir);
0100     if (!wa->dma_count)
0101         return -ENOMEM;
0102
0103     return 0;
0104 }
0105
0106 static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len)
0107 {
0108     unsigned int nbytes = min_t(u64, len, wa->bytes_left);
0109     unsigned int sg_combined_len = 0;
0110
0111     if (!wa->sg)
0112         return;
0113
0114     wa->sg_used += nbytes;
0115     wa->bytes_left -= nbytes;
0116     if (wa->sg_used == sg_dma_len(wa->dma_sg)) {
0117         /* Advance to the next DMA scatterlist entry */
0118         wa->dma_sg = sg_next(wa->dma_sg);
0119
0120         /* In the case that the DMA mapped scatterlist has entries
0121          * that have been merged, the non-DMA mapped scatterlist
0122          * must be advanced multiple times for each merged entry.
0123          * This ensures that the current non-DMA mapped entry
0124          * corresponds to the current DMA mapped entry.
0125          */
0126         do {
0127             sg_combined_len += wa->sg->length;
0128             wa->sg = sg_next(wa->sg);
0129         } while (wa->sg_used > sg_combined_len);
0130
0131         wa->sg_used = 0;
0132     }
0133 }
0134
0135 static void ccp_dm_free(struct ccp_dm_workarea *wa)
0136 {
0137     if (wa->length <= CCP_DMAPOOL_MAX_SIZE) {
0138         if (wa->address)
0139             dma_pool_free(wa->dma_pool, wa->address,
0140                       wa->dma.address);
0141     } else {
0142         if (wa->dma.address)
0143             dma_unmap_single(wa->dev, wa->dma.address, wa->length,
0144                      wa->dma.dir);
0145         kfree(wa->address);
0146     }
0147
0148     wa->address = NULL;
0149     wa->dma.address = 0;
0150 }
0151
0152 static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa,
0153                 struct ccp_cmd_queue *cmd_q,
0154                 unsigned int len,
0155                 enum dma_data_direction dir)
0156 {
0157     memset(wa, 0, sizeof(*wa));
0158
0159     if (!len)
0160         return 0;
0161
0162     wa->dev = cmd_q->ccp->dev;
0163     wa->length = len;
0164
0165     if (len <= CCP_DMAPOOL_MAX_SIZE) {
0166         wa->dma_pool = cmd_q->dma_pool;
0167
0168         wa->address = dma_pool_zalloc(wa->dma_pool, GFP_KERNEL,
0169                          &wa->dma.address);
0170         if (!wa->address)
0171             return -ENOMEM;
0172
0173         wa->dma.length = CCP_DMAPOOL_MAX_SIZE;
0174
0175     } else {
0176         wa->address = kzalloc(len, GFP_KERNEL);
0177         if (!wa->address)
0178             return -ENOMEM;
0179
0180         wa->dma.address = dma_map_single(wa->dev, wa->address, len,
0181                          dir);
0182         if (dma_mapping_error(wa->dev, wa->dma.address))
0183             return -ENOMEM;
0184
0185         wa->dma.length = len;
0186     }
0187     wa->dma.dir = dir;
0188
0189     return 0;
0190 }
0191
0192 static int ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
0193                struct scatterlist *sg, unsigned int sg_offset,
0194                unsigned int len)
0195 {
0196     WARN_ON(!wa->address);
0197
0198     if (len > (wa->length - wa_offset))
0199         return -EINVAL;
0200
0201     scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
0202                  0);
0203     return 0;
0204 }
0205
0206 static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
0207                 struct scatterlist *sg, unsigned int sg_offset,
0208                 unsigned int len)
0209 {
0210     WARN_ON(!wa->address);
0211
0212     scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
0213                  1);
0214 }
0215
0216 static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa,
0217                    unsigned int wa_offset,
0218                    struct scatterlist *sg,
0219                    unsigned int sg_offset,
0220                    unsigned int len)
0221 {
0222     u8 *p, *q;
0223     int rc;
0224
0225     rc = ccp_set_dm_area(wa, wa_offset, sg, sg_offset, len);
0226     if (rc)
0227         return rc;
0228
0229     p = wa->address + wa_offset;
0230     q = p + len - 1;
0231     while (p < q) {
0232         *p = *p ^ *q;
0233         *q = *p ^ *q;
0234         *p = *p ^ *q;
0235         p++;
0236         q--;
0237     }
0238     return 0;
0239 }
0240
0241 static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa,
0242                     unsigned int wa_offset,
0243                     struct scatterlist *sg,
0244                     unsigned int sg_offset,
0245                     unsigned int len)
0246 {
0247     u8 *p, *q;
0248
0249     p = wa->address + wa_offset;
0250     q = p + len - 1;
0251     while (p < q) {
0252         *p = *p ^ *q;
0253         *q = *p ^ *q;
0254         *p = *p ^ *q;
0255         p++;
0256         q--;
0257     }
0258
0259     ccp_get_dm_area(wa, wa_offset, sg, sg_offset, len);
0260 }
0261
0262 static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q)
0263 {
0264     ccp_dm_free(&data->dm_wa);
0265     ccp_sg_free(&data->sg_wa);
0266 }
0267
0268 static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q,
0269              struct scatterlist *sg, u64 sg_len,
0270              unsigned int dm_len,
0271              enum dma_data_direction dir)
0272 {
0273     int ret;
0274
0275     memset(data, 0, sizeof(*data));
0276
0277     ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len,
0278                    dir);
0279     if (ret)
0280         goto e_err;
0281
0282     ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir);
0283     if (ret)
0284         goto e_err;
0285
0286     return 0;
0287
0288 e_err:
0289     ccp_free_data(data, cmd_q);
0290
0291     return ret;
0292 }
0293
0294 static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from)
0295 {
0296     struct ccp_sg_workarea *sg_wa = &data->sg_wa;
0297     struct ccp_dm_workarea *dm_wa = &data->dm_wa;
0298     unsigned int buf_count, nbytes;
0299
0300     /* Clear the buffer if setting it */
0301     if (!from)
0302         memset(dm_wa->address, 0, dm_wa->length);
0303
0304     if (!sg_wa->sg)
0305         return 0;
0306
0307     /* Perform the copy operation
0308      *   nbytes will always be <= UINT_MAX because dm_wa->length is
0309      *   an unsigned int
0310      */
0311     nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length);
0312     scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used,
0313                  nbytes, from);
0314
0315     /* Update the structures and generate the count */
0316     buf_count = 0;
0317     while (sg_wa->bytes_left && (buf_count < dm_wa->length)) {
0318         nbytes = min(sg_dma_len(sg_wa->dma_sg) - sg_wa->sg_used,
0319                  dm_wa->length - buf_count);
0320         nbytes = min_t(u64, sg_wa->bytes_left, nbytes);
0321
0322         buf_count += nbytes;
0323         ccp_update_sg_workarea(sg_wa, nbytes);
0324     }
0325
0326     return buf_count;
0327 }
0328
0329 static unsigned int ccp_fill_queue_buf(struct ccp_data *data)
0330 {
0331     return ccp_queue_buf(data, 0);
0332 }
0333
0334 static unsigned int ccp_empty_queue_buf(struct ccp_data *data)
0335 {
0336     return ccp_queue_buf(data, 1);
0337 }
0338
0339 static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst,
0340                  struct ccp_op *op, unsigned int block_size,
0341                  bool blocksize_op)
0342 {
0343     unsigned int sg_src_len, sg_dst_len, op_len;
0344
0345     /* The CCP can only DMA from/to one address each per operation. This
0346      * requires that we find the smallest DMA area between the source
0347      * and destination. The resulting len values will always be <= UINT_MAX
0348      * because the dma length is an unsigned int.
0349      */
0350     sg_src_len = sg_dma_len(src->sg_wa.dma_sg) - src->sg_wa.sg_used;
0351     sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len);
0352
0353     if (dst) {
0354         sg_dst_len = sg_dma_len(dst->sg_wa.dma_sg) - dst->sg_wa.sg_used;
0355         sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len);
0356         op_len = min(sg_src_len, sg_dst_len);
0357     } else {
0358         op_len = sg_src_len;
0359     }
0360
0361     /* The data operation length will be at least block_size in length
0362      * or the smaller of available sg room remaining for the source or
0363      * the destination
0364      */
0365     op_len = max(op_len, block_size);
0366
0367     /* Unless we have to buffer data, there's no reason to wait */
0368     op->soc = 0;
0369
0370     if (sg_src_len < block_size) {
0371         /* Not enough data in the sg element, so it
0372          * needs to be buffered into a blocksize chunk
0373          */
0374         int cp_len = ccp_fill_queue_buf(src);
0375
0376         op->soc = 1;
0377         op->src.u.dma.address = src->dm_wa.dma.address;
0378         op->src.u.dma.offset = 0;
0379         op->src.u.dma.length = (blocksize_op) ? block_size : cp_len;
0380     } else {
0381         /* Enough data in the sg element, but we need to
0382          * adjust for any previously copied data
0383          */
0384         op->src.u.dma.address = sg_dma_address(src->sg_wa.dma_sg);
0385         op->src.u.dma.offset = src->sg_wa.sg_used;
0386         op->src.u.dma.length = op_len & ~(block_size - 1);
0387
0388         ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length);
0389     }
0390
0391     if (dst) {
0392         if (sg_dst_len < block_size) {
0393             /* Not enough room in the sg element or we're on the
0394              * last piece of data (when using padding), so the
0395              * output needs to be buffered into a blocksize chunk
0396              */
0397             op->soc = 1;
0398             op->dst.u.dma.address = dst->dm_wa.dma.address;
0399             op->dst.u.dma.offset = 0;
0400             op->dst.u.dma.length = op->src.u.dma.length;
0401         } else {
0402             /* Enough room in the sg element, but we need to
0403              * adjust for any previously used area
0404              */
0405             op->dst.u.dma.address = sg_dma_address(dst->sg_wa.dma_sg);
0406             op->dst.u.dma.offset = dst->sg_wa.sg_used;
0407             op->dst.u.dma.length = op->src.u.dma.length;
0408         }
0409     }
0410 }
0411
0412 static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst,
0413                  struct ccp_op *op)
0414 {
0415     op->init = 0;
0416
0417     if (dst) {
0418         if (op->dst.u.dma.address == dst->dm_wa.dma.address)
0419             ccp_empty_queue_buf(dst);
0420         else
0421             ccp_update_sg_workarea(&dst->sg_wa,
0422                            op->dst.u.dma.length);
0423     }
0424 }
0425
0426 static int ccp_copy_to_from_sb(struct ccp_cmd_queue *cmd_q,
0427                    struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
0428                    u32 byte_swap, bool from)
0429 {
0430     struct ccp_op op;
0431
0432     memset(&op, 0, sizeof(op));
0433
0434     op.cmd_q = cmd_q;
0435     op.jobid = jobid;
0436     op.eom = 1;
0437
0438     if (from) {
0439         op.soc = 1;
0440         op.src.type = CCP_MEMTYPE_SB;
0441         op.src.u.sb = sb;
0442         op.dst.type = CCP_MEMTYPE_SYSTEM;
0443         op.dst.u.dma.address = wa->dma.address;
0444         op.dst.u.dma.length = wa->length;
0445     } else {
0446         op.src.type = CCP_MEMTYPE_SYSTEM;
0447         op.src.u.dma.address = wa->dma.address;
0448         op.src.u.dma.length = wa->length;
0449         op.dst.type = CCP_MEMTYPE_SB;
0450         op.dst.u.sb = sb;
0451     }
0452
0453     op.u.passthru.byte_swap = byte_swap;
0454
0455     return cmd_q->ccp->vdata->perform->passthru(&op);
0456 }
0457
0458 static int ccp_copy_to_sb(struct ccp_cmd_queue *cmd_q,
0459               struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
0460               u32 byte_swap)
0461 {
0462     return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, false);
0463 }
0464
0465 static int ccp_copy_from_sb(struct ccp_cmd_queue *cmd_q,
0466                 struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
0467                 u32 byte_swap)
0468 {
0469     return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, true);
0470 }
0471
0472 static noinline_for_stack int
0473 ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
0474 {
0475     struct ccp_aes_engine *aes = &cmd->u.aes;
0476     struct ccp_dm_workarea key, ctx;
0477     struct ccp_data src;
0478     struct ccp_op op;
0479     unsigned int dm_offset;
0480     int ret;
0481
0482     if (!((aes->key_len == AES_KEYSIZE_128) ||
0483           (aes->key_len == AES_KEYSIZE_192) ||
0484           (aes->key_len == AES_KEYSIZE_256)))
0485         return -EINVAL;
0486
0487     if (aes->src_len & (AES_BLOCK_SIZE - 1))
0488         return -EINVAL;
0489
0490     if (aes->iv_len != AES_BLOCK_SIZE)
0491         return -EINVAL;
0492
0493     if (!aes->key || !aes->iv || !aes->src)
0494         return -EINVAL;
0495
0496     if (aes->cmac_final) {
0497         if (aes->cmac_key_len != AES_BLOCK_SIZE)
0498             return -EINVAL;
0499
0500         if (!aes->cmac_key)
0501             return -EINVAL;
0502     }
0503
0504     BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
0505     BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
0506
0507     ret = -EIO;
0508     memset(&op, 0, sizeof(op));
0509     op.cmd_q = cmd_q;
0510     op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
0511     op.sb_key = cmd_q->sb_key;
0512     op.sb_ctx = cmd_q->sb_ctx;
0513     op.init = 1;
0514     op.u.aes.type = aes->type;
0515     op.u.aes.mode = aes->mode;
0516     op.u.aes.action = aes->action;
0517
0518     /* All supported key sizes fit in a single (32-byte) SB entry
0519      * and must be in little endian format. Use the 256-bit byte
0520      * swap passthru option to convert from big endian to little
0521      * endian.
0522      */
0523     ret = ccp_init_dm_workarea(&key, cmd_q,
0524                    CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
0525                    DMA_TO_DEVICE);
0526     if (ret)
0527         return ret;
0528
0529     dm_offset = CCP_SB_BYTES - aes->key_len;
0530     ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
0531     if (ret)
0532         goto e_key;
0533     ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
0534                  CCP_PASSTHRU_BYTESWAP_256BIT);
0535     if (ret) {
0536         cmd->engine_error = cmd_q->cmd_error;
0537         goto e_key;
0538     }
0539
0540     /* The AES context fits in a single (32-byte) SB entry and
0541      * must be in little endian format. Use the 256-bit byte swap
0542      * passthru option to convert from big endian to little endian.
0543      */
0544     ret = ccp_init_dm_workarea(&ctx, cmd_q,
0545                    CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
0546                    DMA_BIDIRECTIONAL);
0547     if (ret)
0548         goto e_key;
0549
0550     dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
0551     ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
0552     if (ret)
0553         goto e_ctx;
0554     ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
0555                  CCP_PASSTHRU_BYTESWAP_256BIT);
0556     if (ret) {
0557         cmd->engine_error = cmd_q->cmd_error;
0558         goto e_ctx;
0559     }
0560
0561     /* Send data to the CCP AES engine */
0562     ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
0563                 AES_BLOCK_SIZE, DMA_TO_DEVICE);
0564     if (ret)
0565         goto e_ctx;
0566
0567     while (src.sg_wa.bytes_left) {
0568         ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true);
0569         if (aes->cmac_final && !src.sg_wa.bytes_left) {
0570             op.eom = 1;
0571
0572             /* Push the K1/K2 key to the CCP now */
0573             ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid,
0574                            op.sb_ctx,
0575                            CCP_PASSTHRU_BYTESWAP_256BIT);
0576             if (ret) {
0577                 cmd->engine_error = cmd_q->cmd_error;
0578                 goto e_src;
0579             }
0580
0581             ret = ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0,
0582                           aes->cmac_key_len);
0583             if (ret)
0584                 goto e_src;
0585             ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
0586                          CCP_PASSTHRU_BYTESWAP_256BIT);
0587             if (ret) {
0588                 cmd->engine_error = cmd_q->cmd_error;
0589                 goto e_src;
0590             }
0591         }
0592
0593         ret = cmd_q->ccp->vdata->perform->aes(&op);
0594         if (ret) {
0595             cmd->engine_error = cmd_q->cmd_error;
0596             goto e_src;
0597         }
0598
0599         ccp_process_data(&src, NULL, &op);
0600     }
0601
0602     /* Retrieve the AES context - convert from LE to BE using
0603      * 32-byte (256-bit) byteswapping
0604      */
0605     ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
0606                    CCP_PASSTHRU_BYTESWAP_256BIT);
0607     if (ret) {
0608         cmd->engine_error = cmd_q->cmd_error;
0609         goto e_src;
0610     }
0611
0612     /* ...but we only need AES_BLOCK_SIZE bytes */
0613     dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
0614     ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
0615
0616 e_src:
0617     ccp_free_data(&src, cmd_q);
0618
0619 e_ctx:
0620     ccp_dm_free(&ctx);
0621
0622 e_key:
0623     ccp_dm_free(&key);
0624
0625     return ret;
0626 }
0627
0628 static noinline_for_stack int
0629 ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
0630 {
0631     struct ccp_aes_engine *aes = &cmd->u.aes;
0632     struct ccp_dm_workarea key, ctx, final_wa, tag;
0633     struct ccp_data src, dst;
0634     struct ccp_data aad;
0635     struct ccp_op op;
0636     unsigned int dm_offset;
0637     unsigned int authsize;
0638     unsigned int jobid;
0639     unsigned int ilen;
0640     bool in_place = true; /* Default value */
0641     __be64 *final;
0642     int ret;
0643
0644     struct scatterlist *p_inp, sg_inp[2];
0645     struct scatterlist *p_tag, sg_tag[2];
0646     struct scatterlist *p_outp, sg_outp[2];
0647     struct scatterlist *p_aad;
0648
0649     if (!aes->iv)
0650         return -EINVAL;
0651
0652     if (!((aes->key_len == AES_KEYSIZE_128) ||
0653         (aes->key_len == AES_KEYSIZE_192) ||
0654         (aes->key_len == AES_KEYSIZE_256)))
0655         return -EINVAL;
0656
0657     if (!aes->key) /* Gotta have a key SGL */
0658         return -EINVAL;
0659
0660     /* Zero defaults to 16 bytes, the maximum size */
0661     authsize = aes->authsize ? aes->authsize : AES_BLOCK_SIZE;
0662     switch (authsize) {
0663     case 16:
0664     case 15:
0665     case 14:
0666     case 13:
0667     case 12:
0668     case 8:
0669     case 4:
0670         break;
0671     default:
0672         return -EINVAL;
0673     }
0674
0675     /* First, decompose the source buffer into AAD & PT,
0676      * and the destination buffer into AAD, CT & tag, or
0677      * the input into CT & tag.
0678      * It is expected that the input and output SGs will
0679      * be valid, even if the AAD and input lengths are 0.
0680      */
0681     p_aad = aes->src;
0682     p_inp = scatterwalk_ffwd(sg_inp, aes->src, aes->aad_len);
0683     p_outp = scatterwalk_ffwd(sg_outp, aes->dst, aes->aad_len);
0684     if (aes->action == CCP_AES_ACTION_ENCRYPT) {
0685         ilen = aes->src_len;
0686         p_tag = scatterwalk_ffwd(sg_tag, p_outp, ilen);
0687     } else {
0688         /* Input length for decryption includes tag */
0689         ilen = aes->src_len - authsize;
0690         p_tag = scatterwalk_ffwd(sg_tag, p_inp, ilen);
0691     }
0692
0693     jobid = CCP_NEW_JOBID(cmd_q->ccp);
0694
0695     memset(&op, 0, sizeof(op));
0696     op.cmd_q = cmd_q;
0697     op.jobid = jobid;
0698     op.sb_key = cmd_q->sb_key; /* Pre-allocated */
0699     op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
0700     op.init = 1;
0701     op.u.aes.type = aes->type;
0702
0703     /* Copy the key to the LSB */
0704     ret = ccp_init_dm_workarea(&key, cmd_q,
0705                    CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
0706                    DMA_TO_DEVICE);
0707     if (ret)
0708         return ret;
0709
0710     dm_offset = CCP_SB_BYTES - aes->key_len;
0711     ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
0712     if (ret)
0713         goto e_key;
0714     ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
0715                  CCP_PASSTHRU_BYTESWAP_256BIT);
0716     if (ret) {
0717         cmd->engine_error = cmd_q->cmd_error;
0718         goto e_key;
0719     }
0720
0721     /* Copy the context (IV) to the LSB.
0722      * There is an assumption here that the IV is 96 bits in length, plus
0723      * a nonce of 32 bits. If no IV is present, use a zeroed buffer.
0724      */
0725     ret = ccp_init_dm_workarea(&ctx, cmd_q,
0726                    CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
0727                    DMA_BIDIRECTIONAL);
0728     if (ret)
0729         goto e_key;
0730
0731     dm_offset = CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES - aes->iv_len;
0732     ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
0733     if (ret)
0734         goto e_ctx;
0735
0736     ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
0737                  CCP_PASSTHRU_BYTESWAP_256BIT);
0738     if (ret) {
0739         cmd->engine_error = cmd_q->cmd_error;
0740         goto e_ctx;
0741     }
0742
0743     op.init = 1;
0744     if (aes->aad_len > 0) {
0745         /* Step 1: Run a GHASH over the Additional Authenticated Data */
0746         ret = ccp_init_data(&aad, cmd_q, p_aad, aes->aad_len,
0747                     AES_BLOCK_SIZE,
0748                     DMA_TO_DEVICE);
0749         if (ret)
0750             goto e_ctx;
0751
0752         op.u.aes.mode = CCP_AES_MODE_GHASH;
0753         op.u.aes.action = CCP_AES_GHASHAAD;
0754
0755         while (aad.sg_wa.bytes_left) {
0756             ccp_prepare_data(&aad, NULL, &op, AES_BLOCK_SIZE, true);
0757
0758             ret = cmd_q->ccp->vdata->perform->aes(&op);
0759             if (ret) {
0760                 cmd->engine_error = cmd_q->cmd_error;
0761                 goto e_aad;
0762             }
0763
0764             ccp_process_data(&aad, NULL, &op);
0765             op.init = 0;
0766         }
0767     }
0768
0769     op.u.aes.mode = CCP_AES_MODE_GCTR;
0770     op.u.aes.action = aes->action;
0771
0772     if (ilen > 0) {
0773         /* Step 2: Run a GCTR over the plaintext */
0774         in_place = (sg_virt(p_inp) == sg_virt(p_outp)) ? true : false;
0775
0776         ret = ccp_init_data(&src, cmd_q, p_inp, ilen,
0777                     AES_BLOCK_SIZE,
0778                     in_place ? DMA_BIDIRECTIONAL
0779                          : DMA_TO_DEVICE);
0780         if (ret)
0781             goto e_aad;
0782
0783         if (in_place) {
0784             dst = src;
0785         } else {
0786             ret = ccp_init_data(&dst, cmd_q, p_outp, ilen,
0787                         AES_BLOCK_SIZE, DMA_FROM_DEVICE);
0788             if (ret)
0789                 goto e_src;
0790         }
0791
0792         op.soc = 0;
0793         op.eom = 0;
0794         op.init = 1;
0795         while (src.sg_wa.bytes_left) {
0796             ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
0797             if (!src.sg_wa.bytes_left) {
0798                 unsigned int nbytes = ilen % AES_BLOCK_SIZE;
0799
0800                 if (nbytes) {
0801                     op.eom = 1;
0802                     op.u.aes.size = (nbytes * 8) - 1;
0803                 }
0804             }
0805
0806             ret = cmd_q->ccp->vdata->perform->aes(&op);
0807             if (ret) {
0808                 cmd->engine_error = cmd_q->cmd_error;
0809                 goto e_dst;
0810             }
0811
0812             ccp_process_data(&src, &dst, &op);
0813             op.init = 0;
0814         }
0815     }
0816
0817     /* Step 3: Update the IV portion of the context with the original IV */
0818     ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
0819                    CCP_PASSTHRU_BYTESWAP_256BIT);
0820     if (ret) {
0821         cmd->engine_error = cmd_q->cmd_error;
0822         goto e_dst;
0823     }
0824
0825     ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
0826     if (ret)
0827         goto e_dst;
0828
0829     ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
0830                  CCP_PASSTHRU_BYTESWAP_256BIT);
0831     if (ret) {
0832         cmd->engine_error = cmd_q->cmd_error;
0833         goto e_dst;
0834     }
0835
0836     /* Step 4: Concatenate the lengths of the AAD and source, and
0837      * hash that 16 byte buffer.
0838      */
0839     ret = ccp_init_dm_workarea(&final_wa, cmd_q, AES_BLOCK_SIZE,
0840                    DMA_BIDIRECTIONAL);
0841     if (ret)
0842         goto e_dst;
0843     final = (__be64 *)final_wa.address;
0844     final[0] = cpu_to_be64(aes->aad_len * 8);
0845     final[1] = cpu_to_be64(ilen * 8);
0846
0847     memset(&op, 0, sizeof(op));
0848     op.cmd_q = cmd_q;
0849     op.jobid = jobid;
0850     op.sb_key = cmd_q->sb_key; /* Pre-allocated */
0851     op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
0852     op.init = 1;
0853     op.u.aes.type = aes->type;
0854     op.u.aes.mode = CCP_AES_MODE_GHASH;
0855     op.u.aes.action = CCP_AES_GHASHFINAL;
0856     op.src.type = CCP_MEMTYPE_SYSTEM;
0857     op.src.u.dma.address = final_wa.dma.address;
0858     op.src.u.dma.length = AES_BLOCK_SIZE;
0859     op.dst.type = CCP_MEMTYPE_SYSTEM;
0860     op.dst.u.dma.address = final_wa.dma.address;
0861     op.dst.u.dma.length = AES_BLOCK_SIZE;
0862     op.eom = 1;
0863     op.u.aes.size = 0;
0864     ret = cmd_q->ccp->vdata->perform->aes(&op);
0865     if (ret)
0866         goto e_final_wa;
0867
0868     if (aes->action == CCP_AES_ACTION_ENCRYPT) {
0869         /* Put the ciphered tag after the ciphertext. */
0870         ccp_get_dm_area(&final_wa, 0, p_tag, 0, authsize);
0871     } else {
0872         /* Does this ciphered tag match the input? */
0873         ret = ccp_init_dm_workarea(&tag, cmd_q, authsize,
0874                        DMA_BIDIRECTIONAL);
0875         if (ret)
0876             goto e_final_wa;
0877         ret = ccp_set_dm_area(&tag, 0, p_tag, 0, authsize);
0878         if (ret) {
0879             ccp_dm_free(&tag);
0880             goto e_final_wa;
0881         }
0882
0883         ret = crypto_memneq(tag.address, final_wa.address,
0884                     authsize) ? -EBADMSG : 0;
0885         ccp_dm_free(&tag);
0886     }
0887
0888 e_final_wa:
0889     ccp_dm_free(&final_wa);
0890
0891 e_dst:
0892     if (ilen > 0 && !in_place)
0893         ccp_free_data(&dst, cmd_q);
0894
0895 e_src:
0896     if (ilen > 0)
0897         ccp_free_data(&src, cmd_q);
0898
0899 e_aad:
0900     if (aes->aad_len)
0901         ccp_free_data(&aad, cmd_q);
0902
0903 e_ctx:
0904     ccp_dm_free(&ctx);
0905
0906 e_key:
0907     ccp_dm_free(&key);
0908
0909     return ret;
0910 }
0911
0912 static noinline_for_stack int
0913 ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
0914 {
0915     struct ccp_aes_engine *aes = &cmd->u.aes;
0916     struct ccp_dm_workarea key, ctx;
0917     struct ccp_data src, dst;
0918     struct ccp_op op;
0919     unsigned int dm_offset;
0920     bool in_place = false;
0921     int ret;
0922
0923     if (!((aes->key_len == AES_KEYSIZE_128) ||
0924           (aes->key_len == AES_KEYSIZE_192) ||
0925           (aes->key_len == AES_KEYSIZE_256)))
0926         return -EINVAL;
0927
0928     if (((aes->mode == CCP_AES_MODE_ECB) ||
0929          (aes->mode == CCP_AES_MODE_CBC)) &&
0930         (aes->src_len & (AES_BLOCK_SIZE - 1)))
0931         return -EINVAL;
0932
0933     if (!aes->key || !aes->src || !aes->dst)
0934         return -EINVAL;
0935
0936     if (aes->mode != CCP_AES_MODE_ECB) {
0937         if (aes->iv_len != AES_BLOCK_SIZE)
0938             return -EINVAL;
0939
0940         if (!aes->iv)
0941             return -EINVAL;
0942     }
0943
0944     BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
0945     BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
0946
0947     ret = -EIO;
0948     memset(&op, 0, sizeof(op));
0949     op.cmd_q = cmd_q;
0950     op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
0951     op.sb_key = cmd_q->sb_key;
0952     op.sb_ctx = cmd_q->sb_ctx;
0953     op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1;
0954     op.u.aes.type = aes->type;
0955     op.u.aes.mode = aes->mode;
0956     op.u.aes.action = aes->action;
0957
0958     /* All supported key sizes fit in a single (32-byte) SB entry
0959      * and must be in little endian format. Use the 256-bit byte
0960      * swap passthru option to convert from big endian to little
0961      * endian.
0962      */
0963     ret = ccp_init_dm_workarea(&key, cmd_q,
0964                    CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
0965                    DMA_TO_DEVICE);
0966     if (ret)
0967         return ret;
0968
0969     dm_offset = CCP_SB_BYTES - aes->key_len;
0970     ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
0971     if (ret)
0972         goto e_key;
0973     ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
0974                  CCP_PASSTHRU_BYTESWAP_256BIT);
0975     if (ret) {
0976         cmd->engine_error = cmd_q->cmd_error;
0977         goto e_key;
0978     }
0979
0980     /* The AES context fits in a single (32-byte) SB entry and
0981      * must be in little endian format. Use the 256-bit byte swap
0982      * passthru option to convert from big endian to little endian.
0983      */
0984     ret = ccp_init_dm_workarea(&ctx, cmd_q,
0985                    CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
0986                    DMA_BIDIRECTIONAL);
0987     if (ret)
0988         goto e_key;
0989
0990     if (aes->mode != CCP_AES_MODE_ECB) {
0991         /* Load the AES context - convert to LE */
0992         dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
0993         ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
0994         if (ret)
0995             goto e_ctx;
0996         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
0997                      CCP_PASSTHRU_BYTESWAP_256BIT);
0998         if (ret) {
0999             cmd->engine_error = cmd_q->cmd_error;
1000             goto e_ctx;
1001         }
1002     }
1003     switch (aes->mode) {
1004     case CCP_AES_MODE_CFB: /* CFB128 only */
1005     case CCP_AES_MODE_CTR:
1006         op.u.aes.size = AES_BLOCK_SIZE * BITS_PER_BYTE - 1;
1007         break;
1008     default:
1009         op.u.aes.size = 0;
1010     }
1011
1012     /* Prepare the input and output data workareas. For in-place
1013      * operations we need to set the dma direction to BIDIRECTIONAL
1014      * and copy the src workarea to the dst workarea.
1015      */
1016     if (sg_virt(aes->src) == sg_virt(aes->dst))
1017         in_place = true;
1018
1019     ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
1020                 AES_BLOCK_SIZE,
1021                 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1022     if (ret)
1023         goto e_ctx;
1024
1025     if (in_place) {
1026         dst = src;
1027     } else {
1028         ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len,
1029                     AES_BLOCK_SIZE, DMA_FROM_DEVICE);
1030         if (ret)
1031             goto e_src;
1032     }
1033
1034     /* Send data to the CCP AES engine */
1035     while (src.sg_wa.bytes_left) {
1036         ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
1037         if (!src.sg_wa.bytes_left) {
1038             op.eom = 1;
1039
1040             /* Since we don't retrieve the AES context in ECB
1041              * mode we have to wait for the operation to complete
1042              * on the last piece of data
1043              */
1044             if (aes->mode == CCP_AES_MODE_ECB)
1045                 op.soc = 1;
1046         }
1047
1048         ret = cmd_q->ccp->vdata->perform->aes(&op);
1049         if (ret) {
1050             cmd->engine_error = cmd_q->cmd_error;
1051             goto e_dst;
1052         }
1053
1054         ccp_process_data(&src, &dst, &op);
1055     }
1056
1057     if (aes->mode != CCP_AES_MODE_ECB) {
1058         /* Retrieve the AES context - convert from LE to BE using
1059          * 32-byte (256-bit) byteswapping
1060          */
1061         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1062                        CCP_PASSTHRU_BYTESWAP_256BIT);
1063         if (ret) {
1064             cmd->engine_error = cmd_q->cmd_error;
1065             goto e_dst;
1066         }
1067
1068         /* ...but we only need AES_BLOCK_SIZE bytes */
1069         dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
1070         ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
1071     }
1072
1073 e_dst:
1074     if (!in_place)
1075         ccp_free_data(&dst, cmd_q);
1076
1077 e_src:
1078     ccp_free_data(&src, cmd_q);
1079
1080 e_ctx:
1081     ccp_dm_free(&ctx);
1082
1083 e_key:
1084     ccp_dm_free(&key);
1085
1086     return ret;
1087 }
1088
1089 static noinline_for_stack int
1090 ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1091 {
1092     struct ccp_xts_aes_engine *xts = &cmd->u.xts;
1093     struct ccp_dm_workarea key, ctx;
1094     struct ccp_data src, dst;
1095     struct ccp_op op;
1096     unsigned int unit_size, dm_offset;
1097     bool in_place = false;
1098     unsigned int sb_count;
1099     enum ccp_aes_type aestype;
1100     int ret;
1101
1102     switch (xts->unit_size) {
1103     case CCP_XTS_AES_UNIT_SIZE_16:
1104         unit_size = 16;
1105         break;
1106     case CCP_XTS_AES_UNIT_SIZE_512:
1107         unit_size = 512;
1108         break;
1109     case CCP_XTS_AES_UNIT_SIZE_1024:
1110         unit_size = 1024;
1111         break;
1112     case CCP_XTS_AES_UNIT_SIZE_2048:
1113         unit_size = 2048;
1114         break;
1115     case CCP_XTS_AES_UNIT_SIZE_4096:
1116         unit_size = 4096;
1117         break;
1118
1119     default:
1120         return -EINVAL;
1121     }
1122
1123     if (xts->key_len == AES_KEYSIZE_128)
1124         aestype = CCP_AES_TYPE_128;
1125     else if (xts->key_len == AES_KEYSIZE_256)
1126         aestype = CCP_AES_TYPE_256;
1127     else
1128         return -EINVAL;
1129
1130     if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1)))
1131         return -EINVAL;
1132
1133     if (xts->iv_len != AES_BLOCK_SIZE)
1134         return -EINVAL;
1135
1136     if (!xts->key || !xts->iv || !xts->src || !xts->dst)
1137         return -EINVAL;
1138
1139     BUILD_BUG_ON(CCP_XTS_AES_KEY_SB_COUNT != 1);
1140     BUILD_BUG_ON(CCP_XTS_AES_CTX_SB_COUNT != 1);
1141
1142     ret = -EIO;
1143     memset(&op, 0, sizeof(op));
1144     op.cmd_q = cmd_q;
1145     op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1146     op.sb_key = cmd_q->sb_key;
1147     op.sb_ctx = cmd_q->sb_ctx;
1148     op.init = 1;
1149     op.u.xts.type = aestype;
1150     op.u.xts.action = xts->action;
1151     op.u.xts.unit_size = xts->unit_size;
1152
1153     /* A version 3 device only supports 128-bit keys, which fits into a
1154      * single SB entry. A version 5 device uses a 512-bit vector, so two
1155      * SB entries.
1156      */
1157     if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0))
1158         sb_count = CCP_XTS_AES_KEY_SB_COUNT;
1159     else
1160         sb_count = CCP5_XTS_AES_KEY_SB_COUNT;
1161     ret = ccp_init_dm_workarea(&key, cmd_q,
1162                    sb_count * CCP_SB_BYTES,
1163                    DMA_TO_DEVICE);
1164     if (ret)
1165         return ret;
1166
1167     if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
1168         /* All supported key sizes must be in little endian format.
1169          * Use the 256-bit byte swap passthru option to convert from
1170          * big endian to little endian.
1171          */
1172         dm_offset = CCP_SB_BYTES - AES_KEYSIZE_128;
1173         ret = ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len);
1174         if (ret)
1175             goto e_key;
1176         ret = ccp_set_dm_area(&key, 0, xts->key, xts->key_len, xts->key_len);
1177         if (ret)
1178             goto e_key;
1179     } else {
1180         /* Version 5 CCPs use a 512-bit space for the key: each portion
1181          * occupies 256 bits, or one entire slot, and is zero-padded.
1182          */
1183         unsigned int pad;
1184
1185         dm_offset = CCP_SB_BYTES;
1186         pad = dm_offset - xts->key_len;
1187         ret = ccp_set_dm_area(&key, pad, xts->key, 0, xts->key_len);
1188         if (ret)
1189             goto e_key;
1190         ret = ccp_set_dm_area(&key, dm_offset + pad, xts->key,
1191                       xts->key_len, xts->key_len);
1192         if (ret)
1193             goto e_key;
1194     }
1195     ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
1196                  CCP_PASSTHRU_BYTESWAP_256BIT);
1197     if (ret) {
1198         cmd->engine_error = cmd_q->cmd_error;
1199         goto e_key;
1200     }
1201
1202     /* The AES context fits in a single (32-byte) SB entry and
1203      * for XTS is already in little endian format so no byte swapping
1204      * is needed.
1205      */
1206     ret = ccp_init_dm_workarea(&ctx, cmd_q,
1207                    CCP_XTS_AES_CTX_SB_COUNT * CCP_SB_BYTES,
1208                    DMA_BIDIRECTIONAL);
1209     if (ret)
1210         goto e_key;
1211
1212     ret = ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len);
1213     if (ret)
1214         goto e_ctx;
1215     ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1216                  CCP_PASSTHRU_BYTESWAP_NOOP);
1217     if (ret) {
1218         cmd->engine_error = cmd_q->cmd_error;
1219         goto e_ctx;
1220     }
1221
1222     /* Prepare the input and output data workareas. For in-place
1223      * operations we need to set the dma direction to BIDIRECTIONAL
1224      * and copy the src workarea to the dst workarea.
1225      */
1226     if (sg_virt(xts->src) == sg_virt(xts->dst))
1227         in_place = true;
1228
1229     ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len,
1230                 unit_size,
1231                 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1232     if (ret)
1233         goto e_ctx;
1234
1235     if (in_place) {
1236         dst = src;
1237     } else {
1238         ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len,
1239                     unit_size, DMA_FROM_DEVICE);
1240         if (ret)
1241             goto e_src;
1242     }
1243
1244     /* Send data to the CCP AES engine */
1245     while (src.sg_wa.bytes_left) {
1246         ccp_prepare_data(&src, &dst, &op, unit_size, true);
1247         if (!src.sg_wa.bytes_left)
1248             op.eom = 1;
1249
1250         ret = cmd_q->ccp->vdata->perform->xts_aes(&op);
1251         if (ret) {
1252             cmd->engine_error = cmd_q->cmd_error;
1253             goto e_dst;
1254         }
1255
1256         ccp_process_data(&src, &dst, &op);
1257     }
1258
1259     /* Retrieve the AES context - convert from LE to BE using
1260      * 32-byte (256-bit) byteswapping
1261      */
1262     ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1263                    CCP_PASSTHRU_BYTESWAP_256BIT);
1264     if (ret) {
1265         cmd->engine_error = cmd_q->cmd_error;
1266         goto e_dst;
1267     }
1268
1269     /* ...but we only need AES_BLOCK_SIZE bytes */
1270     dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
1271     ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len);
1272
1273 e_dst:
1274     if (!in_place)
1275         ccp_free_data(&dst, cmd_q);
1276
1277 e_src:
1278     ccp_free_data(&src, cmd_q);
1279
1280 e_ctx:
1281     ccp_dm_free(&ctx);
1282
1283 e_key:
1284     ccp_dm_free(&key);
1285
1286     return ret;
1287 }
1288
1289 static noinline_for_stack int
1290 ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1291 {
1292     struct ccp_des3_engine *des3 = &cmd->u.des3;
1293
1294     struct ccp_dm_workarea key, ctx;
1295     struct ccp_data src, dst;
1296     struct ccp_op op;
1297     unsigned int dm_offset;
1298     unsigned int len_singlekey;
1299     bool in_place = false;
1300     int ret;
1301
1302     /* Error checks */
1303     if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0))
1304         return -EINVAL;
1305
1306     if (!cmd_q->ccp->vdata->perform->des3)
1307         return -EINVAL;
1308
1309     if (des3->key_len != DES3_EDE_KEY_SIZE)
1310         return -EINVAL;
1311
1312     if (((des3->mode == CCP_DES3_MODE_ECB) ||
1313         (des3->mode == CCP_DES3_MODE_CBC)) &&
1314         (des3->src_len & (DES3_EDE_BLOCK_SIZE - 1)))
1315         return -EINVAL;
1316
1317     if (!des3->key || !des3->src || !des3->dst)
1318         return -EINVAL;
1319
1320     if (des3->mode != CCP_DES3_MODE_ECB) {
1321         if (des3->iv_len != DES3_EDE_BLOCK_SIZE)
1322             return -EINVAL;
1323
1324         if (!des3->iv)
1325             return -EINVAL;
1326     }
1327
1328     /* Zero out all the fields of the command desc */
1329     memset(&op, 0, sizeof(op));
1330
1331     /* Set up the Function field */
1332     op.cmd_q = cmd_q;
1333     op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1334     op.sb_key = cmd_q->sb_key;
1335
1336     op.init = (des3->mode == CCP_DES3_MODE_ECB) ? 0 : 1;
1337     op.u.des3.type = des3->type;
1338     op.u.des3.mode = des3->mode;
1339     op.u.des3.action = des3->action;
1340
1341     /*
1342      * All supported key sizes fit in a single (32-byte) KSB entry and
1343      * (like AES) must be in little endian format. Use the 256-bit byte
1344      * swap passthru option to convert from big endian to little endian.
1345      */
1346     ret = ccp_init_dm_workarea(&key, cmd_q,
1347                    CCP_DES3_KEY_SB_COUNT * CCP_SB_BYTES,
1348                    DMA_TO_DEVICE);
1349     if (ret)
1350         return ret;
1351
1352     /*
1353      * The contents of the key triplet are in the reverse order of what
1354      * is required by the engine. Copy the 3 pieces individually to put
1355      * them where they belong.
1356      */
1357     dm_offset = CCP_SB_BYTES - des3->key_len; /* Basic offset */
1358
1359     len_singlekey = des3->key_len / 3;
1360     ret = ccp_set_dm_area(&key, dm_offset + 2 * len_singlekey,
1361                   des3->key, 0, len_singlekey);
1362     if (ret)
1363         goto e_key;
1364     ret = ccp_set_dm_area(&key, dm_offset + len_singlekey,
1365                   des3->key, len_singlekey, len_singlekey);
1366     if (ret)
1367         goto e_key;
1368     ret = ccp_set_dm_area(&key, dm_offset,
1369                   des3->key, 2 * len_singlekey, len_singlekey);
1370     if (ret)
1371         goto e_key;
1372
1373     /* Copy the key to the SB */
1374     ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
1375                  CCP_PASSTHRU_BYTESWAP_256BIT);
1376     if (ret) {
1377         cmd->engine_error = cmd_q->cmd_error;
1378         goto e_key;
1379     }
1380
1381     /*
1382      * The DES3 context fits in a single (32-byte) KSB entry and
1383      * must be in little endian format. Use the 256-bit byte swap
1384      * passthru option to convert from big endian to little endian.
1385      */
1386     if (des3->mode != CCP_DES3_MODE_ECB) {
1387         op.sb_ctx = cmd_q->sb_ctx;
1388
1389         ret = ccp_init_dm_workarea(&ctx, cmd_q,
1390                        CCP_DES3_CTX_SB_COUNT * CCP_SB_BYTES,
1391                        DMA_BIDIRECTIONAL);
1392         if (ret)
1393             goto e_key;
1394
1395         /* Load the context into the LSB */
1396         dm_offset = CCP_SB_BYTES - des3->iv_len;
1397         ret = ccp_set_dm_area(&ctx, dm_offset, des3->iv, 0,
1398                       des3->iv_len);
1399         if (ret)
1400             goto e_ctx;
1401
1402         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1403                      CCP_PASSTHRU_BYTESWAP_256BIT);
1404         if (ret) {
1405             cmd->engine_error = cmd_q->cmd_error;
1406             goto e_ctx;
1407         }
1408     }
1409
1410     /*
1411      * Prepare the input and output data workareas. For in-place
1412      * operations we need to set the dma direction to BIDIRECTIONAL
1413      * and copy the src workarea to the dst workarea.
1414      */
1415     if (sg_virt(des3->src) == sg_virt(des3->dst))
1416         in_place = true;
1417
1418     ret = ccp_init_data(&src, cmd_q, des3->src, des3->src_len,
1419             DES3_EDE_BLOCK_SIZE,
1420             in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1421     if (ret)
1422         goto e_ctx;
1423
1424     if (in_place)
1425         dst = src;
1426     else {
1427         ret = ccp_init_data(&dst, cmd_q, des3->dst, des3->src_len,
1428                 DES3_EDE_BLOCK_SIZE, DMA_FROM_DEVICE);
1429         if (ret)
1430             goto e_src;
1431     }
1432
1433     /* Send data to the CCP DES3 engine */
1434     while (src.sg_wa.bytes_left) {
1435         ccp_prepare_data(&src, &dst, &op, DES3_EDE_BLOCK_SIZE, true);
1436         if (!src.sg_wa.bytes_left) {
1437             op.eom = 1;
1438
1439             /* Since we don't retrieve the context in ECB mode
1440              * we have to wait for the operation to complete
1441              * on the last piece of data
1442              */
1443             op.soc = 0;
1444         }
1445
1446         ret = cmd_q->ccp->vdata->perform->des3(&op);
1447         if (ret) {
1448             cmd->engine_error = cmd_q->cmd_error;
1449             goto e_dst;
1450         }
1451
1452         ccp_process_data(&src, &dst, &op);
1453     }
1454
1455     if (des3->mode != CCP_DES3_MODE_ECB) {
1456         /* Retrieve the context and make BE */
1457         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1458                        CCP_PASSTHRU_BYTESWAP_256BIT);
1459         if (ret) {
1460             cmd->engine_error = cmd_q->cmd_error;
1461             goto e_dst;
1462         }
1463
1464         /* ...but we only need the last DES3_EDE_BLOCK_SIZE bytes */
1465         ccp_get_dm_area(&ctx, dm_offset, des3->iv, 0,
1466                 DES3_EDE_BLOCK_SIZE);
1467     }
1468 e_dst:
1469     if (!in_place)
1470         ccp_free_data(&dst, cmd_q);
1471
1472 e_src:
1473     ccp_free_data(&src, cmd_q);
1474
1475 e_ctx:
1476     if (des3->mode != CCP_DES3_MODE_ECB)
1477         ccp_dm_free(&ctx);
1478
1479 e_key:
1480     ccp_dm_free(&key);
1481
1482     return ret;
1483 }
1484
1485 static noinline_for_stack int
1486 ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1487 {
1488     struct ccp_sha_engine *sha = &cmd->u.sha;
1489     struct ccp_dm_workarea ctx;
1490     struct ccp_data src;
1491     struct ccp_op op;
1492     unsigned int ioffset, ooffset;
1493     unsigned int digest_size;
1494     int sb_count;
1495     const void *init;
1496     u64 block_size;
1497     int ctx_size;
1498     int ret;
1499
1500     switch (sha->type) {
1501     case CCP_SHA_TYPE_1:
1502         if (sha->ctx_len < SHA1_DIGEST_SIZE)
1503             return -EINVAL;
1504         block_size = SHA1_BLOCK_SIZE;
1505         break;
1506     case CCP_SHA_TYPE_224:
1507         if (sha->ctx_len < SHA224_DIGEST_SIZE)
1508             return -EINVAL;
1509         block_size = SHA224_BLOCK_SIZE;
1510         break;
1511     case CCP_SHA_TYPE_256:
1512         if (sha->ctx_len < SHA256_DIGEST_SIZE)
1513             return -EINVAL;
1514         block_size = SHA256_BLOCK_SIZE;
1515         break;
1516     case CCP_SHA_TYPE_384:
1517         if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
1518             || sha->ctx_len < SHA384_DIGEST_SIZE)
1519             return -EINVAL;
1520         block_size = SHA384_BLOCK_SIZE;
1521         break;
1522     case CCP_SHA_TYPE_512:
1523         if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
1524             || sha->ctx_len < SHA512_DIGEST_SIZE)
1525             return -EINVAL;
1526         block_size = SHA512_BLOCK_SIZE;
1527         break;
1528     default:
1529         return -EINVAL;
1530     }
1531
1532     if (!sha->ctx)
1533         return -EINVAL;
1534
1535     if (!sha->final && (sha->src_len & (block_size - 1)))
1536         return -EINVAL;
1537
1538     /* The version 3 device can't handle zero-length input */
1539     if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
1540
1541         if (!sha->src_len) {
1542             unsigned int digest_len;
1543             const u8 *sha_zero;
1544
1545             /* Not final, just return */
1546             if (!sha->final)
1547                 return 0;
1548
1549             /* CCP can't do a zero length sha operation so the
1550              * caller must buffer the data.
1551              */
1552             if (sha->msg_bits)
1553                 return -EINVAL;
1554
1555             /* The CCP cannot perform zero-length sha operations
1556              * so the caller is required to buffer data for the
1557              * final operation. However, a sha operation for a
1558              * message with a total length of zero is valid so
1559              * known values are required to supply the result.
1560              */
1561             switch (sha->type) {
1562             case CCP_SHA_TYPE_1:
1563                 sha_zero = sha1_zero_message_hash;
1564                 digest_len = SHA1_DIGEST_SIZE;
1565                 break;
1566             case CCP_SHA_TYPE_224:
1567                 sha_zero = sha224_zero_message_hash;
1568                 digest_len = SHA224_DIGEST_SIZE;
1569                 break;
1570             case CCP_SHA_TYPE_256:
1571                 sha_zero = sha256_zero_message_hash;
1572                 digest_len = SHA256_DIGEST_SIZE;
1573                 break;
1574             default:
1575                 return -EINVAL;
1576             }
1577
1578             scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0,
1579                          digest_len, 1);
1580
1581             return 0;
1582         }
1583     }
1584
1585     /* Set variables used throughout */
1586     switch (sha->type) {
1587     case CCP_SHA_TYPE_1:
1588         digest_size = SHA1_DIGEST_SIZE;
1589         init = (void *) ccp_sha1_init;
1590         ctx_size = SHA1_DIGEST_SIZE;
1591         sb_count = 1;
1592         if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1593             ooffset = ioffset = CCP_SB_BYTES - SHA1_DIGEST_SIZE;
1594         else
1595             ooffset = ioffset = 0;
1596         break;
1597     case CCP_SHA_TYPE_224:
1598         digest_size = SHA224_DIGEST_SIZE;
1599         init = (void *) ccp_sha224_init;
1600         ctx_size = SHA256_DIGEST_SIZE;
1601         sb_count = 1;
1602         ioffset = 0;
1603         if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1604             ooffset = CCP_SB_BYTES - SHA224_DIGEST_SIZE;
1605         else
1606             ooffset = 0;
1607         break;
1608     case CCP_SHA_TYPE_256:
1609         digest_size = SHA256_DIGEST_SIZE;
1610         init = (void *) ccp_sha256_init;
1611         ctx_size = SHA256_DIGEST_SIZE;
1612         sb_count = 1;
1613         ooffset = ioffset = 0;
1614         break;
1615     case CCP_SHA_TYPE_384:
1616         digest_size = SHA384_DIGEST_SIZE;
1617         init = (void *) ccp_sha384_init;
1618         ctx_size = SHA512_DIGEST_SIZE;
1619         sb_count = 2;
1620         ioffset = 0;
1621         ooffset = 2 * CCP_SB_BYTES - SHA384_DIGEST_SIZE;
1622         break;
1623     case CCP_SHA_TYPE_512:
1624         digest_size = SHA512_DIGEST_SIZE;
1625         init = (void *) ccp_sha512_init;
1626         ctx_size = SHA512_DIGEST_SIZE;
1627         sb_count = 2;
1628         ooffset = ioffset = 0;
1629         break;
1630     default:
1631         ret = -EINVAL;
1632         goto e_data;
1633     }
1634
1635     /* For zero-length plaintext the src pointer is ignored;
1636      * otherwise both parts must be valid
1637      */
1638     if (sha->src_len && !sha->src)
1639         return -EINVAL;
1640
1641     memset(&op, 0, sizeof(op));
1642     op.cmd_q = cmd_q;
1643     op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1644     op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
1645     op.u.sha.type = sha->type;
1646     op.u.sha.msg_bits = sha->msg_bits;
1647
1648     /* For SHA1/224/256 the context fits in a single (32-byte) SB entry;
1649      * SHA384/512 require 2 adjacent SB slots, with the right half in the
1650      * first slot, and the left half in the second. Each portion must then
1651      * be in little endian format: use the 256-bit byte swap option.
1652      */
1653     ret = ccp_init_dm_workarea(&ctx, cmd_q, sb_count * CCP_SB_BYTES,
1654                    DMA_BIDIRECTIONAL);
1655     if (ret)
1656         return ret;
1657     if (sha->first) {
1658         switch (sha->type) {
1659         case CCP_SHA_TYPE_1:
1660         case CCP_SHA_TYPE_224:
1661         case CCP_SHA_TYPE_256:
1662             memcpy(ctx.address + ioffset, init, ctx_size);
1663             break;
1664         case CCP_SHA_TYPE_384:
1665         case CCP_SHA_TYPE_512:
1666             memcpy(ctx.address + ctx_size / 2, init,
1667                    ctx_size / 2);
1668             memcpy(ctx.address, init + ctx_size / 2,
1669                    ctx_size / 2);
1670             break;
1671         default:
1672             ret = -EINVAL;
1673             goto e_ctx;
1674         }
1675     } else {
1676         /* Restore the context */
1677         ret = ccp_set_dm_area(&ctx, 0, sha->ctx, 0,
1678                       sb_count * CCP_SB_BYTES);
1679         if (ret)
1680             goto e_ctx;
1681     }
1682
1683     ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1684                  CCP_PASSTHRU_BYTESWAP_256BIT);
1685     if (ret) {
1686         cmd->engine_error = cmd_q->cmd_error;
1687         goto e_ctx;
1688     }
1689
1690     if (sha->src) {
1691         /* Send data to the CCP SHA engine; block_size is set above */
1692         ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len,
1693                     block_size, DMA_TO_DEVICE);
1694         if (ret)
1695             goto e_ctx;
1696
1697         while (src.sg_wa.bytes_left) {
1698             ccp_prepare_data(&src, NULL, &op, block_size, false);
1699             if (sha->final && !src.sg_wa.bytes_left)
1700                 op.eom = 1;
1701
1702             ret = cmd_q->ccp->vdata->perform->sha(&op);
1703             if (ret) {
1704                 cmd->engine_error = cmd_q->cmd_error;
1705                 goto e_data;
1706             }
1707
1708             ccp_process_data(&src, NULL, &op);
1709         }
1710     } else {
1711         op.eom = 1;
1712         ret = cmd_q->ccp->vdata->perform->sha(&op);
1713         if (ret) {
1714             cmd->engine_error = cmd_q->cmd_error;
1715             goto e_data;
1716         }
1717     }
1718
1719     /* Retrieve the SHA context - convert from LE to BE using
1720      * 32-byte (256-bit) byteswapping to BE
1721      */
1722     ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1723                    CCP_PASSTHRU_BYTESWAP_256BIT);
1724     if (ret) {
1725         cmd->engine_error = cmd_q->cmd_error;
1726         goto e_data;
1727     }
1728
1729     if (sha->final) {
1730         /* Finishing up, so get the digest */
1731         switch (sha->type) {
1732         case CCP_SHA_TYPE_1:
1733         case CCP_SHA_TYPE_224:
1734         case CCP_SHA_TYPE_256:
1735             ccp_get_dm_area(&ctx, ooffset,
1736                     sha->ctx, 0,
1737                     digest_size);
1738             break;
1739         case CCP_SHA_TYPE_384:
1740         case CCP_SHA_TYPE_512:
1741             ccp_get_dm_area(&ctx, 0,
1742                     sha->ctx, LSB_ITEM_SIZE - ooffset,
1743                     LSB_ITEM_SIZE);
1744             ccp_get_dm_area(&ctx, LSB_ITEM_SIZE + ooffset,
1745                     sha->ctx, 0,
1746                     LSB_ITEM_SIZE - ooffset);
1747             break;
1748         default:
1749             ret = -EINVAL;
1750             goto e_data;
1751         }
1752     } else {
1753         /* Stash the context */
1754         ccp_get_dm_area(&ctx, 0, sha->ctx, 0,
1755                 sb_count * CCP_SB_BYTES);
1756     }
1757
1758     if (sha->final && sha->opad) {
1759         /* HMAC operation, recursively perform final SHA */
1760         struct ccp_cmd hmac_cmd;
1761         struct scatterlist sg;
1762         u8 *hmac_buf;
1763
1764         if (sha->opad_len != block_size) {
1765             ret = -EINVAL;
1766             goto e_data;
1767         }
1768
1769         hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL);
1770         if (!hmac_buf) {
1771             ret = -ENOMEM;
1772             goto e_data;
1773         }
1774         sg_init_one(&sg, hmac_buf, block_size + digest_size);
1775
1776         scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0);
1777         switch (sha->type) {
1778         case CCP_SHA_TYPE_1:
1779         case CCP_SHA_TYPE_224:
1780         case CCP_SHA_TYPE_256:
1781             memcpy(hmac_buf + block_size,
1782                    ctx.address + ooffset,
1783                    digest_size);
1784             break;
1785         case CCP_SHA_TYPE_384:
1786         case CCP_SHA_TYPE_512:
1787             memcpy(hmac_buf + block_size,
1788                    ctx.address + LSB_ITEM_SIZE + ooffset,
1789                    LSB_ITEM_SIZE);
1790             memcpy(hmac_buf + block_size +
1791                    (LSB_ITEM_SIZE - ooffset),
1792                    ctx.address,
1793                    LSB_ITEM_SIZE);
1794             break;
1795         default:
1796             kfree(hmac_buf);
1797             ret = -EINVAL;
1798             goto e_data;
1799         }
1800
1801         memset(&hmac_cmd, 0, sizeof(hmac_cmd));
1802         hmac_cmd.engine = CCP_ENGINE_SHA;
1803         hmac_cmd.u.sha.type = sha->type;
1804         hmac_cmd.u.sha.ctx = sha->ctx;
1805         hmac_cmd.u.sha.ctx_len = sha->ctx_len;
1806         hmac_cmd.u.sha.src = &sg;
1807         hmac_cmd.u.sha.src_len = block_size + digest_size;
1808         hmac_cmd.u.sha.opad = NULL;
1809         hmac_cmd.u.sha.opad_len = 0;
1810         hmac_cmd.u.sha.first = 1;
1811         hmac_cmd.u.sha.final = 1;
1812         hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3;
1813
1814         ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd);
1815         if (ret)
1816             cmd->engine_error = hmac_cmd.engine_error;
1817
1818         kfree(hmac_buf);
1819     }
1820
1821 e_data:
1822     if (sha->src)
1823         ccp_free_data(&src, cmd_q);
1824
1825 e_ctx:
1826     ccp_dm_free(&ctx);
1827
1828     return ret;
1829 }
1830
1831 static noinline_for_stack int
1832 ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1833 {
1834     struct ccp_rsa_engine *rsa = &cmd->u.rsa;
1835     struct ccp_dm_workarea exp, src, dst;
1836     struct ccp_op op;
1837     unsigned int sb_count, i_len, o_len;
1838     int ret;
1839
1840     /* Check against the maximum allowable size, in bits */
1841     if (rsa->key_size > cmd_q->ccp->vdata->rsamax)
1842         return -EINVAL;
1843
1844     if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst)
1845         return -EINVAL;
1846
1847     memset(&op, 0, sizeof(op));
1848     op.cmd_q = cmd_q;
1849     op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1850
1851     /* The RSA modulus must precede the message being acted upon, so
1852      * it must be copied to a DMA area where the message and the
1853      * modulus can be concatenated.  Therefore the input buffer
1854      * length required is twice the output buffer length (which
1855      * must be a multiple of 256-bits).  Compute o_len, i_len in bytes.
1856      * Buffer sizes must be a multiple of 32 bytes; rounding up may be
1857      * required.
1858      */
1859     o_len = 32 * ((rsa->key_size + 255) / 256);
1860     i_len = o_len * 2;
1861
1862     sb_count = 0;
1863     if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
1864         /* sb_count is the number of storage block slots required
1865          * for the modulus.
1866          */
1867         sb_count = o_len / CCP_SB_BYTES;
1868         op.sb_key = cmd_q->ccp->vdata->perform->sballoc(cmd_q,
1869                                 sb_count);
1870         if (!op.sb_key)
1871             return -EIO;
1872     } else {
1873         /* A version 5 device allows a modulus size that will not fit
1874          * in the LSB, so the command will transfer it from memory.
1875          * Set the sb key to the default, even though it's not used.
1876          */
1877         op.sb_key = cmd_q->sb_key;
1878     }
1879
1880     /* The RSA exponent must be in little endian format. Reverse its
1881      * byte order.
1882      */
1883     ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE);
1884     if (ret)
1885         goto e_sb;
1886
1887     ret = ccp_reverse_set_dm_area(&exp, 0, rsa->exp, 0, rsa->exp_len);
1888     if (ret)
1889         goto e_exp;
1890
1891     if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
1892         /* Copy the exponent to the local storage block, using
1893          * as many 32-byte blocks as were allocated above. It's
1894          * already little endian, so no further change is required.
1895          */
1896         ret = ccp_copy_to_sb(cmd_q, &exp, op.jobid, op.sb_key,
1897                      CCP_PASSTHRU_BYTESWAP_NOOP);
1898         if (ret) {
1899             cmd->engine_error = cmd_q->cmd_error;
1900             goto e_exp;
1901         }
1902     } else {
1903         /* The exponent can be retrieved from memory via DMA. */
1904         op.exp.u.dma.address = exp.dma.address;
1905         op.exp.u.dma.offset = 0;
1906     }
1907
1908     /* Concatenate the modulus and the message. Both the modulus and
1909      * the operands must be in little endian format.  Since the input
1910      * is in big endian format it must be converted.
1911      */
1912     ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE);
1913     if (ret)
1914         goto e_exp;
1915
1916     ret = ccp_reverse_set_dm_area(&src, 0, rsa->mod, 0, rsa->mod_len);
1917     if (ret)
1918         goto e_src;
1919     ret = ccp_reverse_set_dm_area(&src, o_len, rsa->src, 0, rsa->src_len);
1920     if (ret)
1921         goto e_src;
1922
1923     /* Prepare the output area for the operation */
1924     ret = ccp_init_dm_workarea(&dst, cmd_q, o_len, DMA_FROM_DEVICE);
1925     if (ret)
1926         goto e_src;
1927
1928     op.soc = 1;
1929     op.src.u.dma.address = src.dma.address;
1930     op.src.u.dma.offset = 0;
1931     op.src.u.dma.length = i_len;
1932     op.dst.u.dma.address = dst.dma.address;
1933     op.dst.u.dma.offset = 0;
1934     op.dst.u.dma.length = o_len;
1935
1936     op.u.rsa.mod_size = rsa->key_size;
1937     op.u.rsa.input_len = i_len;
1938
1939     ret = cmd_q->ccp->vdata->perform->rsa(&op);
1940     if (ret) {
1941         cmd->engine_error = cmd_q->cmd_error;
1942         goto e_dst;
1943     }
1944
1945     ccp_reverse_get_dm_area(&dst, 0, rsa->dst, 0, rsa->mod_len);
1946
1947 e_dst:
1948     ccp_dm_free(&dst);
1949
1950 e_src:
1951     ccp_dm_free(&src);
1952
1953 e_exp:
1954     ccp_dm_free(&exp);
1955
1956 e_sb:
1957     if (sb_count)
1958         cmd_q->ccp->vdata->perform->sbfree(cmd_q, op.sb_key, sb_count);
1959
1960     return ret;
1961 }
1962
1963 static noinline_for_stack int
1964 ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1965 {
1966     struct ccp_passthru_engine *pt = &cmd->u.passthru;
1967     struct ccp_dm_workarea mask;
1968     struct ccp_data src, dst;
1969     struct ccp_op op;
1970     bool in_place = false;
1971     unsigned int i;
1972     int ret = 0;
1973
1974     if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
1975         return -EINVAL;
1976
1977     if (!pt->src || !pt->dst)
1978         return -EINVAL;
1979
1980     if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1981         if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
1982             return -EINVAL;
1983         if (!pt->mask)
1984             return -EINVAL;
1985     }
1986
1987     BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
1988
1989     memset(&op, 0, sizeof(op));
1990     op.cmd_q = cmd_q;
1991     op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1992
1993     if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1994         /* Load the mask */
1995         op.sb_key = cmd_q->sb_key;
1996
1997         ret = ccp_init_dm_workarea(&mask, cmd_q,
1998                        CCP_PASSTHRU_SB_COUNT *
1999                        CCP_SB_BYTES,
2000                        DMA_TO_DEVICE);
2001         if (ret)
2002             return ret;
2003
2004         ret = ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len);
2005         if (ret)
2006             goto e_mask;
2007         ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
2008                      CCP_PASSTHRU_BYTESWAP_NOOP);
2009         if (ret) {
2010             cmd->engine_error = cmd_q->cmd_error;
2011             goto e_mask;
2012         }
2013     }
2014
2015     /* Prepare the input and output data workareas. For in-place
2016      * operations we need to set the dma direction to BIDIRECTIONAL
2017      * and copy the src workarea to the dst workarea.
2018      */
2019     if (sg_virt(pt->src) == sg_virt(pt->dst))
2020         in_place = true;
2021
2022     ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len,
2023                 CCP_PASSTHRU_MASKSIZE,
2024                 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
2025     if (ret)
2026         goto e_mask;
2027
2028     if (in_place) {
2029         dst = src;
2030     } else {
2031         ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len,
2032                     CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE);
2033         if (ret)
2034             goto e_src;
2035     }
2036
2037     /* Send data to the CCP Passthru engine
2038      *   Because the CCP engine works on a single source and destination
2039      *   dma address at a time, each entry in the source scatterlist
2040      *   (after the dma_map_sg call) must be less than or equal to the
2041      *   (remaining) length in the destination scatterlist entry and the
2042      *   length must be a multiple of CCP_PASSTHRU_BLOCKSIZE
2043      */
2044     dst.sg_wa.sg_used = 0;
2045     for (i = 1; i <= src.sg_wa.dma_count; i++) {
2046         if (!dst.sg_wa.sg ||
2047             (sg_dma_len(dst.sg_wa.sg) < sg_dma_len(src.sg_wa.sg))) {
2048             ret = -EINVAL;
2049             goto e_dst;
2050         }
2051
2052         if (i == src.sg_wa.dma_count) {
2053             op.eom = 1;
2054             op.soc = 1;
2055         }
2056
2057         op.src.type = CCP_MEMTYPE_SYSTEM;
2058         op.src.u.dma.address = sg_dma_address(src.sg_wa.sg);
2059         op.src.u.dma.offset = 0;
2060         op.src.u.dma.length = sg_dma_len(src.sg_wa.sg);
2061
2062         op.dst.type = CCP_MEMTYPE_SYSTEM;
2063         op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg);
2064         op.dst.u.dma.offset = dst.sg_wa.sg_used;
2065         op.dst.u.dma.length = op.src.u.dma.length;
2066
2067         ret = cmd_q->ccp->vdata->perform->passthru(&op);
2068         if (ret) {
2069             cmd->engine_error = cmd_q->cmd_error;
2070             goto e_dst;
2071         }
2072
2073         dst.sg_wa.sg_used += sg_dma_len(src.sg_wa.sg);
2074         if (dst.sg_wa.sg_used == sg_dma_len(dst.sg_wa.sg)) {
2075             dst.sg_wa.sg = sg_next(dst.sg_wa.sg);
2076             dst.sg_wa.sg_used = 0;
2077         }
2078         src.sg_wa.sg = sg_next(src.sg_wa.sg);
2079     }
2080
2081 e_dst:
2082     if (!in_place)
2083         ccp_free_data(&dst, cmd_q);
2084
2085 e_src:
2086     ccp_free_data(&src, cmd_q);
2087
2088 e_mask:
2089     if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
2090         ccp_dm_free(&mask);
2091
2092     return ret;
2093 }
2094
2095 static noinline_for_stack int
2096 ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q,
2097                       struct ccp_cmd *cmd)
2098 {
2099     struct ccp_passthru_nomap_engine *pt = &cmd->u.passthru_nomap;
2100     struct ccp_dm_workarea mask;
2101     struct ccp_op op;
2102     int ret;
2103
2104     if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
2105         return -EINVAL;
2106
2107     if (!pt->src_dma || !pt->dst_dma)
2108         return -EINVAL;
2109
2110     if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
2111         if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
2112             return -EINVAL;
2113         if (!pt->mask)
2114             return -EINVAL;
2115     }
2116
2117     BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
2118
2119     memset(&op, 0, sizeof(op));
2120     op.cmd_q = cmd_q;
2121     op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2122
2123     if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
2124         /* Load the mask */
2125         op.sb_key = cmd_q->sb_key;
2126
2127         mask.length = pt->mask_len;
2128         mask.dma.address = pt->mask;
2129         mask.dma.length = pt->mask_len;
2130
2131         ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
2132                      CCP_PASSTHRU_BYTESWAP_NOOP);
2133         if (ret) {
2134             cmd->engine_error = cmd_q->cmd_error;
2135             return ret;
2136         }
2137     }
2138
2139     /* Send data to the CCP Passthru engine */
2140     op.eom = 1;
2141     op.soc = 1;
2142
2143     op.src.type = CCP_MEMTYPE_SYSTEM;
2144     op.src.u.dma.address = pt->src_dma;
2145     op.src.u.dma.offset = 0;
2146     op.src.u.dma.length = pt->src_len;
2147
2148     op.dst.type = CCP_MEMTYPE_SYSTEM;
2149     op.dst.u.dma.address = pt->dst_dma;
2150     op.dst.u.dma.offset = 0;
2151     op.dst.u.dma.length = pt->src_len;
2152
2153     ret = cmd_q->ccp->vdata->perform->passthru(&op);
2154     if (ret)
2155         cmd->engine_error = cmd_q->cmd_error;
2156
2157     return ret;
2158 }
2159
2160 static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2161 {
2162     struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2163     struct ccp_dm_workarea src, dst;
2164     struct ccp_op op;
2165     int ret;
2166     u8 *save;
2167
2168     if (!ecc->u.mm.operand_1 ||
2169         (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES))
2170         return -EINVAL;
2171
2172     if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT)
2173         if (!ecc->u.mm.operand_2 ||
2174             (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES))
2175             return -EINVAL;
2176
2177     if (!ecc->u.mm.result ||
2178         (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES))
2179         return -EINVAL;
2180
2181     memset(&op, 0, sizeof(op));
2182     op.cmd_q = cmd_q;
2183     op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2184
2185     /* Concatenate the modulus and the operands. Both the modulus and
2186      * the operands must be in little endian format.  Since the input
2187      * is in big endian format it must be converted and placed in a
2188      * fixed length buffer.
2189      */
2190     ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
2191                    DMA_TO_DEVICE);
2192     if (ret)
2193         return ret;
2194
2195     /* Save the workarea address since it is updated in order to perform
2196      * the concatenation
2197      */
2198     save = src.address;
2199
2200     /* Copy the ECC modulus */
2201     ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len);
2202     if (ret)
2203         goto e_src;
2204     src.address += CCP_ECC_OPERAND_SIZE;
2205
2206     /* Copy the first operand */
2207     ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_1, 0,
2208                       ecc->u.mm.operand_1_len);
2209     if (ret)
2210         goto e_src;
2211     src.address += CCP_ECC_OPERAND_SIZE;
2212
2213     if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) {
2214         /* Copy the second operand */
2215         ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_2, 0,
2216                           ecc->u.mm.operand_2_len);
2217         if (ret)
2218             goto e_src;
2219         src.address += CCP_ECC_OPERAND_SIZE;
2220     }
2221
2222     /* Restore the workarea address */
2223     src.address = save;
2224
2225     /* Prepare the output area for the operation */
2226     ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
2227                    DMA_FROM_DEVICE);
2228     if (ret)
2229         goto e_src;
2230
2231     op.soc = 1;
2232     op.src.u.dma.address = src.dma.address;
2233     op.src.u.dma.offset = 0;
2234     op.src.u.dma.length = src.length;
2235     op.dst.u.dma.address = dst.dma.address;
2236     op.dst.u.dma.offset = 0;
2237     op.dst.u.dma.length = dst.length;
2238
2239     op.u.ecc.function = cmd->u.ecc.function;
2240
2241     ret = cmd_q->ccp->vdata->perform->ecc(&op);
2242     if (ret) {
2243         cmd->engine_error = cmd_q->cmd_error;
2244         goto e_dst;
2245     }
2246
2247     ecc->ecc_result = le16_to_cpup(
2248         (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
2249     if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
2250         ret = -EIO;
2251         goto e_dst;
2252     }
2253
2254     /* Save the ECC result */
2255     ccp_reverse_get_dm_area(&dst, 0, ecc->u.mm.result, 0,
2256                 CCP_ECC_MODULUS_BYTES);
2257
2258 e_dst:
2259     ccp_dm_free(&dst);
2260
2261 e_src:
2262     ccp_dm_free(&src);
2263
2264     return ret;
2265 }
2266
2267 static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2268 {
2269     struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2270     struct ccp_dm_workarea src, dst;
2271     struct ccp_op op;
2272     int ret;
2273     u8 *save;
2274
2275     if (!ecc->u.pm.point_1.x ||
2276         (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) ||
2277         !ecc->u.pm.point_1.y ||
2278         (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES))
2279         return -EINVAL;
2280
2281     if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
2282         if (!ecc->u.pm.point_2.x ||
2283             (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) ||
2284             !ecc->u.pm.point_2.y ||
2285             (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES))
2286             return -EINVAL;
2287     } else {
2288         if (!ecc->u.pm.domain_a ||
2289             (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES))
2290             return -EINVAL;
2291
2292         if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT)
2293             if (!ecc->u.pm.scalar ||
2294                 (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES))
2295                 return -EINVAL;
2296     }
2297
2298     if (!ecc->u.pm.result.x ||
2299         (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) ||
2300         !ecc->u.pm.result.y ||
2301         (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES))
2302         return -EINVAL;
2303
2304     memset(&op, 0, sizeof(op));
2305     op.cmd_q = cmd_q;
2306     op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2307
2308     /* Concatenate the modulus and the operands. Both the modulus and
2309      * the operands must be in little endian format.  Since the input
2310      * is in big endian format it must be converted and placed in a
2311      * fixed length buffer.
2312      */
2313     ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
2314                    DMA_TO_DEVICE);
2315     if (ret)
2316         return ret;
2317
2318     /* Save the workarea address since it is updated in order to perform
2319      * the concatenation
2320      */
2321     save = src.address;
2322
2323     /* Copy the ECC modulus */
2324     ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len);
2325     if (ret)
2326         goto e_src;
2327     src.address += CCP_ECC_OPERAND_SIZE;
2328
2329     /* Copy the first point X and Y coordinate */
2330     ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.x, 0,
2331                       ecc->u.pm.point_1.x_len);
2332     if (ret)
2333         goto e_src;
2334     src.address += CCP_ECC_OPERAND_SIZE;
2335     ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.y, 0,
2336                       ecc->u.pm.point_1.y_len);
2337     if (ret)
2338         goto e_src;
2339     src.address += CCP_ECC_OPERAND_SIZE;
2340
2341     /* Set the first point Z coordinate to 1 */
2342     *src.address = 0x01;
2343     src.address += CCP_ECC_OPERAND_SIZE;
2344
2345     if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
2346         /* Copy the second point X and Y coordinate */
2347         ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.x, 0,
2348                           ecc->u.pm.point_2.x_len);
2349         if (ret)
2350             goto e_src;
2351         src.address += CCP_ECC_OPERAND_SIZE;
2352         ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.y, 0,
2353                           ecc->u.pm.point_2.y_len);
2354         if (ret)
2355             goto e_src;
2356         src.address += CCP_ECC_OPERAND_SIZE;
2357
2358         /* Set the second point Z coordinate to 1 */
2359         *src.address = 0x01;
2360         src.address += CCP_ECC_OPERAND_SIZE;
2361     } else {
2362         /* Copy the Domain "a" parameter */
2363         ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.domain_a, 0,
2364                           ecc->u.pm.domain_a_len);
2365         if (ret)
2366             goto e_src;
2367         src.address += CCP_ECC_OPERAND_SIZE;
2368
2369         if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) {
2370             /* Copy the scalar value */
2371             ret = ccp_reverse_set_dm_area(&src, 0,
2372                               ecc->u.pm.scalar, 0,
2373                               ecc->u.pm.scalar_len);
2374             if (ret)
2375                 goto e_src;
2376             src.address += CCP_ECC_OPERAND_SIZE;
2377         }
2378     }
2379
2380     /* Restore the workarea address */
2381     src.address = save;
2382
2383     /* Prepare the output area for the operation */
2384     ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
2385                    DMA_FROM_DEVICE);
2386     if (ret)
2387         goto e_src;
2388
2389     op.soc = 1;
2390     op.src.u.dma.address = src.dma.address;
2391     op.src.u.dma.offset = 0;
2392     op.src.u.dma.length = src.length;
2393     op.dst.u.dma.address = dst.dma.address;
2394     op.dst.u.dma.offset = 0;
2395     op.dst.u.dma.length = dst.length;
2396
2397     op.u.ecc.function = cmd->u.ecc.function;
2398
2399     ret = cmd_q->ccp->vdata->perform->ecc(&op);
2400     if (ret) {
2401         cmd->engine_error = cmd_q->cmd_error;
2402         goto e_dst;
2403     }
2404
2405     ecc->ecc_result = le16_to_cpup(
2406         (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
2407     if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
2408         ret = -EIO;
2409         goto e_dst;
2410     }
2411
2412     /* Save the workarea address since it is updated as we walk through
2413      * to copy the point math result
2414      */
2415     save = dst.address;
2416
2417     /* Save the ECC result X and Y coordinates */
2418     ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.x, 0,
2419                 CCP_ECC_MODULUS_BYTES);
2420     dst.address += CCP_ECC_OUTPUT_SIZE;
2421     ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.y, 0,
2422                 CCP_ECC_MODULUS_BYTES);
2423
2424     /* Restore the workarea address */
2425     dst.address = save;
2426
2427 e_dst:
2428     ccp_dm_free(&dst);
2429
2430 e_src:
2431     ccp_dm_free(&src);
2432
2433     return ret;
2434 }
2435
2436 static noinline_for_stack int
2437 ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2438 {
2439     struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2440
2441     ecc->ecc_result = 0;
2442
2443     if (!ecc->mod ||
2444         (ecc->mod_len > CCP_ECC_MODULUS_BYTES))
2445         return -EINVAL;
2446
2447     switch (ecc->function) {
2448     case CCP_ECC_FUNCTION_MMUL_384BIT:
2449     case CCP_ECC_FUNCTION_MADD_384BIT:
2450     case CCP_ECC_FUNCTION_MINV_384BIT:
2451         return ccp_run_ecc_mm_cmd(cmd_q, cmd);
2452
2453     case CCP_ECC_FUNCTION_PADD_384BIT:
2454     case CCP_ECC_FUNCTION_PMUL_384BIT:
2455     case CCP_ECC_FUNCTION_PDBL_384BIT:
2456         return ccp_run_ecc_pm_cmd(cmd_q, cmd);
2457
2458     default:
2459         return -EINVAL;
2460     }
2461 }
2462
2463 int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2464 {
2465     int ret;
2466
2467     cmd->engine_error = 0;
2468     cmd_q->cmd_error = 0;
2469     cmd_q->int_rcvd = 0;
2470     cmd_q->free_slots = cmd_q->ccp->vdata->perform->get_free_slots(cmd_q);
2471
2472     switch (cmd->engine) {
2473     case CCP_ENGINE_AES:
2474         switch (cmd->u.aes.mode) {
2475         case CCP_AES_MODE_CMAC:
2476             ret = ccp_run_aes_cmac_cmd(cmd_q, cmd);
2477             break;
2478         case CCP_AES_MODE_GCM:
2479             ret = ccp_run_aes_gcm_cmd(cmd_q, cmd);
2480             break;
2481         default:
2482             ret = ccp_run_aes_cmd(cmd_q, cmd);
2483             break;
2484         }
2485         break;
2486     case CCP_ENGINE_XTS_AES_128:
2487         ret = ccp_run_xts_aes_cmd(cmd_q, cmd);
2488         break;
2489     case CCP_ENGINE_DES3:
2490         ret = ccp_run_des3_cmd(cmd_q, cmd);
2491         break;
2492     case CCP_ENGINE_SHA:
2493         ret = ccp_run_sha_cmd(cmd_q, cmd);
2494         break;
2495     case CCP_ENGINE_RSA:
2496         ret = ccp_run_rsa_cmd(cmd_q, cmd);
2497         break;
2498     case CCP_ENGINE_PASSTHRU:
2499         if (cmd->flags & CCP_CMD_PASSTHRU_NO_DMA_MAP)
2500             ret = ccp_run_passthru_nomap_cmd(cmd_q, cmd);
2501         else
2502             ret = ccp_run_passthru_cmd(cmd_q, cmd);
2503         break;
2504     case CCP_ENGINE_ECC:
2505         ret = ccp_run_ecc_cmd(cmd_q, cmd);
2506         break;
2507     default:
2508         ret = -EINVAL;
2509     }
2510
2511     return ret;
2512 }