crypto/qce/sha.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Copyright (c) 2010-2014, The Linux Foundation. All rights reserved.
0004  */
0005
0006 #include <linux/device.h>
0007 #include <linux/dma-mapping.h>
0008 #include <linux/interrupt.h>
0009 #include <crypto/internal/hash.h>
0010
0011 #include "common.h"
0012 #include "core.h"
0013 #include "sha.h"
0014
0015 struct qce_sha_saved_state {
0016     u8 pending_buf[QCE_SHA_MAX_BLOCKSIZE];
0017     u8 partial_digest[QCE_SHA_MAX_DIGESTSIZE];
0018     __be32 byte_count[2];
0019     unsigned int pending_buflen;
0020     unsigned int flags;
0021     u64 count;
0022     bool first_blk;
0023 };
0024
0025 static LIST_HEAD(ahash_algs);
0026
0027 static const u32 std_iv_sha1[SHA256_DIGEST_SIZE / sizeof(u32)] = {
0028     SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4, 0, 0, 0
0029 };
0030
0031 static const u32 std_iv_sha256[SHA256_DIGEST_SIZE / sizeof(u32)] = {
0032     SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3,
0033     SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7
0034 };
0035
0036 static void qce_ahash_done(void *data)
0037 {
0038     struct crypto_async_request *async_req = data;
0039     struct ahash_request *req = ahash_request_cast(async_req);
0040     struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
0041     struct qce_sha_reqctx *rctx = ahash_request_ctx(req);
0042     struct qce_alg_template *tmpl = to_ahash_tmpl(async_req->tfm);
0043     struct qce_device *qce = tmpl->qce;
0044     struct qce_result_dump *result = qce->dma.result_buf;
0045     unsigned int digestsize = crypto_ahash_digestsize(ahash);
0046     int error;
0047     u32 status;
0048
0049     error = qce_dma_terminate_all(&qce->dma);
0050     if (error)
0051         dev_dbg(qce->dev, "ahash dma termination error (%d)\n", error);
0052
0053     dma_unmap_sg(qce->dev, req->src, rctx->src_nents, DMA_TO_DEVICE);
0054     dma_unmap_sg(qce->dev, &rctx->result_sg, 1, DMA_FROM_DEVICE);
0055
0056     memcpy(rctx->digest, result->auth_iv, digestsize);
0057     if (req->result && rctx->last_blk)
0058         memcpy(req->result, result->auth_iv, digestsize);
0059
0060     rctx->byte_count[0] = cpu_to_be32(result->auth_byte_count[0]);
0061     rctx->byte_count[1] = cpu_to_be32(result->auth_byte_count[1]);
0062
0063     error = qce_check_status(qce, &status);
0064     if (error < 0)
0065         dev_dbg(qce->dev, "ahash operation error (%x)\n", status);
0066
0067     req->src = rctx->src_orig;
0068     req->nbytes = rctx->nbytes_orig;
0069     rctx->last_blk = false;
0070     rctx->first_blk = false;
0071
0072     qce->async_req_done(tmpl->qce, error);
0073 }
0074
0075 static int qce_ahash_async_req_handle(struct crypto_async_request *async_req)
0076 {
0077     struct ahash_request *req = ahash_request_cast(async_req);
0078     struct qce_sha_reqctx *rctx = ahash_request_ctx(req);
0079     struct qce_sha_ctx *ctx = crypto_tfm_ctx(async_req->tfm);
0080     struct qce_alg_template *tmpl = to_ahash_tmpl(async_req->tfm);
0081     struct qce_device *qce = tmpl->qce;
0082     unsigned long flags = rctx->flags;
0083     int ret;
0084
0085     if (IS_SHA_HMAC(flags)) {
0086         rctx->authkey = ctx->authkey;
0087         rctx->authklen = QCE_SHA_HMAC_KEY_SIZE;
0088     } else if (IS_CMAC(flags)) {
0089         rctx->authkey = ctx->authkey;
0090         rctx->authklen = AES_KEYSIZE_128;
0091     }
0092
0093     rctx->src_nents = sg_nents_for_len(req->src, req->nbytes);
0094     if (rctx->src_nents < 0) {
0095         dev_err(qce->dev, "Invalid numbers of src SG.\n");
0096         return rctx->src_nents;
0097     }
0098
0099     ret = dma_map_sg(qce->dev, req->src, rctx->src_nents, DMA_TO_DEVICE);
0100     if (ret < 0)
0101         return ret;
0102
0103     sg_init_one(&rctx->result_sg, qce->dma.result_buf, QCE_RESULT_BUF_SZ);
0104
0105     ret = dma_map_sg(qce->dev, &rctx->result_sg, 1, DMA_FROM_DEVICE);
0106     if (ret < 0)
0107         goto error_unmap_src;
0108
0109     ret = qce_dma_prep_sgs(&qce->dma, req->src, rctx->src_nents,
0110                    &rctx->result_sg, 1, qce_ahash_done, async_req);
0111     if (ret)
0112         goto error_unmap_dst;
0113
0114     qce_dma_issue_pending(&qce->dma);
0115
0116     ret = qce_start(async_req, tmpl->crypto_alg_type);
0117     if (ret)
0118         goto error_terminate;
0119
0120     return 0;
0121
0122 error_terminate:
0123     qce_dma_terminate_all(&qce->dma);
0124 error_unmap_dst:
0125     dma_unmap_sg(qce->dev, &rctx->result_sg, 1, DMA_FROM_DEVICE);
0126 error_unmap_src:
0127     dma_unmap_sg(qce->dev, req->src, rctx->src_nents, DMA_TO_DEVICE);
0128     return ret;
0129 }
0130
0131 static int qce_ahash_init(struct ahash_request *req)
0132 {
0133     struct qce_sha_reqctx *rctx = ahash_request_ctx(req);
0134     struct qce_alg_template *tmpl = to_ahash_tmpl(req->base.tfm);
0135     const u32 *std_iv = tmpl->std_iv;
0136
0137     memset(rctx, 0, sizeof(*rctx));
0138     rctx->first_blk = true;
0139     rctx->last_blk = false;
0140     rctx->flags = tmpl->alg_flags;
0141     memcpy(rctx->digest, std_iv, sizeof(rctx->digest));
0142
0143     return 0;
0144 }
0145
0146 static int qce_ahash_export(struct ahash_request *req, void *out)
0147 {
0148     struct qce_sha_reqctx *rctx = ahash_request_ctx(req);
0149     struct qce_sha_saved_state *export_state = out;
0150
0151     memcpy(export_state->pending_buf, rctx->buf, rctx->buflen);
0152     memcpy(export_state->partial_digest, rctx->digest, sizeof(rctx->digest));
0153     export_state->byte_count[0] = rctx->byte_count[0];
0154     export_state->byte_count[1] = rctx->byte_count[1];
0155     export_state->pending_buflen = rctx->buflen;
0156     export_state->count = rctx->count;
0157     export_state->first_blk = rctx->first_blk;
0158     export_state->flags = rctx->flags;
0159
0160     return 0;
0161 }
0162
0163 static int qce_ahash_import(struct ahash_request *req, const void *in)
0164 {
0165     struct qce_sha_reqctx *rctx = ahash_request_ctx(req);
0166     const struct qce_sha_saved_state *import_state = in;
0167
0168     memset(rctx, 0, sizeof(*rctx));
0169     rctx->count = import_state->count;
0170     rctx->buflen = import_state->pending_buflen;
0171     rctx->first_blk = import_state->first_blk;
0172     rctx->flags = import_state->flags;
0173     rctx->byte_count[0] = import_state->byte_count[0];
0174     rctx->byte_count[1] = import_state->byte_count[1];
0175     memcpy(rctx->buf, import_state->pending_buf, rctx->buflen);
0176     memcpy(rctx->digest, import_state->partial_digest, sizeof(rctx->digest));
0177
0178     return 0;
0179 }
0180
0181 static int qce_ahash_update(struct ahash_request *req)
0182 {
0183     struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
0184     struct qce_sha_reqctx *rctx = ahash_request_ctx(req);
0185     struct qce_alg_template *tmpl = to_ahash_tmpl(req->base.tfm);
0186     struct qce_device *qce = tmpl->qce;
0187     struct scatterlist *sg_last, *sg;
0188     unsigned int total, len;
0189     unsigned int hash_later;
0190     unsigned int nbytes;
0191     unsigned int blocksize;
0192
0193     blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
0194     rctx->count += req->nbytes;
0195
0196     /* check for buffer from previous updates and append it */
0197     total = req->nbytes + rctx->buflen;
0198
0199     if (total <= blocksize) {
0200         scatterwalk_map_and_copy(rctx->buf + rctx->buflen, req->src,
0201                      0, req->nbytes, 0);
0202         rctx->buflen += req->nbytes;
0203         return 0;
0204     }
0205
0206     /* save the original req structure fields */
0207     rctx->src_orig = req->src;
0208     rctx->nbytes_orig = req->nbytes;
0209
0210     /*
0211      * if we have data from previous update copy them on buffer. The old
0212      * data will be combined with current request bytes.
0213      */
0214     if (rctx->buflen)
0215         memcpy(rctx->tmpbuf, rctx->buf, rctx->buflen);
0216
0217     /* calculate how many bytes will be hashed later */
0218     hash_later = total % blocksize;
0219
0220     /*
0221      * At this point, there is more than one block size of data.  If
0222      * the available data to transfer is exactly a multiple of block
0223      * size, save the last block to be transferred in qce_ahash_final
0224      * (with the last block bit set) if this is indeed the end of data
0225      * stream. If not this saved block will be transferred as part of
0226      * next update. If this block is not held back and if this is
0227      * indeed the end of data stream, the digest obtained will be wrong
0228      * since qce_ahash_final will see that rctx->buflen is 0 and return
0229      * doing nothing which in turn means that a digest will not be
0230      * copied to the destination result buffer.  qce_ahash_final cannot
0231      * be made to alter this behavior and allowed to proceed if
0232      * rctx->buflen is 0 because the crypto engine BAM does not allow
0233      * for zero length transfers.
0234      */
0235     if (!hash_later)
0236         hash_later = blocksize;
0237
0238     if (hash_later) {
0239         unsigned int src_offset = req->nbytes - hash_later;
0240         scatterwalk_map_and_copy(rctx->buf, req->src, src_offset,
0241                      hash_later, 0);
0242     }
0243
0244     /* here nbytes is multiple of blocksize */
0245     nbytes = total - hash_later;
0246
0247     len = rctx->buflen;
0248     sg = sg_last = req->src;
0249
0250     while (len < nbytes && sg) {
0251         if (len + sg_dma_len(sg) > nbytes)
0252             break;
0253         len += sg_dma_len(sg);
0254         sg_last = sg;
0255         sg = sg_next(sg);
0256     }
0257
0258     if (!sg_last)
0259         return -EINVAL;
0260
0261     if (rctx->buflen) {
0262         sg_init_table(rctx->sg, 2);
0263         sg_set_buf(rctx->sg, rctx->tmpbuf, rctx->buflen);
0264         sg_chain(rctx->sg, 2, req->src);
0265         req->src = rctx->sg;
0266     }
0267
0268     req->nbytes = nbytes;
0269     rctx->buflen = hash_later;
0270
0271     return qce->async_req_enqueue(tmpl->qce, &req->base);
0272 }
0273
0274 static int qce_ahash_final(struct ahash_request *req)
0275 {
0276     struct qce_sha_reqctx *rctx = ahash_request_ctx(req);
0277     struct qce_alg_template *tmpl = to_ahash_tmpl(req->base.tfm);
0278     struct qce_device *qce = tmpl->qce;
0279
0280     if (!rctx->buflen) {
0281         if (tmpl->hash_zero)
0282             memcpy(req->result, tmpl->hash_zero,
0283                     tmpl->alg.ahash.halg.digestsize);
0284         return 0;
0285     }
0286
0287     rctx->last_blk = true;
0288
0289     rctx->src_orig = req->src;
0290     rctx->nbytes_orig = req->nbytes;
0291
0292     memcpy(rctx->tmpbuf, rctx->buf, rctx->buflen);
0293     sg_init_one(rctx->sg, rctx->tmpbuf, rctx->buflen);
0294
0295     req->src = rctx->sg;
0296     req->nbytes = rctx->buflen;
0297
0298     return qce->async_req_enqueue(tmpl->qce, &req->base);
0299 }
0300
0301 static int qce_ahash_digest(struct ahash_request *req)
0302 {
0303     struct qce_sha_reqctx *rctx = ahash_request_ctx(req);
0304     struct qce_alg_template *tmpl = to_ahash_tmpl(req->base.tfm);
0305     struct qce_device *qce = tmpl->qce;
0306     int ret;
0307
0308     ret = qce_ahash_init(req);
0309     if (ret)
0310         return ret;
0311
0312     rctx->src_orig = req->src;
0313     rctx->nbytes_orig = req->nbytes;
0314     rctx->first_blk = true;
0315     rctx->last_blk = true;
0316
0317     if (!rctx->nbytes_orig) {
0318         if (tmpl->hash_zero)
0319             memcpy(req->result, tmpl->hash_zero,
0320                     tmpl->alg.ahash.halg.digestsize);
0321         return 0;
0322     }
0323
0324     return qce->async_req_enqueue(tmpl->qce, &req->base);
0325 }
0326
0327 static int qce_ahash_hmac_setkey(struct crypto_ahash *tfm, const u8 *key,
0328                  unsigned int keylen)
0329 {
0330     unsigned int digestsize = crypto_ahash_digestsize(tfm);
0331     struct qce_sha_ctx *ctx = crypto_tfm_ctx(&tfm->base);
0332     struct crypto_wait wait;
0333     struct ahash_request *req;
0334     struct scatterlist sg;
0335     unsigned int blocksize;
0336     struct crypto_ahash *ahash_tfm;
0337     u8 *buf;
0338     int ret;
0339     const char *alg_name;
0340
0341     blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
0342     memset(ctx->authkey, 0, sizeof(ctx->authkey));
0343
0344     if (keylen <= blocksize) {
0345         memcpy(ctx->authkey, key, keylen);
0346         return 0;
0347     }
0348
0349     if (digestsize == SHA1_DIGEST_SIZE)
0350         alg_name = "sha1-qce";
0351     else if (digestsize == SHA256_DIGEST_SIZE)
0352         alg_name = "sha256-qce";
0353     else
0354         return -EINVAL;
0355
0356     ahash_tfm = crypto_alloc_ahash(alg_name, 0, 0);
0357     if (IS_ERR(ahash_tfm))
0358         return PTR_ERR(ahash_tfm);
0359
0360     req = ahash_request_alloc(ahash_tfm, GFP_KERNEL);
0361     if (!req) {
0362         ret = -ENOMEM;
0363         goto err_free_ahash;
0364     }
0365
0366     crypto_init_wait(&wait);
0367     ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
0368                    crypto_req_done, &wait);
0369     crypto_ahash_clear_flags(ahash_tfm, ~0);
0370
0371     buf = kzalloc(keylen + QCE_MAX_ALIGN_SIZE, GFP_KERNEL);
0372     if (!buf) {
0373         ret = -ENOMEM;
0374         goto err_free_req;
0375     }
0376
0377     memcpy(buf, key, keylen);
0378     sg_init_one(&sg, buf, keylen);
0379     ahash_request_set_crypt(req, &sg, ctx->authkey, keylen);
0380
0381     ret = crypto_wait_req(crypto_ahash_digest(req), &wait);
0382
0383     kfree(buf);
0384 err_free_req:
0385     ahash_request_free(req);
0386 err_free_ahash:
0387     crypto_free_ahash(ahash_tfm);
0388     return ret;
0389 }
0390
0391 static int qce_ahash_cra_init(struct crypto_tfm *tfm)
0392 {
0393     struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
0394     struct qce_sha_ctx *ctx = crypto_tfm_ctx(tfm);
0395
0396     crypto_ahash_set_reqsize(ahash, sizeof(struct qce_sha_reqctx));
0397     memset(ctx, 0, sizeof(*ctx));
0398     return 0;
0399 }
0400
0401 struct qce_ahash_def {
0402     unsigned long flags;
0403     const char *name;
0404     const char *drv_name;
0405     unsigned int digestsize;
0406     unsigned int blocksize;
0407     unsigned int statesize;
0408     const u32 *std_iv;
0409 };
0410
0411 static const struct qce_ahash_def ahash_def[] = {
0412     {
0413         .flags      = QCE_HASH_SHA1,
0414         .name       = "sha1",
0415         .drv_name   = "sha1-qce",
0416         .digestsize = SHA1_DIGEST_SIZE,
0417         .blocksize  = SHA1_BLOCK_SIZE,
0418         .statesize  = sizeof(struct qce_sha_saved_state),
0419         .std_iv     = std_iv_sha1,
0420     },
0421     {
0422         .flags      = QCE_HASH_SHA256,
0423         .name       = "sha256",
0424         .drv_name   = "sha256-qce",
0425         .digestsize = SHA256_DIGEST_SIZE,
0426         .blocksize  = SHA256_BLOCK_SIZE,
0427         .statesize  = sizeof(struct qce_sha_saved_state),
0428         .std_iv     = std_iv_sha256,
0429     },
0430     {
0431         .flags      = QCE_HASH_SHA1_HMAC,
0432         .name       = "hmac(sha1)",
0433         .drv_name   = "hmac-sha1-qce",
0434         .digestsize = SHA1_DIGEST_SIZE,
0435         .blocksize  = SHA1_BLOCK_SIZE,
0436         .statesize  = sizeof(struct qce_sha_saved_state),
0437         .std_iv     = std_iv_sha1,
0438     },
0439     {
0440         .flags      = QCE_HASH_SHA256_HMAC,
0441         .name       = "hmac(sha256)",
0442         .drv_name   = "hmac-sha256-qce",
0443         .digestsize = SHA256_DIGEST_SIZE,
0444         .blocksize  = SHA256_BLOCK_SIZE,
0445         .statesize  = sizeof(struct qce_sha_saved_state),
0446         .std_iv     = std_iv_sha256,
0447     },
0448 };
0449
0450 static int qce_ahash_register_one(const struct qce_ahash_def *def,
0451                   struct qce_device *qce)
0452 {
0453     struct qce_alg_template *tmpl;
0454     struct ahash_alg *alg;
0455     struct crypto_alg *base;
0456     int ret;
0457
0458     tmpl = kzalloc(sizeof(*tmpl), GFP_KERNEL);
0459     if (!tmpl)
0460         return -ENOMEM;
0461
0462     tmpl->std_iv = def->std_iv;
0463
0464     alg = &tmpl->alg.ahash;
0465     alg->init = qce_ahash_init;
0466     alg->update = qce_ahash_update;
0467     alg->final = qce_ahash_final;
0468     alg->digest = qce_ahash_digest;
0469     alg->export = qce_ahash_export;
0470     alg->import = qce_ahash_import;
0471     if (IS_SHA_HMAC(def->flags))
0472         alg->setkey = qce_ahash_hmac_setkey;
0473     alg->halg.digestsize = def->digestsize;
0474     alg->halg.statesize = def->statesize;
0475
0476     if (IS_SHA1(def->flags))
0477         tmpl->hash_zero = sha1_zero_message_hash;
0478     else if (IS_SHA256(def->flags))
0479         tmpl->hash_zero = sha256_zero_message_hash;
0480
0481     base = &alg->halg.base;
0482     base->cra_blocksize = def->blocksize;
0483     base->cra_priority = 300;
0484     base->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY;
0485     base->cra_ctxsize = sizeof(struct qce_sha_ctx);
0486     base->cra_alignmask = 0;
0487     base->cra_module = THIS_MODULE;
0488     base->cra_init = qce_ahash_cra_init;
0489
0490     snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "%s", def->name);
0491     snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
0492          def->drv_name);
0493
0494     INIT_LIST_HEAD(&tmpl->entry);
0495     tmpl->crypto_alg_type = CRYPTO_ALG_TYPE_AHASH;
0496     tmpl->alg_flags = def->flags;
0497     tmpl->qce = qce;
0498
0499     ret = crypto_register_ahash(alg);
0500     if (ret) {
0501         dev_err(qce->dev, "%s registration failed\n", base->cra_name);
0502         kfree(tmpl);
0503         return ret;
0504     }
0505
0506     list_add_tail(&tmpl->entry, &ahash_algs);
0507     dev_dbg(qce->dev, "%s is registered\n", base->cra_name);
0508     return 0;
0509 }
0510
0511 static void qce_ahash_unregister(struct qce_device *qce)
0512 {
0513     struct qce_alg_template *tmpl, *n;
0514
0515     list_for_each_entry_safe(tmpl, n, &ahash_algs, entry) {
0516         crypto_unregister_ahash(&tmpl->alg.ahash);
0517         list_del(&tmpl->entry);
0518         kfree(tmpl);
0519     }
0520 }
0521
0522 static int qce_ahash_register(struct qce_device *qce)
0523 {
0524     int ret, i;
0525
0526     for (i = 0; i < ARRAY_SIZE(ahash_def); i++) {
0527         ret = qce_ahash_register_one(&ahash_def[i], qce);
0528         if (ret)
0529             goto err;
0530     }
0531
0532     return 0;
0533 err:
0534     qce_ahash_unregister(qce);
0535     return ret;
0536 }
0537
0538 const struct qce_algo_ops ahash_ops = {
0539     .type = CRYPTO_ALG_TYPE_AHASH,
0540     .register_algs = qce_ahash_register,
0541     .unregister_algs = qce_ahash_unregister,
0542     .async_req_handle = qce_ahash_async_req_handle,
0543 };