the-tree/crypto/tcrypt.c

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  * Quick & dirty crypto testing module.
0004  *
0005  * This will only exist until we have a better testing mechanism
0006  * (e.g. a char device).
0007  *
0008  * Copyright (c) 2002 James Morris <jmorris@intercode.com.au>
0009  * Copyright (c) 2002 Jean-Francois Dive <jef@linuxbe.org>
0010  * Copyright (c) 2007 Nokia Siemens Networks
0011  *
0012  * Updated RFC4106 AES-GCM testing.
0013  *    Authors: Aidan O'Mahony (aidan.o.mahony@intel.com)
0014  *             Adrian Hoban <adrian.hoban@intel.com>
0015  *             Gabriele Paoloni <gabriele.paoloni@intel.com>
0016  *             Tadeusz Struk (tadeusz.struk@intel.com)
0017  *             Copyright (c) 2010, Intel Corporation.
0018  */
0019
0020 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
0021
0022 #include <crypto/aead.h>
0023 #include <crypto/hash.h>
0024 #include <crypto/skcipher.h>
0025 #include <linux/err.h>
0026 #include <linux/fips.h>
0027 #include <linux/init.h>
0028 #include <linux/gfp.h>
0029 #include <linux/module.h>
0030 #include <linux/scatterlist.h>
0031 #include <linux/string.h>
0032 #include <linux/moduleparam.h>
0033 #include <linux/jiffies.h>
0034 #include <linux/timex.h>
0035 #include <linux/interrupt.h>
0036 #include "tcrypt.h"
0037
0038 /*
0039  * Need slab memory for testing (size in number of pages).
0040  */
0041 #define TVMEMSIZE   4
0042
0043 /*
0044 * Used by test_cipher_speed()
0045 */
0046 #define ENCRYPT 1
0047 #define DECRYPT 0
0048
0049 #define MAX_DIGEST_SIZE     64
0050
0051 /*
0052  * return a string with the driver name
0053  */
0054 #define get_driver_name(tfm_type, tfm) crypto_tfm_alg_driver_name(tfm_type ## _tfm(tfm))
0055
0056 /*
0057  * Used by test_cipher_speed()
0058  */
0059 static unsigned int sec;
0060
0061 static char *alg;
0062 static u32 type;
0063 static u32 mask;
0064 static int mode;
0065 static u32 num_mb = 8;
0066 static unsigned int klen;
0067 static char *tvmem[TVMEMSIZE];
0068
0069 static const char *check[] = {
0070     "des", "md5", "des3_ede", "rot13", "sha1", "sha224", "sha256", "sm3",
0071     "blowfish", "twofish", "serpent", "sha384", "sha512", "md4", "aes",
0072     "cast6", "arc4", "michael_mic", "deflate", "crc32c", "tea", "xtea",
0073     "khazad", "wp512", "wp384", "wp256", "xeta",  "fcrypt",
0074     "camellia", "seed", "rmd160", "aria",
0075     "lzo", "lzo-rle", "cts", "sha3-224", "sha3-256", "sha3-384",
0076     "sha3-512", "streebog256", "streebog512",
0077     NULL
0078 };
0079
0080 static const int block_sizes[] = { 16, 64, 128, 256, 1024, 1420, 4096, 0 };
0081 static const int aead_sizes[] = { 16, 64, 256, 512, 1024, 1420, 4096, 8192, 0 };
0082
0083 #define XBUFSIZE 8
0084 #define MAX_IVLEN 32
0085
0086 static int testmgr_alloc_buf(char *buf[XBUFSIZE])
0087 {
0088     int i;
0089
0090     for (i = 0; i < XBUFSIZE; i++) {
0091         buf[i] = (void *)__get_free_page(GFP_KERNEL);
0092         if (!buf[i])
0093             goto err_free_buf;
0094     }
0095
0096     return 0;
0097
0098 err_free_buf:
0099     while (i-- > 0)
0100         free_page((unsigned long)buf[i]);
0101
0102     return -ENOMEM;
0103 }
0104
0105 static void testmgr_free_buf(char *buf[XBUFSIZE])
0106 {
0107     int i;
0108
0109     for (i = 0; i < XBUFSIZE; i++)
0110         free_page((unsigned long)buf[i]);
0111 }
0112
0113 static void sg_init_aead(struct scatterlist *sg, char *xbuf[XBUFSIZE],
0114              unsigned int buflen, const void *assoc,
0115              unsigned int aad_size)
0116 {
0117     int np = (buflen + PAGE_SIZE - 1)/PAGE_SIZE;
0118     int k, rem;
0119
0120     if (np > XBUFSIZE) {
0121         rem = PAGE_SIZE;
0122         np = XBUFSIZE;
0123     } else {
0124         rem = buflen % PAGE_SIZE;
0125     }
0126
0127     sg_init_table(sg, np + 1);
0128
0129     sg_set_buf(&sg[0], assoc, aad_size);
0130
0131     if (rem)
0132         np--;
0133     for (k = 0; k < np; k++)
0134         sg_set_buf(&sg[k + 1], xbuf[k], PAGE_SIZE);
0135
0136     if (rem)
0137         sg_set_buf(&sg[k + 1], xbuf[k], rem);
0138 }
0139
0140 static inline int do_one_aead_op(struct aead_request *req, int ret)
0141 {
0142     struct crypto_wait *wait = req->base.data;
0143
0144     return crypto_wait_req(ret, wait);
0145 }
0146
0147 struct test_mb_aead_data {
0148     struct scatterlist sg[XBUFSIZE];
0149     struct scatterlist sgout[XBUFSIZE];
0150     struct aead_request *req;
0151     struct crypto_wait wait;
0152     char *xbuf[XBUFSIZE];
0153     char *xoutbuf[XBUFSIZE];
0154     char *axbuf[XBUFSIZE];
0155 };
0156
0157 static int do_mult_aead_op(struct test_mb_aead_data *data, int enc,
0158                 u32 num_mb, int *rc)
0159 {
0160     int i, err = 0;
0161
0162     /* Fire up a bunch of concurrent requests */
0163     for (i = 0; i < num_mb; i++) {
0164         if (enc == ENCRYPT)
0165             rc[i] = crypto_aead_encrypt(data[i].req);
0166         else
0167             rc[i] = crypto_aead_decrypt(data[i].req);
0168     }
0169
0170     /* Wait for all requests to finish */
0171     for (i = 0; i < num_mb; i++) {
0172         rc[i] = crypto_wait_req(rc[i], &data[i].wait);
0173
0174         if (rc[i]) {
0175             pr_info("concurrent request %d error %d\n", i, rc[i]);
0176             err = rc[i];
0177         }
0178     }
0179
0180     return err;
0181 }
0182
0183 static int test_mb_aead_jiffies(struct test_mb_aead_data *data, int enc,
0184                 int blen, int secs, u32 num_mb)
0185 {
0186     unsigned long start, end;
0187     int bcount;
0188     int ret = 0;
0189     int *rc;
0190
0191     rc = kcalloc(num_mb, sizeof(*rc), GFP_KERNEL);
0192     if (!rc)
0193         return -ENOMEM;
0194
0195     for (start = jiffies, end = start + secs * HZ, bcount = 0;
0196          time_before(jiffies, end); bcount++) {
0197         ret = do_mult_aead_op(data, enc, num_mb, rc);
0198         if (ret)
0199             goto out;
0200     }
0201
0202     pr_cont("%d operations in %d seconds (%llu bytes)\n",
0203         bcount * num_mb, secs, (u64)bcount * blen * num_mb);
0204
0205 out:
0206     kfree(rc);
0207     return ret;
0208 }
0209
0210 static int test_mb_aead_cycles(struct test_mb_aead_data *data, int enc,
0211                    int blen, u32 num_mb)
0212 {
0213     unsigned long cycles = 0;
0214     int ret = 0;
0215     int i;
0216     int *rc;
0217
0218     rc = kcalloc(num_mb, sizeof(*rc), GFP_KERNEL);
0219     if (!rc)
0220         return -ENOMEM;
0221
0222     /* Warm-up run. */
0223     for (i = 0; i < 4; i++) {
0224         ret = do_mult_aead_op(data, enc, num_mb, rc);
0225         if (ret)
0226             goto out;
0227     }
0228
0229     /* The real thing. */
0230     for (i = 0; i < 8; i++) {
0231         cycles_t start, end;
0232
0233         start = get_cycles();
0234         ret = do_mult_aead_op(data, enc, num_mb, rc);
0235         end = get_cycles();
0236
0237         if (ret)
0238             goto out;
0239
0240         cycles += end - start;
0241     }
0242
0243     pr_cont("1 operation in %lu cycles (%d bytes)\n",
0244         (cycles + 4) / (8 * num_mb), blen);
0245
0246 out:
0247     kfree(rc);
0248     return ret;
0249 }
0250
0251 static void test_mb_aead_speed(const char *algo, int enc, int secs,
0252                    struct aead_speed_template *template,
0253                    unsigned int tcount, u8 authsize,
0254                    unsigned int aad_size, u8 *keysize, u32 num_mb)
0255 {
0256     struct test_mb_aead_data *data;
0257     struct crypto_aead *tfm;
0258     unsigned int i, j, iv_len;
0259     const int *b_size;
0260     const char *key;
0261     const char *e;
0262     void *assoc;
0263     char *iv;
0264     int ret;
0265
0266
0267     if (aad_size >= PAGE_SIZE) {
0268         pr_err("associate data length (%u) too big\n", aad_size);
0269         return;
0270     }
0271
0272     iv = kzalloc(MAX_IVLEN, GFP_KERNEL);
0273     if (!iv)
0274         return;
0275
0276     if (enc == ENCRYPT)
0277         e = "encryption";
0278     else
0279         e = "decryption";
0280
0281     data = kcalloc(num_mb, sizeof(*data), GFP_KERNEL);
0282     if (!data)
0283         goto out_free_iv;
0284
0285     tfm = crypto_alloc_aead(algo, 0, 0);
0286     if (IS_ERR(tfm)) {
0287         pr_err("failed to load transform for %s: %ld\n",
0288             algo, PTR_ERR(tfm));
0289         goto out_free_data;
0290     }
0291
0292     ret = crypto_aead_setauthsize(tfm, authsize);
0293     if (ret) {
0294         pr_err("alg: aead: Failed to setauthsize for %s: %d\n", algo,
0295                ret);
0296         goto out_free_tfm;
0297     }
0298
0299     for (i = 0; i < num_mb; ++i)
0300         if (testmgr_alloc_buf(data[i].xbuf)) {
0301             while (i--)
0302                 testmgr_free_buf(data[i].xbuf);
0303             goto out_free_tfm;
0304         }
0305
0306     for (i = 0; i < num_mb; ++i)
0307         if (testmgr_alloc_buf(data[i].axbuf)) {
0308             while (i--)
0309                 testmgr_free_buf(data[i].axbuf);
0310             goto out_free_xbuf;
0311         }
0312
0313     for (i = 0; i < num_mb; ++i)
0314         if (testmgr_alloc_buf(data[i].xoutbuf)) {
0315             while (i--)
0316                 testmgr_free_buf(data[i].xoutbuf);
0317             goto out_free_axbuf;
0318         }
0319
0320     for (i = 0; i < num_mb; ++i) {
0321         data[i].req = aead_request_alloc(tfm, GFP_KERNEL);
0322         if (!data[i].req) {
0323             pr_err("alg: aead: Failed to allocate request for %s\n",
0324                    algo);
0325             while (i--)
0326                 aead_request_free(data[i].req);
0327             goto out_free_xoutbuf;
0328         }
0329     }
0330
0331     for (i = 0; i < num_mb; ++i) {
0332         crypto_init_wait(&data[i].wait);
0333         aead_request_set_callback(data[i].req,
0334                       CRYPTO_TFM_REQ_MAY_BACKLOG,
0335                       crypto_req_done, &data[i].wait);
0336     }
0337
0338     pr_info("\ntesting speed of multibuffer %s (%s) %s\n", algo,
0339         get_driver_name(crypto_aead, tfm), e);
0340
0341     i = 0;
0342     do {
0343         b_size = aead_sizes;
0344         do {
0345             int bs = round_up(*b_size, crypto_aead_blocksize(tfm));
0346
0347             if (bs + authsize > XBUFSIZE * PAGE_SIZE) {
0348                 pr_err("template (%u) too big for buffer (%lu)\n",
0349                        authsize + bs,
0350                        XBUFSIZE * PAGE_SIZE);
0351                 goto out;
0352             }
0353
0354             pr_info("test %u (%d bit key, %d byte blocks): ", i,
0355                 *keysize * 8, bs);
0356
0357             /* Set up tfm global state, i.e. the key */
0358
0359             memset(tvmem[0], 0xff, PAGE_SIZE);
0360             key = tvmem[0];
0361             for (j = 0; j < tcount; j++) {
0362                 if (template[j].klen == *keysize) {
0363                     key = template[j].key;
0364                     break;
0365                 }
0366             }
0367
0368             crypto_aead_clear_flags(tfm, ~0);
0369
0370             ret = crypto_aead_setkey(tfm, key, *keysize);
0371             if (ret) {
0372                 pr_err("setkey() failed flags=%x\n",
0373                        crypto_aead_get_flags(tfm));
0374                 goto out;
0375             }
0376
0377             iv_len = crypto_aead_ivsize(tfm);
0378             if (iv_len)
0379                 memset(iv, 0xff, iv_len);
0380
0381             /* Now setup per request stuff, i.e. buffers */
0382
0383             for (j = 0; j < num_mb; ++j) {
0384                 struct test_mb_aead_data *cur = &data[j];
0385
0386                 assoc = cur->axbuf[0];
0387                 memset(assoc, 0xff, aad_size);
0388
0389                 sg_init_aead(cur->sg, cur->xbuf,
0390                          bs + (enc ? 0 : authsize),
0391                          assoc, aad_size);
0392
0393                 sg_init_aead(cur->sgout, cur->xoutbuf,
0394                          bs + (enc ? authsize : 0),
0395                          assoc, aad_size);
0396
0397                 aead_request_set_ad(cur->req, aad_size);
0398
0399                 if (!enc) {
0400
0401                     aead_request_set_crypt(cur->req,
0402                                    cur->sgout,
0403                                    cur->sg,
0404                                    bs, iv);
0405                     ret = crypto_aead_encrypt(cur->req);
0406                     ret = do_one_aead_op(cur->req, ret);
0407
0408                     if (ret) {
0409                         pr_err("calculating auth failed (%d)\n",
0410                                ret);
0411                         break;
0412                     }
0413                 }
0414
0415                 aead_request_set_crypt(cur->req, cur->sg,
0416                                cur->sgout, bs +
0417                                (enc ? 0 : authsize),
0418                                iv);
0419
0420             }
0421
0422             if (secs) {
0423                 ret = test_mb_aead_jiffies(data, enc, bs,
0424                                secs, num_mb);
0425                 cond_resched();
0426             } else {
0427                 ret = test_mb_aead_cycles(data, enc, bs,
0428                               num_mb);
0429             }
0430
0431             if (ret) {
0432                 pr_err("%s() failed return code=%d\n", e, ret);
0433                 break;
0434             }
0435             b_size++;
0436             i++;
0437         } while (*b_size);
0438         keysize++;
0439     } while (*keysize);
0440
0441 out:
0442     for (i = 0; i < num_mb; ++i)
0443         aead_request_free(data[i].req);
0444 out_free_xoutbuf:
0445     for (i = 0; i < num_mb; ++i)
0446         testmgr_free_buf(data[i].xoutbuf);
0447 out_free_axbuf:
0448     for (i = 0; i < num_mb; ++i)
0449         testmgr_free_buf(data[i].axbuf);
0450 out_free_xbuf:
0451     for (i = 0; i < num_mb; ++i)
0452         testmgr_free_buf(data[i].xbuf);
0453 out_free_tfm:
0454     crypto_free_aead(tfm);
0455 out_free_data:
0456     kfree(data);
0457 out_free_iv:
0458     kfree(iv);
0459 }
0460
0461 static int test_aead_jiffies(struct aead_request *req, int enc,
0462                 int blen, int secs)
0463 {
0464     unsigned long start, end;
0465     int bcount;
0466     int ret;
0467
0468     for (start = jiffies, end = start + secs * HZ, bcount = 0;
0469          time_before(jiffies, end); bcount++) {
0470         if (enc)
0471             ret = do_one_aead_op(req, crypto_aead_encrypt(req));
0472         else
0473             ret = do_one_aead_op(req, crypto_aead_decrypt(req));
0474
0475         if (ret)
0476             return ret;
0477     }
0478
0479     pr_cont("%d operations in %d seconds (%llu bytes)\n",
0480             bcount, secs, (u64)bcount * blen);
0481     return 0;
0482 }
0483
0484 static int test_aead_cycles(struct aead_request *req, int enc, int blen)
0485 {
0486     unsigned long cycles = 0;
0487     int ret = 0;
0488     int i;
0489
0490     /* Warm-up run. */
0491     for (i = 0; i < 4; i++) {
0492         if (enc)
0493             ret = do_one_aead_op(req, crypto_aead_encrypt(req));
0494         else
0495             ret = do_one_aead_op(req, crypto_aead_decrypt(req));
0496
0497         if (ret)
0498             goto out;
0499     }
0500
0501     /* The real thing. */
0502     for (i = 0; i < 8; i++) {
0503         cycles_t start, end;
0504
0505         start = get_cycles();
0506         if (enc)
0507             ret = do_one_aead_op(req, crypto_aead_encrypt(req));
0508         else
0509             ret = do_one_aead_op(req, crypto_aead_decrypt(req));
0510         end = get_cycles();
0511
0512         if (ret)
0513             goto out;
0514
0515         cycles += end - start;
0516     }
0517
0518 out:
0519     if (ret == 0)
0520         printk("1 operation in %lu cycles (%d bytes)\n",
0521                (cycles + 4) / 8, blen);
0522
0523     return ret;
0524 }
0525
0526 static void test_aead_speed(const char *algo, int enc, unsigned int secs,
0527                 struct aead_speed_template *template,
0528                 unsigned int tcount, u8 authsize,
0529                 unsigned int aad_size, u8 *keysize)
0530 {
0531     unsigned int i, j;
0532     struct crypto_aead *tfm;
0533     int ret = -ENOMEM;
0534     const char *key;
0535     struct aead_request *req;
0536     struct scatterlist *sg;
0537     struct scatterlist *sgout;
0538     const char *e;
0539     void *assoc;
0540     char *iv;
0541     char *xbuf[XBUFSIZE];
0542     char *xoutbuf[XBUFSIZE];
0543     char *axbuf[XBUFSIZE];
0544     const int *b_size;
0545     unsigned int iv_len;
0546     struct crypto_wait wait;
0547
0548     iv = kzalloc(MAX_IVLEN, GFP_KERNEL);
0549     if (!iv)
0550         return;
0551
0552     if (aad_size >= PAGE_SIZE) {
0553         pr_err("associate data length (%u) too big\n", aad_size);
0554         goto out_noxbuf;
0555     }
0556
0557     if (enc == ENCRYPT)
0558         e = "encryption";
0559     else
0560         e = "decryption";
0561
0562     if (testmgr_alloc_buf(xbuf))
0563         goto out_noxbuf;
0564     if (testmgr_alloc_buf(axbuf))
0565         goto out_noaxbuf;
0566     if (testmgr_alloc_buf(xoutbuf))
0567         goto out_nooutbuf;
0568
0569     sg = kmalloc(sizeof(*sg) * 9 * 2, GFP_KERNEL);
0570     if (!sg)
0571         goto out_nosg;
0572     sgout = &sg[9];
0573
0574     tfm = crypto_alloc_aead(algo, 0, 0);
0575     if (IS_ERR(tfm)) {
0576         pr_err("alg: aead: Failed to load transform for %s: %ld\n", algo,
0577                PTR_ERR(tfm));
0578         goto out_notfm;
0579     }
0580
0581     ret = crypto_aead_setauthsize(tfm, authsize);
0582     if (ret) {
0583         pr_err("alg: aead: Failed to setauthsize for %s: %d\n", algo,
0584                ret);
0585         goto out_noreq;
0586     }
0587
0588     crypto_init_wait(&wait);
0589     printk(KERN_INFO "\ntesting speed of %s (%s) %s\n", algo,
0590             get_driver_name(crypto_aead, tfm), e);
0591
0592     req = aead_request_alloc(tfm, GFP_KERNEL);
0593     if (!req) {
0594         pr_err("alg: aead: Failed to allocate request for %s\n",
0595                algo);
0596         goto out_noreq;
0597     }
0598
0599     aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
0600                   crypto_req_done, &wait);
0601
0602     i = 0;
0603     do {
0604         b_size = aead_sizes;
0605         do {
0606             u32 bs = round_up(*b_size, crypto_aead_blocksize(tfm));
0607
0608             assoc = axbuf[0];
0609             memset(assoc, 0xff, aad_size);
0610
0611             if ((*keysize + bs) > TVMEMSIZE * PAGE_SIZE) {
0612                 pr_err("template (%u) too big for tvmem (%lu)\n",
0613                        *keysize + bs,
0614                     TVMEMSIZE * PAGE_SIZE);
0615                 goto out;
0616             }
0617
0618             key = tvmem[0];
0619             for (j = 0; j < tcount; j++) {
0620                 if (template[j].klen == *keysize) {
0621                     key = template[j].key;
0622                     break;
0623                 }
0624             }
0625
0626             ret = crypto_aead_setkey(tfm, key, *keysize);
0627             if (ret) {
0628                 pr_err("setkey() failed flags=%x: %d\n",
0629                     crypto_aead_get_flags(tfm), ret);
0630                 goto out;
0631             }
0632
0633             iv_len = crypto_aead_ivsize(tfm);
0634             if (iv_len)
0635                 memset(iv, 0xff, iv_len);
0636
0637             crypto_aead_clear_flags(tfm, ~0);
0638             printk(KERN_INFO "test %u (%d bit key, %d byte blocks): ",
0639                     i, *keysize * 8, bs);
0640
0641             memset(tvmem[0], 0xff, PAGE_SIZE);
0642
0643             sg_init_aead(sg, xbuf, bs + (enc ? 0 : authsize),
0644                      assoc, aad_size);
0645
0646             sg_init_aead(sgout, xoutbuf,
0647                      bs + (enc ? authsize : 0), assoc,
0648                      aad_size);
0649
0650             aead_request_set_ad(req, aad_size);
0651
0652             if (!enc) {
0653
0654                 /*
0655                  * For decryption we need a proper auth so
0656                  * we do the encryption path once with buffers
0657                  * reversed (input <-> output) to calculate it
0658                  */
0659                 aead_request_set_crypt(req, sgout, sg,
0660                                bs, iv);
0661                 ret = do_one_aead_op(req,
0662                              crypto_aead_encrypt(req));
0663
0664                 if (ret) {
0665                     pr_err("calculating auth failed (%d)\n",
0666                            ret);
0667                     break;
0668                 }
0669             }
0670
0671             aead_request_set_crypt(req, sg, sgout,
0672                            bs + (enc ? 0 : authsize),
0673                            iv);
0674
0675             if (secs) {
0676                 ret = test_aead_jiffies(req, enc, bs,
0677                             secs);
0678                 cond_resched();
0679             } else {
0680                 ret = test_aead_cycles(req, enc, bs);
0681             }
0682
0683             if (ret) {
0684                 pr_err("%s() failed return code=%d\n", e, ret);
0685                 break;
0686             }
0687             b_size++;
0688             i++;
0689         } while (*b_size);
0690         keysize++;
0691     } while (*keysize);
0692
0693 out:
0694     aead_request_free(req);
0695 out_noreq:
0696     crypto_free_aead(tfm);
0697 out_notfm:
0698     kfree(sg);
0699 out_nosg:
0700     testmgr_free_buf(xoutbuf);
0701 out_nooutbuf:
0702     testmgr_free_buf(axbuf);
0703 out_noaxbuf:
0704     testmgr_free_buf(xbuf);
0705 out_noxbuf:
0706     kfree(iv);
0707 }
0708
0709 static void test_hash_sg_init(struct scatterlist *sg)
0710 {
0711     int i;
0712
0713     sg_init_table(sg, TVMEMSIZE);
0714     for (i = 0; i < TVMEMSIZE; i++) {
0715         sg_set_buf(sg + i, tvmem[i], PAGE_SIZE);
0716         memset(tvmem[i], 0xff, PAGE_SIZE);
0717     }
0718 }
0719
0720 static inline int do_one_ahash_op(struct ahash_request *req, int ret)
0721 {
0722     struct crypto_wait *wait = req->base.data;
0723
0724     return crypto_wait_req(ret, wait);
0725 }
0726
0727 static int test_ahash_jiffies_digest(struct ahash_request *req, int blen,
0728                      char *out, int secs)
0729 {
0730     unsigned long start, end;
0731     int bcount;
0732     int ret;
0733
0734     for (start = jiffies, end = start + secs * HZ, bcount = 0;
0735          time_before(jiffies, end); bcount++) {
0736         ret = do_one_ahash_op(req, crypto_ahash_digest(req));
0737         if (ret)
0738             return ret;
0739     }
0740
0741     printk("%6u opers/sec, %9lu bytes/sec\n",
0742            bcount / secs, ((long)bcount * blen) / secs);
0743
0744     return 0;
0745 }
0746
0747 static int test_ahash_jiffies(struct ahash_request *req, int blen,
0748                   int plen, char *out, int secs)
0749 {
0750     unsigned long start, end;
0751     int bcount, pcount;
0752     int ret;
0753
0754     if (plen == blen)
0755         return test_ahash_jiffies_digest(req, blen, out, secs);
0756
0757     for (start = jiffies, end = start + secs * HZ, bcount = 0;
0758          time_before(jiffies, end); bcount++) {
0759         ret = do_one_ahash_op(req, crypto_ahash_init(req));
0760         if (ret)
0761             return ret;
0762         for (pcount = 0; pcount < blen; pcount += plen) {
0763             ret = do_one_ahash_op(req, crypto_ahash_update(req));
0764             if (ret)
0765                 return ret;
0766         }
0767         /* we assume there is enough space in 'out' for the result */
0768         ret = do_one_ahash_op(req, crypto_ahash_final(req));
0769         if (ret)
0770             return ret;
0771     }
0772
0773     pr_cont("%6u opers/sec, %9lu bytes/sec\n",
0774         bcount / secs, ((long)bcount * blen) / secs);
0775
0776     return 0;
0777 }
0778
0779 static int test_ahash_cycles_digest(struct ahash_request *req, int blen,
0780                     char *out)
0781 {
0782     unsigned long cycles = 0;
0783     int ret, i;
0784
0785     /* Warm-up run. */
0786     for (i = 0; i < 4; i++) {
0787         ret = do_one_ahash_op(req, crypto_ahash_digest(req));
0788         if (ret)
0789             goto out;
0790     }
0791
0792     /* The real thing. */
0793     for (i = 0; i < 8; i++) {
0794         cycles_t start, end;
0795
0796         start = get_cycles();
0797
0798         ret = do_one_ahash_op(req, crypto_ahash_digest(req));
0799         if (ret)
0800             goto out;
0801
0802         end = get_cycles();
0803
0804         cycles += end - start;
0805     }
0806
0807 out:
0808     if (ret)
0809         return ret;
0810
0811     pr_cont("%6lu cycles/operation, %4lu cycles/byte\n",
0812         cycles / 8, cycles / (8 * blen));
0813
0814     return 0;
0815 }
0816
0817 static int test_ahash_cycles(struct ahash_request *req, int blen,
0818                  int plen, char *out)
0819 {
0820     unsigned long cycles = 0;
0821     int i, pcount, ret;
0822
0823     if (plen == blen)
0824         return test_ahash_cycles_digest(req, blen, out);
0825
0826     /* Warm-up run. */
0827     for (i = 0; i < 4; i++) {
0828         ret = do_one_ahash_op(req, crypto_ahash_init(req));
0829         if (ret)
0830             goto out;
0831         for (pcount = 0; pcount < blen; pcount += plen) {
0832             ret = do_one_ahash_op(req, crypto_ahash_update(req));
0833             if (ret)
0834                 goto out;
0835         }
0836         ret = do_one_ahash_op(req, crypto_ahash_final(req));
0837         if (ret)
0838             goto out;
0839     }
0840
0841     /* The real thing. */
0842     for (i = 0; i < 8; i++) {
0843         cycles_t start, end;
0844
0845         start = get_cycles();
0846
0847         ret = do_one_ahash_op(req, crypto_ahash_init(req));
0848         if (ret)
0849             goto out;
0850         for (pcount = 0; pcount < blen; pcount += plen) {
0851             ret = do_one_ahash_op(req, crypto_ahash_update(req));
0852             if (ret)
0853                 goto out;
0854         }
0855         ret = do_one_ahash_op(req, crypto_ahash_final(req));
0856         if (ret)
0857             goto out;
0858
0859         end = get_cycles();
0860
0861         cycles += end - start;
0862     }
0863
0864 out:
0865     if (ret)
0866         return ret;
0867
0868     pr_cont("%6lu cycles/operation, %4lu cycles/byte\n",
0869         cycles / 8, cycles / (8 * blen));
0870
0871     return 0;
0872 }
0873
0874 static void test_ahash_speed_common(const char *algo, unsigned int secs,
0875                     struct hash_speed *speed, unsigned mask)
0876 {
0877     struct scatterlist sg[TVMEMSIZE];
0878     struct crypto_wait wait;
0879     struct ahash_request *req;
0880     struct crypto_ahash *tfm;
0881     char *output;
0882     int i, ret;
0883
0884     tfm = crypto_alloc_ahash(algo, 0, mask);
0885     if (IS_ERR(tfm)) {
0886         pr_err("failed to load transform for %s: %ld\n",
0887                algo, PTR_ERR(tfm));
0888         return;
0889     }
0890
0891     printk(KERN_INFO "\ntesting speed of async %s (%s)\n", algo,
0892             get_driver_name(crypto_ahash, tfm));
0893
0894     if (crypto_ahash_digestsize(tfm) > MAX_DIGEST_SIZE) {
0895         pr_err("digestsize(%u) > %d\n", crypto_ahash_digestsize(tfm),
0896                MAX_DIGEST_SIZE);
0897         goto out;
0898     }
0899
0900     test_hash_sg_init(sg);
0901     req = ahash_request_alloc(tfm, GFP_KERNEL);
0902     if (!req) {
0903         pr_err("ahash request allocation failure\n");
0904         goto out;
0905     }
0906
0907     crypto_init_wait(&wait);
0908     ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
0909                    crypto_req_done, &wait);
0910
0911     output = kmalloc(MAX_DIGEST_SIZE, GFP_KERNEL);
0912     if (!output)
0913         goto out_nomem;
0914
0915     for (i = 0; speed[i].blen != 0; i++) {
0916         if (speed[i].blen > TVMEMSIZE * PAGE_SIZE) {
0917             pr_err("template (%u) too big for tvmem (%lu)\n",
0918                    speed[i].blen, TVMEMSIZE * PAGE_SIZE);
0919             break;
0920         }
0921
0922         if (klen)
0923             crypto_ahash_setkey(tfm, tvmem[0], klen);
0924
0925         pr_info("test%3u "
0926             "(%5u byte blocks,%5u bytes per update,%4u updates): ",
0927             i, speed[i].blen, speed[i].plen, speed[i].blen / speed[i].plen);
0928
0929         ahash_request_set_crypt(req, sg, output, speed[i].plen);
0930
0931         if (secs) {
0932             ret = test_ahash_jiffies(req, speed[i].blen,
0933                          speed[i].plen, output, secs);
0934             cond_resched();
0935         } else {
0936             ret = test_ahash_cycles(req, speed[i].blen,
0937                         speed[i].plen, output);
0938         }
0939
0940         if (ret) {
0941             pr_err("hashing failed ret=%d\n", ret);
0942             break;
0943         }
0944     }
0945
0946     kfree(output);
0947
0948 out_nomem:
0949     ahash_request_free(req);
0950
0951 out:
0952     crypto_free_ahash(tfm);
0953 }
0954
0955 static void test_ahash_speed(const char *algo, unsigned int secs,
0956                  struct hash_speed *speed)
0957 {
0958     return test_ahash_speed_common(algo, secs, speed, 0);
0959 }
0960
0961 static void test_hash_speed(const char *algo, unsigned int secs,
0962                 struct hash_speed *speed)
0963 {
0964     return test_ahash_speed_common(algo, secs, speed, CRYPTO_ALG_ASYNC);
0965 }
0966
0967 struct test_mb_skcipher_data {
0968     struct scatterlist sg[XBUFSIZE];
0969     struct skcipher_request *req;
0970     struct crypto_wait wait;
0971     char *xbuf[XBUFSIZE];
0972 };
0973
0974 static int do_mult_acipher_op(struct test_mb_skcipher_data *data, int enc,
0975                 u32 num_mb, int *rc)
0976 {
0977     int i, err = 0;
0978
0979     /* Fire up a bunch of concurrent requests */
0980     for (i = 0; i < num_mb; i++) {
0981         if (enc == ENCRYPT)
0982             rc[i] = crypto_skcipher_encrypt(data[i].req);
0983         else
0984             rc[i] = crypto_skcipher_decrypt(data[i].req);
0985     }
0986
0987     /* Wait for all requests to finish */
0988     for (i = 0; i < num_mb; i++) {
0989         rc[i] = crypto_wait_req(rc[i], &data[i].wait);
0990
0991         if (rc[i]) {
0992             pr_info("concurrent request %d error %d\n", i, rc[i]);
0993             err = rc[i];
0994         }
0995     }
0996
0997     return err;
0998 }
0999
1000 static int test_mb_acipher_jiffies(struct test_mb_skcipher_data *data, int enc,
1001                 int blen, int secs, u32 num_mb)
1002 {
1003     unsigned long start, end;
1004     int bcount;
1005     int ret = 0;
1006     int *rc;
1007
1008     rc = kcalloc(num_mb, sizeof(*rc), GFP_KERNEL);
1009     if (!rc)
1010         return -ENOMEM;
1011
1012     for (start = jiffies, end = start + secs * HZ, bcount = 0;
1013          time_before(jiffies, end); bcount++) {
1014         ret = do_mult_acipher_op(data, enc, num_mb, rc);
1015         if (ret)
1016             goto out;
1017     }
1018
1019     pr_cont("%d operations in %d seconds (%llu bytes)\n",
1020         bcount * num_mb, secs, (u64)bcount * blen * num_mb);
1021
1022 out:
1023     kfree(rc);
1024     return ret;
1025 }
1026
1027 static int test_mb_acipher_cycles(struct test_mb_skcipher_data *data, int enc,
1028                    int blen, u32 num_mb)
1029 {
1030     unsigned long cycles = 0;
1031     int ret = 0;
1032     int i;
1033     int *rc;
1034
1035     rc = kcalloc(num_mb, sizeof(*rc), GFP_KERNEL);
1036     if (!rc)
1037         return -ENOMEM;
1038
1039     /* Warm-up run. */
1040     for (i = 0; i < 4; i++) {
1041         ret = do_mult_acipher_op(data, enc, num_mb, rc);
1042         if (ret)
1043             goto out;
1044     }
1045
1046     /* The real thing. */
1047     for (i = 0; i < 8; i++) {
1048         cycles_t start, end;
1049
1050         start = get_cycles();
1051         ret = do_mult_acipher_op(data, enc, num_mb, rc);
1052         end = get_cycles();
1053
1054         if (ret)
1055             goto out;
1056
1057         cycles += end - start;
1058     }
1059
1060     pr_cont("1 operation in %lu cycles (%d bytes)\n",
1061         (cycles + 4) / (8 * num_mb), blen);
1062
1063 out:
1064     kfree(rc);
1065     return ret;
1066 }
1067
1068 static void test_mb_skcipher_speed(const char *algo, int enc, int secs,
1069                    struct cipher_speed_template *template,
1070                    unsigned int tcount, u8 *keysize, u32 num_mb)
1071 {
1072     struct test_mb_skcipher_data *data;
1073     struct crypto_skcipher *tfm;
1074     unsigned int i, j, iv_len;
1075     const int *b_size;
1076     const char *key;
1077     const char *e;
1078     char iv[128];
1079     int ret;
1080
1081     if (enc == ENCRYPT)
1082         e = "encryption";
1083     else
1084         e = "decryption";
1085
1086     data = kcalloc(num_mb, sizeof(*data), GFP_KERNEL);
1087     if (!data)
1088         return;
1089
1090     tfm = crypto_alloc_skcipher(algo, 0, 0);
1091     if (IS_ERR(tfm)) {
1092         pr_err("failed to load transform for %s: %ld\n",
1093             algo, PTR_ERR(tfm));
1094         goto out_free_data;
1095     }
1096
1097     for (i = 0; i < num_mb; ++i)
1098         if (testmgr_alloc_buf(data[i].xbuf)) {
1099             while (i--)
1100                 testmgr_free_buf(data[i].xbuf);
1101             goto out_free_tfm;
1102         }
1103
1104
1105     for (i = 0; i < num_mb; ++i)
1106         if (testmgr_alloc_buf(data[i].xbuf)) {
1107             while (i--)
1108                 testmgr_free_buf(data[i].xbuf);
1109             goto out_free_tfm;
1110         }
1111
1112
1113     for (i = 0; i < num_mb; ++i) {
1114         data[i].req = skcipher_request_alloc(tfm, GFP_KERNEL);
1115         if (!data[i].req) {
1116             pr_err("alg: skcipher: Failed to allocate request for %s\n",
1117                    algo);
1118             while (i--)
1119                 skcipher_request_free(data[i].req);
1120             goto out_free_xbuf;
1121         }
1122     }
1123
1124     for (i = 0; i < num_mb; ++i) {
1125         skcipher_request_set_callback(data[i].req,
1126                           CRYPTO_TFM_REQ_MAY_BACKLOG,
1127                           crypto_req_done, &data[i].wait);
1128         crypto_init_wait(&data[i].wait);
1129     }
1130
1131     pr_info("\ntesting speed of multibuffer %s (%s) %s\n", algo,
1132         get_driver_name(crypto_skcipher, tfm), e);
1133
1134     i = 0;
1135     do {
1136         b_size = block_sizes;
1137         do {
1138             u32 bs = round_up(*b_size, crypto_skcipher_blocksize(tfm));
1139
1140             if (bs > XBUFSIZE * PAGE_SIZE) {
1141                 pr_err("template (%u) too big for buffer (%lu)\n",
1142                        bs, XBUFSIZE * PAGE_SIZE);
1143                 goto out;
1144             }
1145
1146             pr_info("test %u (%d bit key, %d byte blocks): ", i,
1147                 *keysize * 8, bs);
1148
1149             /* Set up tfm global state, i.e. the key */
1150
1151             memset(tvmem[0], 0xff, PAGE_SIZE);
1152             key = tvmem[0];
1153             for (j = 0; j < tcount; j++) {
1154                 if (template[j].klen == *keysize) {
1155                     key = template[j].key;
1156                     break;
1157                 }
1158             }
1159
1160             crypto_skcipher_clear_flags(tfm, ~0);
1161
1162             ret = crypto_skcipher_setkey(tfm, key, *keysize);
1163             if (ret) {
1164                 pr_err("setkey() failed flags=%x\n",
1165                        crypto_skcipher_get_flags(tfm));
1166                 goto out;
1167             }
1168
1169             iv_len = crypto_skcipher_ivsize(tfm);
1170             if (iv_len)
1171                 memset(&iv, 0xff, iv_len);
1172
1173             /* Now setup per request stuff, i.e. buffers */
1174
1175             for (j = 0; j < num_mb; ++j) {
1176                 struct test_mb_skcipher_data *cur = &data[j];
1177                 unsigned int k = bs;
1178                 unsigned int pages = DIV_ROUND_UP(k, PAGE_SIZE);
1179                 unsigned int p = 0;
1180
1181                 sg_init_table(cur->sg, pages);
1182
1183                 while (k > PAGE_SIZE) {
1184                     sg_set_buf(cur->sg + p, cur->xbuf[p],
1185                            PAGE_SIZE);
1186                     memset(cur->xbuf[p], 0xff, PAGE_SIZE);
1187                     p++;
1188                     k -= PAGE_SIZE;
1189                 }
1190
1191                 sg_set_buf(cur->sg + p, cur->xbuf[p], k);
1192                 memset(cur->xbuf[p], 0xff, k);
1193
1194                 skcipher_request_set_crypt(cur->req, cur->sg,
1195                                cur->sg, bs, iv);
1196             }
1197
1198             if (secs) {
1199                 ret = test_mb_acipher_jiffies(data, enc,
1200                                   bs, secs,
1201                                   num_mb);
1202                 cond_resched();
1203             } else {
1204                 ret = test_mb_acipher_cycles(data, enc,
1205                                  bs, num_mb);
1206             }
1207
1208             if (ret) {
1209                 pr_err("%s() failed flags=%x\n", e,
1210                        crypto_skcipher_get_flags(tfm));
1211                 break;
1212             }
1213             b_size++;
1214             i++;
1215         } while (*b_size);
1216         keysize++;
1217     } while (*keysize);
1218
1219 out:
1220     for (i = 0; i < num_mb; ++i)
1221         skcipher_request_free(data[i].req);
1222 out_free_xbuf:
1223     for (i = 0; i < num_mb; ++i)
1224         testmgr_free_buf(data[i].xbuf);
1225 out_free_tfm:
1226     crypto_free_skcipher(tfm);
1227 out_free_data:
1228     kfree(data);
1229 }
1230
1231 static inline int do_one_acipher_op(struct skcipher_request *req, int ret)
1232 {
1233     struct crypto_wait *wait = req->base.data;
1234
1235     return crypto_wait_req(ret, wait);
1236 }
1237
1238 static int test_acipher_jiffies(struct skcipher_request *req, int enc,
1239                 int blen, int secs)
1240 {
1241     unsigned long start, end;
1242     int bcount;
1243     int ret;
1244
1245     for (start = jiffies, end = start + secs * HZ, bcount = 0;
1246          time_before(jiffies, end); bcount++) {
1247         if (enc)
1248             ret = do_one_acipher_op(req,
1249                         crypto_skcipher_encrypt(req));
1250         else
1251             ret = do_one_acipher_op(req,
1252                         crypto_skcipher_decrypt(req));
1253
1254         if (ret)
1255             return ret;
1256     }
1257
1258     pr_cont("%d operations in %d seconds (%llu bytes)\n",
1259         bcount, secs, (u64)bcount * blen);
1260     return 0;
1261 }
1262
1263 static int test_acipher_cycles(struct skcipher_request *req, int enc,
1264                    int blen)
1265 {
1266     unsigned long cycles = 0;
1267     int ret = 0;
1268     int i;
1269
1270     /* Warm-up run. */
1271     for (i = 0; i < 4; i++) {
1272         if (enc)
1273             ret = do_one_acipher_op(req,
1274                         crypto_skcipher_encrypt(req));
1275         else
1276             ret = do_one_acipher_op(req,
1277                         crypto_skcipher_decrypt(req));
1278
1279         if (ret)
1280             goto out;
1281     }
1282
1283     /* The real thing. */
1284     for (i = 0; i < 8; i++) {
1285         cycles_t start, end;
1286
1287         start = get_cycles();
1288         if (enc)
1289             ret = do_one_acipher_op(req,
1290                         crypto_skcipher_encrypt(req));
1291         else
1292             ret = do_one_acipher_op(req,
1293                         crypto_skcipher_decrypt(req));
1294         end = get_cycles();
1295
1296         if (ret)
1297             goto out;
1298
1299         cycles += end - start;
1300     }
1301
1302 out:
1303     if (ret == 0)
1304         pr_cont("1 operation in %lu cycles (%d bytes)\n",
1305             (cycles + 4) / 8, blen);
1306
1307     return ret;
1308 }
1309
1310 static void test_skcipher_speed(const char *algo, int enc, unsigned int secs,
1311                 struct cipher_speed_template *template,
1312                 unsigned int tcount, u8 *keysize, bool async)
1313 {
1314     unsigned int ret, i, j, k, iv_len;
1315     struct crypto_wait wait;
1316     const char *key;
1317     char iv[128];
1318     struct skcipher_request *req;
1319     struct crypto_skcipher *tfm;
1320     const int *b_size;
1321     const char *e;
1322
1323     if (enc == ENCRYPT)
1324         e = "encryption";
1325     else
1326         e = "decryption";
1327
1328     crypto_init_wait(&wait);
1329
1330     tfm = crypto_alloc_skcipher(algo, 0, async ? 0 : CRYPTO_ALG_ASYNC);
1331
1332     if (IS_ERR(tfm)) {
1333         pr_err("failed to load transform for %s: %ld\n", algo,
1334                PTR_ERR(tfm));
1335         return;
1336     }
1337
1338     pr_info("\ntesting speed of %s %s (%s) %s\n", async ? "async" : "sync",
1339         algo, get_driver_name(crypto_skcipher, tfm), e);
1340
1341     req = skcipher_request_alloc(tfm, GFP_KERNEL);
1342     if (!req) {
1343         pr_err("tcrypt: skcipher: Failed to allocate request for %s\n",
1344                algo);
1345         goto out;
1346     }
1347
1348     skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
1349                       crypto_req_done, &wait);
1350
1351     i = 0;
1352     do {
1353         b_size = block_sizes;
1354
1355         do {
1356             u32 bs = round_up(*b_size, crypto_skcipher_blocksize(tfm));
1357             struct scatterlist sg[TVMEMSIZE];
1358
1359             if ((*keysize + bs) > TVMEMSIZE * PAGE_SIZE) {
1360                 pr_err("template (%u) too big for "
1361                        "tvmem (%lu)\n", *keysize + bs,
1362                        TVMEMSIZE * PAGE_SIZE);
1363                 goto out_free_req;
1364             }
1365
1366             pr_info("test %u (%d bit key, %d byte blocks): ", i,
1367                 *keysize * 8, bs);
1368
1369             memset(tvmem[0], 0xff, PAGE_SIZE);
1370
1371             /* set key, plain text and IV */
1372             key = tvmem[0];
1373             for (j = 0; j < tcount; j++) {
1374                 if (template[j].klen == *keysize) {
1375                     key = template[j].key;
1376                     break;
1377                 }
1378             }
1379
1380             crypto_skcipher_clear_flags(tfm, ~0);
1381
1382             ret = crypto_skcipher_setkey(tfm, key, *keysize);
1383             if (ret) {
1384                 pr_err("setkey() failed flags=%x\n",
1385                     crypto_skcipher_get_flags(tfm));
1386                 goto out_free_req;
1387             }
1388
1389             k = *keysize + bs;
1390             sg_init_table(sg, DIV_ROUND_UP(k, PAGE_SIZE));
1391
1392             if (k > PAGE_SIZE) {
1393                 sg_set_buf(sg, tvmem[0] + *keysize,
1394                    PAGE_SIZE - *keysize);
1395                 k -= PAGE_SIZE;
1396                 j = 1;
1397                 while (k > PAGE_SIZE) {
1398                     sg_set_buf(sg + j, tvmem[j], PAGE_SIZE);
1399                     memset(tvmem[j], 0xff, PAGE_SIZE);
1400                     j++;
1401                     k -= PAGE_SIZE;
1402                 }
1403                 sg_set_buf(sg + j, tvmem[j], k);
1404                 memset(tvmem[j], 0xff, k);
1405             } else {
1406                 sg_set_buf(sg, tvmem[0] + *keysize, bs);
1407             }
1408
1409             iv_len = crypto_skcipher_ivsize(tfm);
1410             if (iv_len)
1411                 memset(&iv, 0xff, iv_len);
1412
1413             skcipher_request_set_crypt(req, sg, sg, bs, iv);
1414
1415             if (secs) {
1416                 ret = test_acipher_jiffies(req, enc,
1417                                bs, secs);
1418                 cond_resched();
1419             } else {
1420                 ret = test_acipher_cycles(req, enc,
1421                               bs);
1422             }
1423
1424             if (ret) {
1425                 pr_err("%s() failed flags=%x\n", e,
1426                        crypto_skcipher_get_flags(tfm));
1427                 break;
1428             }
1429             b_size++;
1430             i++;
1431         } while (*b_size);
1432         keysize++;
1433     } while (*keysize);
1434
1435 out_free_req:
1436     skcipher_request_free(req);
1437 out:
1438     crypto_free_skcipher(tfm);
1439 }
1440
1441 static void test_acipher_speed(const char *algo, int enc, unsigned int secs,
1442                    struct cipher_speed_template *template,
1443                    unsigned int tcount, u8 *keysize)
1444 {
1445     return test_skcipher_speed(algo, enc, secs, template, tcount, keysize,
1446                    true);
1447 }
1448
1449 static void test_cipher_speed(const char *algo, int enc, unsigned int secs,
1450                   struct cipher_speed_template *template,
1451                   unsigned int tcount, u8 *keysize)
1452 {
1453     return test_skcipher_speed(algo, enc, secs, template, tcount, keysize,
1454                    false);
1455 }
1456
1457 static void test_available(void)
1458 {
1459     const char **name = check;
1460
1461     while (*name) {
1462         printk("alg %s ", *name);
1463         printk(crypto_has_alg(*name, 0, 0) ?
1464                "found\n" : "not found\n");
1465         name++;
1466     }
1467 }
1468
1469 static inline int tcrypt_test(const char *alg)
1470 {
1471     int ret;
1472
1473     pr_debug("testing %s\n", alg);
1474
1475     ret = alg_test(alg, alg, 0, 0);
1476     /* non-fips algs return -EINVAL or -ECANCELED in fips mode */
1477     if (fips_enabled && (ret == -EINVAL || ret == -ECANCELED))
1478         ret = 0;
1479     return ret;
1480 }
1481
1482 static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb)
1483 {
1484     int i;
1485     int ret = 0;
1486
1487     switch (m) {
1488     case 0:
1489         if (alg) {
1490             if (!crypto_has_alg(alg, type,
1491                         mask ?: CRYPTO_ALG_TYPE_MASK))
1492                 ret = -ENOENT;
1493             break;
1494         }
1495
1496         for (i = 1; i < 200; i++)
1497             ret += do_test(NULL, 0, 0, i, num_mb);
1498         break;
1499
1500     case 1:
1501         ret += tcrypt_test("md5");
1502         break;
1503
1504     case 2:
1505         ret += tcrypt_test("sha1");
1506         break;
1507
1508     case 3:
1509         ret += tcrypt_test("ecb(des)");
1510         ret += tcrypt_test("cbc(des)");
1511         ret += tcrypt_test("ctr(des)");
1512         break;
1513
1514     case 4:
1515         ret += tcrypt_test("ecb(des3_ede)");
1516         ret += tcrypt_test("cbc(des3_ede)");
1517         ret += tcrypt_test("ctr(des3_ede)");
1518         break;
1519
1520     case 5:
1521         ret += tcrypt_test("md4");
1522         break;
1523
1524     case 6:
1525         ret += tcrypt_test("sha256");
1526         break;
1527
1528     case 7:
1529         ret += tcrypt_test("ecb(blowfish)");
1530         ret += tcrypt_test("cbc(blowfish)");
1531         ret += tcrypt_test("ctr(blowfish)");
1532         break;
1533
1534     case 8:
1535         ret += tcrypt_test("ecb(twofish)");
1536         ret += tcrypt_test("cbc(twofish)");
1537         ret += tcrypt_test("ctr(twofish)");
1538         ret += tcrypt_test("lrw(twofish)");
1539         ret += tcrypt_test("xts(twofish)");
1540         break;
1541
1542     case 9:
1543         ret += tcrypt_test("ecb(serpent)");
1544         ret += tcrypt_test("cbc(serpent)");
1545         ret += tcrypt_test("ctr(serpent)");
1546         ret += tcrypt_test("lrw(serpent)");
1547         ret += tcrypt_test("xts(serpent)");
1548         break;
1549
1550     case 10:
1551         ret += tcrypt_test("ecb(aes)");
1552         ret += tcrypt_test("cbc(aes)");
1553         ret += tcrypt_test("lrw(aes)");
1554         ret += tcrypt_test("xts(aes)");
1555         ret += tcrypt_test("ctr(aes)");
1556         ret += tcrypt_test("rfc3686(ctr(aes))");
1557         ret += tcrypt_test("ofb(aes)");
1558         ret += tcrypt_test("cfb(aes)");
1559         ret += tcrypt_test("xctr(aes)");
1560         break;
1561
1562     case 11:
1563         ret += tcrypt_test("sha384");
1564         break;
1565
1566     case 12:
1567         ret += tcrypt_test("sha512");
1568         break;
1569
1570     case 13:
1571         ret += tcrypt_test("deflate");
1572         break;
1573
1574     case 14:
1575         ret += tcrypt_test("ecb(cast5)");
1576         ret += tcrypt_test("cbc(cast5)");
1577         ret += tcrypt_test("ctr(cast5)");
1578         break;
1579
1580     case 15:
1581         ret += tcrypt_test("ecb(cast6)");
1582         ret += tcrypt_test("cbc(cast6)");
1583         ret += tcrypt_test("ctr(cast6)");
1584         ret += tcrypt_test("lrw(cast6)");
1585         ret += tcrypt_test("xts(cast6)");
1586         break;
1587
1588     case 16:
1589         ret += tcrypt_test("ecb(arc4)");
1590         break;
1591
1592     case 17:
1593         ret += tcrypt_test("michael_mic");
1594         break;
1595
1596     case 18:
1597         ret += tcrypt_test("crc32c");
1598         break;
1599
1600     case 19:
1601         ret += tcrypt_test("ecb(tea)");
1602         break;
1603
1604     case 20:
1605         ret += tcrypt_test("ecb(xtea)");
1606         break;
1607
1608     case 21:
1609         ret += tcrypt_test("ecb(khazad)");
1610         break;
1611
1612     case 22:
1613         ret += tcrypt_test("wp512");
1614         break;
1615
1616     case 23:
1617         ret += tcrypt_test("wp384");
1618         break;
1619
1620     case 24:
1621         ret += tcrypt_test("wp256");
1622         break;
1623
1624     case 26:
1625         ret += tcrypt_test("ecb(anubis)");
1626         ret += tcrypt_test("cbc(anubis)");
1627         break;
1628
1629     case 30:
1630         ret += tcrypt_test("ecb(xeta)");
1631         break;
1632
1633     case 31:
1634         ret += tcrypt_test("pcbc(fcrypt)");
1635         break;
1636
1637     case 32:
1638         ret += tcrypt_test("ecb(camellia)");
1639         ret += tcrypt_test("cbc(camellia)");
1640         ret += tcrypt_test("ctr(camellia)");
1641         ret += tcrypt_test("lrw(camellia)");
1642         ret += tcrypt_test("xts(camellia)");
1643         break;
1644
1645     case 33:
1646         ret += tcrypt_test("sha224");
1647         break;
1648
1649     case 35:
1650         ret += tcrypt_test("gcm(aes)");
1651         break;
1652
1653     case 36:
1654         ret += tcrypt_test("lzo");
1655         break;
1656
1657     case 37:
1658         ret += tcrypt_test("ccm(aes)");
1659         break;
1660
1661     case 38:
1662         ret += tcrypt_test("cts(cbc(aes))");
1663         break;
1664
1665         case 39:
1666         ret += tcrypt_test("xxhash64");
1667         break;
1668
1669         case 40:
1670         ret += tcrypt_test("rmd160");
1671         break;
1672
1673     case 42:
1674         ret += tcrypt_test("blake2b-512");
1675         break;
1676
1677     case 43:
1678         ret += tcrypt_test("ecb(seed)");
1679         break;
1680
1681     case 45:
1682         ret += tcrypt_test("rfc4309(ccm(aes))");
1683         break;
1684
1685     case 46:
1686         ret += tcrypt_test("ghash");
1687         break;
1688
1689     case 47:
1690         ret += tcrypt_test("crct10dif");
1691         break;
1692
1693     case 48:
1694         ret += tcrypt_test("sha3-224");
1695         break;
1696
1697     case 49:
1698         ret += tcrypt_test("sha3-256");
1699         break;
1700
1701     case 50:
1702         ret += tcrypt_test("sha3-384");
1703         break;
1704
1705     case 51:
1706         ret += tcrypt_test("sha3-512");
1707         break;
1708
1709     case 52:
1710         ret += tcrypt_test("sm3");
1711         break;
1712
1713     case 53:
1714         ret += tcrypt_test("streebog256");
1715         break;
1716
1717     case 54:
1718         ret += tcrypt_test("streebog512");
1719         break;
1720
1721     case 55:
1722         ret += tcrypt_test("gcm(sm4)");
1723         break;
1724
1725     case 56:
1726         ret += tcrypt_test("ccm(sm4)");
1727         break;
1728
1729     case 57:
1730         ret += tcrypt_test("polyval");
1731         break;
1732
1733     case 58:
1734         ret += tcrypt_test("gcm(aria)");
1735         break;
1736
1737     case 100:
1738         ret += tcrypt_test("hmac(md5)");
1739         break;
1740
1741     case 101:
1742         ret += tcrypt_test("hmac(sha1)");
1743         break;
1744
1745     case 102:
1746         ret += tcrypt_test("hmac(sha256)");
1747         break;
1748
1749     case 103:
1750         ret += tcrypt_test("hmac(sha384)");
1751         break;
1752
1753     case 104:
1754         ret += tcrypt_test("hmac(sha512)");
1755         break;
1756
1757     case 105:
1758         ret += tcrypt_test("hmac(sha224)");
1759         break;
1760
1761     case 106:
1762         ret += tcrypt_test("xcbc(aes)");
1763         break;
1764
1765     case 108:
1766         ret += tcrypt_test("hmac(rmd160)");
1767         break;
1768
1769     case 109:
1770         ret += tcrypt_test("vmac64(aes)");
1771         break;
1772
1773     case 111:
1774         ret += tcrypt_test("hmac(sha3-224)");
1775         break;
1776
1777     case 112:
1778         ret += tcrypt_test("hmac(sha3-256)");
1779         break;
1780
1781     case 113:
1782         ret += tcrypt_test("hmac(sha3-384)");
1783         break;
1784
1785     case 114:
1786         ret += tcrypt_test("hmac(sha3-512)");
1787         break;
1788
1789     case 115:
1790         ret += tcrypt_test("hmac(streebog256)");
1791         break;
1792
1793     case 116:
1794         ret += tcrypt_test("hmac(streebog512)");
1795         break;
1796
1797     case 150:
1798         ret += tcrypt_test("ansi_cprng");
1799         break;
1800
1801     case 151:
1802         ret += tcrypt_test("rfc4106(gcm(aes))");
1803         break;
1804
1805     case 152:
1806         ret += tcrypt_test("rfc4543(gcm(aes))");
1807         break;
1808
1809     case 153:
1810         ret += tcrypt_test("cmac(aes)");
1811         break;
1812
1813     case 154:
1814         ret += tcrypt_test("cmac(des3_ede)");
1815         break;
1816
1817     case 155:
1818         ret += tcrypt_test("authenc(hmac(sha1),cbc(aes))");
1819         break;
1820
1821     case 156:
1822         ret += tcrypt_test("authenc(hmac(md5),ecb(cipher_null))");
1823         break;
1824
1825     case 157:
1826         ret += tcrypt_test("authenc(hmac(sha1),ecb(cipher_null))");
1827         break;
1828
1829     case 158:
1830         ret += tcrypt_test("cbcmac(sm4)");
1831         break;
1832
1833     case 159:
1834         ret += tcrypt_test("cmac(sm4)");
1835         break;
1836
1837     case 181:
1838         ret += tcrypt_test("authenc(hmac(sha1),cbc(des))");
1839         break;
1840     case 182:
1841         ret += tcrypt_test("authenc(hmac(sha1),cbc(des3_ede))");
1842         break;
1843     case 183:
1844         ret += tcrypt_test("authenc(hmac(sha224),cbc(des))");
1845         break;
1846     case 184:
1847         ret += tcrypt_test("authenc(hmac(sha224),cbc(des3_ede))");
1848         break;
1849     case 185:
1850         ret += tcrypt_test("authenc(hmac(sha256),cbc(des))");
1851         break;
1852     case 186:
1853         ret += tcrypt_test("authenc(hmac(sha256),cbc(des3_ede))");
1854         break;
1855     case 187:
1856         ret += tcrypt_test("authenc(hmac(sha384),cbc(des))");
1857         break;
1858     case 188:
1859         ret += tcrypt_test("authenc(hmac(sha384),cbc(des3_ede))");
1860         break;
1861     case 189:
1862         ret += tcrypt_test("authenc(hmac(sha512),cbc(des))");
1863         break;
1864     case 190:
1865         ret += tcrypt_test("authenc(hmac(sha512),cbc(des3_ede))");
1866         break;
1867     case 191:
1868         ret += tcrypt_test("ecb(sm4)");
1869         ret += tcrypt_test("cbc(sm4)");
1870         ret += tcrypt_test("cfb(sm4)");
1871         ret += tcrypt_test("ctr(sm4)");
1872         break;
1873     case 192:
1874         ret += tcrypt_test("ecb(aria)");
1875         ret += tcrypt_test("cbc(aria)");
1876         ret += tcrypt_test("cfb(aria)");
1877         ret += tcrypt_test("ctr(aria)");
1878         break;
1879     case 200:
1880         test_cipher_speed("ecb(aes)", ENCRYPT, sec, NULL, 0,
1881                 speed_template_16_24_32);
1882         test_cipher_speed("ecb(aes)", DECRYPT, sec, NULL, 0,
1883                 speed_template_16_24_32);
1884         test_cipher_speed("cbc(aes)", ENCRYPT, sec, NULL, 0,
1885                 speed_template_16_24_32);
1886         test_cipher_speed("cbc(aes)", DECRYPT, sec, NULL, 0,
1887                 speed_template_16_24_32);
1888         test_cipher_speed("lrw(aes)", ENCRYPT, sec, NULL, 0,
1889                 speed_template_32_40_48);
1890         test_cipher_speed("lrw(aes)", DECRYPT, sec, NULL, 0,
1891                 speed_template_32_40_48);
1892         test_cipher_speed("xts(aes)", ENCRYPT, sec, NULL, 0,
1893                 speed_template_32_64);
1894         test_cipher_speed("xts(aes)", DECRYPT, sec, NULL, 0,
1895                 speed_template_32_64);
1896         test_cipher_speed("cts(cbc(aes))", ENCRYPT, sec, NULL, 0,
1897                 speed_template_16_24_32);
1898         test_cipher_speed("cts(cbc(aes))", DECRYPT, sec, NULL, 0,
1899                 speed_template_16_24_32);
1900         test_cipher_speed("ctr(aes)", ENCRYPT, sec, NULL, 0,
1901                 speed_template_16_24_32);
1902         test_cipher_speed("ctr(aes)", DECRYPT, sec, NULL, 0,
1903                 speed_template_16_24_32);
1904         test_cipher_speed("cfb(aes)", ENCRYPT, sec, NULL, 0,
1905                 speed_template_16_24_32);
1906         test_cipher_speed("cfb(aes)", DECRYPT, sec, NULL, 0,
1907                 speed_template_16_24_32);
1908         break;
1909
1910     case 201:
1911         test_cipher_speed("ecb(des3_ede)", ENCRYPT, sec,
1912                 des3_speed_template, DES3_SPEED_VECTORS,
1913                 speed_template_24);
1914         test_cipher_speed("ecb(des3_ede)", DECRYPT, sec,
1915                 des3_speed_template, DES3_SPEED_VECTORS,
1916                 speed_template_24);
1917         test_cipher_speed("cbc(des3_ede)", ENCRYPT, sec,
1918                 des3_speed_template, DES3_SPEED_VECTORS,
1919                 speed_template_24);
1920         test_cipher_speed("cbc(des3_ede)", DECRYPT, sec,
1921                 des3_speed_template, DES3_SPEED_VECTORS,
1922                 speed_template_24);
1923         test_cipher_speed("ctr(des3_ede)", ENCRYPT, sec,
1924                 des3_speed_template, DES3_SPEED_VECTORS,
1925                 speed_template_24);
1926         test_cipher_speed("ctr(des3_ede)", DECRYPT, sec,
1927                 des3_speed_template, DES3_SPEED_VECTORS,
1928                 speed_template_24);
1929         break;
1930
1931     case 202:
1932         test_cipher_speed("ecb(twofish)", ENCRYPT, sec, NULL, 0,
1933                 speed_template_16_24_32);
1934         test_cipher_speed("ecb(twofish)", DECRYPT, sec, NULL, 0,
1935                 speed_template_16_24_32);
1936         test_cipher_speed("cbc(twofish)", ENCRYPT, sec, NULL, 0,
1937                 speed_template_16_24_32);
1938         test_cipher_speed("cbc(twofish)", DECRYPT, sec, NULL, 0,
1939                 speed_template_16_24_32);
1940         test_cipher_speed("ctr(twofish)", ENCRYPT, sec, NULL, 0,
1941                 speed_template_16_24_32);
1942         test_cipher_speed("ctr(twofish)", DECRYPT, sec, NULL, 0,
1943                 speed_template_16_24_32);
1944         test_cipher_speed("lrw(twofish)", ENCRYPT, sec, NULL, 0,
1945                 speed_template_32_40_48);
1946         test_cipher_speed("lrw(twofish)", DECRYPT, sec, NULL, 0,
1947                 speed_template_32_40_48);
1948         test_cipher_speed("xts(twofish)", ENCRYPT, sec, NULL, 0,
1949                 speed_template_32_48_64);
1950         test_cipher_speed("xts(twofish)", DECRYPT, sec, NULL, 0,
1951                 speed_template_32_48_64);
1952         break;
1953
1954     case 203:
1955         test_cipher_speed("ecb(blowfish)", ENCRYPT, sec, NULL, 0,
1956                   speed_template_8_32);
1957         test_cipher_speed("ecb(blowfish)", DECRYPT, sec, NULL, 0,
1958                   speed_template_8_32);
1959         test_cipher_speed("cbc(blowfish)", ENCRYPT, sec, NULL, 0,
1960                   speed_template_8_32);
1961         test_cipher_speed("cbc(blowfish)", DECRYPT, sec, NULL, 0,
1962                   speed_template_8_32);
1963         test_cipher_speed("ctr(blowfish)", ENCRYPT, sec, NULL, 0,
1964                   speed_template_8_32);
1965         test_cipher_speed("ctr(blowfish)", DECRYPT, sec, NULL, 0,
1966                   speed_template_8_32);
1967         break;
1968
1969     case 204:
1970         test_cipher_speed("ecb(des)", ENCRYPT, sec, NULL, 0,
1971                   speed_template_8);
1972         test_cipher_speed("ecb(des)", DECRYPT, sec, NULL, 0,
1973                   speed_template_8);
1974         test_cipher_speed("cbc(des)", ENCRYPT, sec, NULL, 0,
1975                   speed_template_8);
1976         test_cipher_speed("cbc(des)", DECRYPT, sec, NULL, 0,
1977                   speed_template_8);
1978         break;
1979
1980     case 205:
1981         test_cipher_speed("ecb(camellia)", ENCRYPT, sec, NULL, 0,
1982                 speed_template_16_24_32);
1983         test_cipher_speed("ecb(camellia)", DECRYPT, sec, NULL, 0,
1984                 speed_template_16_24_32);
1985         test_cipher_speed("cbc(camellia)", ENCRYPT, sec, NULL, 0,
1986                 speed_template_16_24_32);
1987         test_cipher_speed("cbc(camellia)", DECRYPT, sec, NULL, 0,
1988                 speed_template_16_24_32);
1989         test_cipher_speed("ctr(camellia)", ENCRYPT, sec, NULL, 0,
1990                 speed_template_16_24_32);
1991         test_cipher_speed("ctr(camellia)", DECRYPT, sec, NULL, 0,
1992                 speed_template_16_24_32);
1993         test_cipher_speed("lrw(camellia)", ENCRYPT, sec, NULL, 0,
1994                 speed_template_32_40_48);
1995         test_cipher_speed("lrw(camellia)", DECRYPT, sec, NULL, 0,
1996                 speed_template_32_40_48);
1997         test_cipher_speed("xts(camellia)", ENCRYPT, sec, NULL, 0,
1998                 speed_template_32_48_64);
1999         test_cipher_speed("xts(camellia)", DECRYPT, sec, NULL, 0,
2000                 speed_template_32_48_64);
2001         break;
2002
2003     case 207:
2004         test_cipher_speed("ecb(serpent)", ENCRYPT, sec, NULL, 0,
2005                   speed_template_16_32);
2006         test_cipher_speed("ecb(serpent)", DECRYPT, sec, NULL, 0,
2007                   speed_template_16_32);
2008         test_cipher_speed("cbc(serpent)", ENCRYPT, sec, NULL, 0,
2009                   speed_template_16_32);
2010         test_cipher_speed("cbc(serpent)", DECRYPT, sec, NULL, 0,
2011                   speed_template_16_32);
2012         test_cipher_speed("ctr(serpent)", ENCRYPT, sec, NULL, 0,
2013                   speed_template_16_32);
2014         test_cipher_speed("ctr(serpent)", DECRYPT, sec, NULL, 0,
2015                   speed_template_16_32);
2016         test_cipher_speed("lrw(serpent)", ENCRYPT, sec, NULL, 0,
2017                   speed_template_32_48);
2018         test_cipher_speed("lrw(serpent)", DECRYPT, sec, NULL, 0,
2019                   speed_template_32_48);
2020         test_cipher_speed("xts(serpent)", ENCRYPT, sec, NULL, 0,
2021                   speed_template_32_64);
2022         test_cipher_speed("xts(serpent)", DECRYPT, sec, NULL, 0,
2023                   speed_template_32_64);
2024         break;
2025
2026     case 208:
2027         test_cipher_speed("ecb(arc4)", ENCRYPT, sec, NULL, 0,
2028                   speed_template_8);
2029         break;
2030
2031     case 209:
2032         test_cipher_speed("ecb(cast5)", ENCRYPT, sec, NULL, 0,
2033                   speed_template_8_16);
2034         test_cipher_speed("ecb(cast5)", DECRYPT, sec, NULL, 0,
2035                   speed_template_8_16);
2036         test_cipher_speed("cbc(cast5)", ENCRYPT, sec, NULL, 0,
2037                   speed_template_8_16);
2038         test_cipher_speed("cbc(cast5)", DECRYPT, sec, NULL, 0,
2039                   speed_template_8_16);
2040         test_cipher_speed("ctr(cast5)", ENCRYPT, sec, NULL, 0,
2041                   speed_template_8_16);
2042         test_cipher_speed("ctr(cast5)", DECRYPT, sec, NULL, 0,
2043                   speed_template_8_16);
2044         break;
2045
2046     case 210:
2047         test_cipher_speed("ecb(cast6)", ENCRYPT, sec, NULL, 0,
2048                   speed_template_16_32);
2049         test_cipher_speed("ecb(cast6)", DECRYPT, sec, NULL, 0,
2050                   speed_template_16_32);
2051         test_cipher_speed("cbc(cast6)", ENCRYPT, sec, NULL, 0,
2052                   speed_template_16_32);
2053         test_cipher_speed("cbc(cast6)", DECRYPT, sec, NULL, 0,
2054                   speed_template_16_32);
2055         test_cipher_speed("ctr(cast6)", ENCRYPT, sec, NULL, 0,
2056                   speed_template_16_32);
2057         test_cipher_speed("ctr(cast6)", DECRYPT, sec, NULL, 0,
2058                   speed_template_16_32);
2059         test_cipher_speed("lrw(cast6)", ENCRYPT, sec, NULL, 0,
2060                   speed_template_32_48);
2061         test_cipher_speed("lrw(cast6)", DECRYPT, sec, NULL, 0,
2062                   speed_template_32_48);
2063         test_cipher_speed("xts(cast6)", ENCRYPT, sec, NULL, 0,
2064                   speed_template_32_64);
2065         test_cipher_speed("xts(cast6)", DECRYPT, sec, NULL, 0,
2066                   speed_template_32_64);
2067         break;
2068
2069     case 211:
2070         test_aead_speed("rfc4106(gcm(aes))", ENCRYPT, sec,
2071                 NULL, 0, 16, 16, aead_speed_template_20);
2072         test_aead_speed("gcm(aes)", ENCRYPT, sec,
2073                 NULL, 0, 16, 8, speed_template_16_24_32);
2074         test_aead_speed("rfc4106(gcm(aes))", DECRYPT, sec,
2075                 NULL, 0, 16, 16, aead_speed_template_20);
2076         test_aead_speed("gcm(aes)", DECRYPT, sec,
2077                 NULL, 0, 16, 8, speed_template_16_24_32);
2078         break;
2079
2080     case 212:
2081         test_aead_speed("rfc4309(ccm(aes))", ENCRYPT, sec,
2082                 NULL, 0, 16, 16, aead_speed_template_19);
2083         test_aead_speed("rfc4309(ccm(aes))", DECRYPT, sec,
2084                 NULL, 0, 16, 16, aead_speed_template_19);
2085         break;
2086
2087     case 213:
2088         test_aead_speed("rfc7539esp(chacha20,poly1305)", ENCRYPT, sec,
2089                 NULL, 0, 16, 8, aead_speed_template_36);
2090         test_aead_speed("rfc7539esp(chacha20,poly1305)", DECRYPT, sec,
2091                 NULL, 0, 16, 8, aead_speed_template_36);
2092         break;
2093
2094     case 214:
2095         test_cipher_speed("chacha20", ENCRYPT, sec, NULL, 0,
2096                   speed_template_32);
2097         break;
2098
2099     case 215:
2100         test_mb_aead_speed("rfc4106(gcm(aes))", ENCRYPT, sec, NULL,
2101                    0, 16, 16, aead_speed_template_20, num_mb);
2102         test_mb_aead_speed("gcm(aes)", ENCRYPT, sec, NULL, 0, 16, 8,
2103                    speed_template_16_24_32, num_mb);
2104         test_mb_aead_speed("rfc4106(gcm(aes))", DECRYPT, sec, NULL,
2105                    0, 16, 16, aead_speed_template_20, num_mb);
2106         test_mb_aead_speed("gcm(aes)", DECRYPT, sec, NULL, 0, 16, 8,
2107                    speed_template_16_24_32, num_mb);
2108         break;
2109
2110     case 216:
2111         test_mb_aead_speed("rfc4309(ccm(aes))", ENCRYPT, sec, NULL, 0,
2112                    16, 16, aead_speed_template_19, num_mb);
2113         test_mb_aead_speed("rfc4309(ccm(aes))", DECRYPT, sec, NULL, 0,
2114                    16, 16, aead_speed_template_19, num_mb);
2115         break;
2116
2117     case 217:
2118         test_mb_aead_speed("rfc7539esp(chacha20,poly1305)", ENCRYPT,
2119                    sec, NULL, 0, 16, 8, aead_speed_template_36,
2120                    num_mb);
2121         test_mb_aead_speed("rfc7539esp(chacha20,poly1305)", DECRYPT,
2122                    sec, NULL, 0, 16, 8, aead_speed_template_36,
2123                    num_mb);
2124         break;
2125
2126     case 218:
2127         test_cipher_speed("ecb(sm4)", ENCRYPT, sec, NULL, 0,
2128                 speed_template_16);
2129         test_cipher_speed("ecb(sm4)", DECRYPT, sec, NULL, 0,
2130                 speed_template_16);
2131         test_cipher_speed("cbc(sm4)", ENCRYPT, sec, NULL, 0,
2132                 speed_template_16);
2133         test_cipher_speed("cbc(sm4)", DECRYPT, sec, NULL, 0,
2134                 speed_template_16);
2135         test_cipher_speed("cfb(sm4)", ENCRYPT, sec, NULL, 0,
2136                 speed_template_16);
2137         test_cipher_speed("cfb(sm4)", DECRYPT, sec, NULL, 0,
2138                 speed_template_16);
2139         test_cipher_speed("ctr(sm4)", ENCRYPT, sec, NULL, 0,
2140                 speed_template_16);
2141         test_cipher_speed("ctr(sm4)", DECRYPT, sec, NULL, 0,
2142                 speed_template_16);
2143         break;
2144
2145     case 219:
2146         test_cipher_speed("adiantum(xchacha12,aes)", ENCRYPT, sec, NULL,
2147                   0, speed_template_32);
2148         test_cipher_speed("adiantum(xchacha12,aes)", DECRYPT, sec, NULL,
2149                   0, speed_template_32);
2150         test_cipher_speed("adiantum(xchacha20,aes)", ENCRYPT, sec, NULL,
2151                   0, speed_template_32);
2152         test_cipher_speed("adiantum(xchacha20,aes)", DECRYPT, sec, NULL,
2153                   0, speed_template_32);
2154         break;
2155
2156     case 220:
2157         test_acipher_speed("essiv(cbc(aes),sha256)",
2158                   ENCRYPT, sec, NULL, 0,
2159                   speed_template_16_24_32);
2160         test_acipher_speed("essiv(cbc(aes),sha256)",
2161                   DECRYPT, sec, NULL, 0,
2162                   speed_template_16_24_32);
2163         break;
2164
2165     case 221:
2166         test_aead_speed("aegis128", ENCRYPT, sec,
2167                 NULL, 0, 16, 8, speed_template_16);
2168         test_aead_speed("aegis128", DECRYPT, sec,
2169                 NULL, 0, 16, 8, speed_template_16);
2170         break;
2171
2172     case 222:
2173         test_aead_speed("gcm(sm4)", ENCRYPT, sec,
2174                 NULL, 0, 16, 8, speed_template_16);
2175         test_aead_speed("gcm(sm4)", DECRYPT, sec,
2176                 NULL, 0, 16, 8, speed_template_16);
2177         break;
2178
2179     case 223:
2180         test_aead_speed("rfc4309(ccm(sm4))", ENCRYPT, sec,
2181                 NULL, 0, 16, 16, aead_speed_template_19);
2182         test_aead_speed("rfc4309(ccm(sm4))", DECRYPT, sec,
2183                 NULL, 0, 16, 16, aead_speed_template_19);
2184         break;
2185
2186     case 224:
2187         test_mb_aead_speed("gcm(sm4)", ENCRYPT, sec, NULL, 0, 16, 8,
2188                    speed_template_16, num_mb);
2189         test_mb_aead_speed("gcm(sm4)", DECRYPT, sec, NULL, 0, 16, 8,
2190                    speed_template_16, num_mb);
2191         break;
2192
2193     case 225:
2194         test_mb_aead_speed("rfc4309(ccm(sm4))", ENCRYPT, sec, NULL, 0,
2195                    16, 16, aead_speed_template_19, num_mb);
2196         test_mb_aead_speed("rfc4309(ccm(sm4))", DECRYPT, sec, NULL, 0,
2197                    16, 16, aead_speed_template_19, num_mb);
2198         break;
2199
2200     case 226:
2201         test_cipher_speed("hctr2(aes)", ENCRYPT, sec, NULL,
2202                   0, speed_template_32);
2203         break;
2204
2205     case 227:
2206         test_cipher_speed("ecb(aria)", ENCRYPT, sec, NULL, 0,
2207                   speed_template_16_24_32);
2208         test_cipher_speed("ecb(aria)", DECRYPT, sec, NULL, 0,
2209                   speed_template_16_24_32);
2210         test_cipher_speed("cbc(aria)", ENCRYPT, sec, NULL, 0,
2211                   speed_template_16_24_32);
2212         test_cipher_speed("cbc(aria)", DECRYPT, sec, NULL, 0,
2213                   speed_template_16_24_32);
2214         test_cipher_speed("cfb(aria)", ENCRYPT, sec, NULL, 0,
2215                   speed_template_16_24_32);
2216         test_cipher_speed("cfb(aria)", DECRYPT, sec, NULL, 0,
2217                   speed_template_16_24_32);
2218         test_cipher_speed("ctr(aria)", ENCRYPT, sec, NULL, 0,
2219                   speed_template_16_24_32);
2220         test_cipher_speed("ctr(aria)", DECRYPT, sec, NULL, 0,
2221                   speed_template_16_24_32);
2222         break;
2223
2224     case 228:
2225         test_aead_speed("gcm(aria)", ENCRYPT, sec,
2226                 NULL, 0, 16, 8, speed_template_16_24_32);
2227         test_aead_speed("gcm(aria)", DECRYPT, sec,
2228                 NULL, 0, 16, 8, speed_template_16_24_32);
2229         break;
2230
2231     case 300:
2232         if (alg) {
2233             test_hash_speed(alg, sec, generic_hash_speed_template);
2234             break;
2235         }
2236         fallthrough;
2237     case 301:
2238         test_hash_speed("md4", sec, generic_hash_speed_template);
2239         if (mode > 300 && mode < 400) break;
2240         fallthrough;
2241     case 302:
2242         test_hash_speed("md5", sec, generic_hash_speed_template);
2243         if (mode > 300 && mode < 400) break;
2244         fallthrough;
2245     case 303:
2246         test_hash_speed("sha1", sec, generic_hash_speed_template);
2247         if (mode > 300 && mode < 400) break;
2248         fallthrough;
2249     case 304:
2250         test_hash_speed("sha256", sec, generic_hash_speed_template);
2251         if (mode > 300 && mode < 400) break;
2252         fallthrough;
2253     case 305:
2254         test_hash_speed("sha384", sec, generic_hash_speed_template);
2255         if (mode > 300 && mode < 400) break;
2256         fallthrough;
2257     case 306:
2258         test_hash_speed("sha512", sec, generic_hash_speed_template);
2259         if (mode > 300 && mode < 400) break;
2260         fallthrough;
2261     case 307:
2262         test_hash_speed("wp256", sec, generic_hash_speed_template);
2263         if (mode > 300 && mode < 400) break;
2264         fallthrough;
2265     case 308:
2266         test_hash_speed("wp384", sec, generic_hash_speed_template);
2267         if (mode > 300 && mode < 400) break;
2268         fallthrough;
2269     case 309:
2270         test_hash_speed("wp512", sec, generic_hash_speed_template);
2271         if (mode > 300 && mode < 400) break;
2272         fallthrough;
2273     case 313:
2274         test_hash_speed("sha224", sec, generic_hash_speed_template);
2275         if (mode > 300 && mode < 400) break;
2276         fallthrough;
2277     case 314:
2278         test_hash_speed("xxhash64", sec, generic_hash_speed_template);
2279         if (mode > 300 && mode < 400) break;
2280         fallthrough;
2281     case 315:
2282         test_hash_speed("rmd160", sec, generic_hash_speed_template);
2283         if (mode > 300 && mode < 400) break;
2284         fallthrough;
2285     case 317:
2286         test_hash_speed("blake2b-512", sec, generic_hash_speed_template);
2287         if (mode > 300 && mode < 400) break;
2288         fallthrough;
2289     case 318:
2290         klen = 16;
2291         test_hash_speed("ghash", sec, generic_hash_speed_template);
2292         if (mode > 300 && mode < 400) break;
2293         fallthrough;
2294     case 319:
2295         test_hash_speed("crc32c", sec, generic_hash_speed_template);
2296         if (mode > 300 && mode < 400) break;
2297         fallthrough;
2298     case 320:
2299         test_hash_speed("crct10dif", sec, generic_hash_speed_template);
2300         if (mode > 300 && mode < 400) break;
2301         fallthrough;
2302     case 321:
2303         test_hash_speed("poly1305", sec, poly1305_speed_template);
2304         if (mode > 300 && mode < 400) break;
2305         fallthrough;
2306     case 322:
2307         test_hash_speed("sha3-224", sec, generic_hash_speed_template);
2308         if (mode > 300 && mode < 400) break;
2309         fallthrough;
2310     case 323:
2311         test_hash_speed("sha3-256", sec, generic_hash_speed_template);
2312         if (mode > 300 && mode < 400) break;
2313         fallthrough;
2314     case 324:
2315         test_hash_speed("sha3-384", sec, generic_hash_speed_template);
2316         if (mode > 300 && mode < 400) break;
2317         fallthrough;
2318     case 325:
2319         test_hash_speed("sha3-512", sec, generic_hash_speed_template);
2320         if (mode > 300 && mode < 400) break;
2321         fallthrough;
2322     case 326:
2323         test_hash_speed("sm3", sec, generic_hash_speed_template);
2324         if (mode > 300 && mode < 400) break;
2325         fallthrough;
2326     case 327:
2327         test_hash_speed("streebog256", sec,
2328                 generic_hash_speed_template);
2329         if (mode > 300 && mode < 400) break;
2330         fallthrough;
2331     case 328:
2332         test_hash_speed("streebog512", sec,
2333                 generic_hash_speed_template);
2334         if (mode > 300 && mode < 400) break;
2335         fallthrough;
2336     case 399:
2337         break;
2338
2339     case 400:
2340         if (alg) {
2341             test_ahash_speed(alg, sec, generic_hash_speed_template);
2342             break;
2343         }
2344         fallthrough;
2345     case 401:
2346         test_ahash_speed("md4", sec, generic_hash_speed_template);
2347         if (mode > 400 && mode < 500) break;
2348         fallthrough;
2349     case 402:
2350         test_ahash_speed("md5", sec, generic_hash_speed_template);
2351         if (mode > 400 && mode < 500) break;
2352         fallthrough;
2353     case 403:
2354         test_ahash_speed("sha1", sec, generic_hash_speed_template);
2355         if (mode > 400 && mode < 500) break;
2356         fallthrough;
2357     case 404:
2358         test_ahash_speed("sha256", sec, generic_hash_speed_template);
2359         if (mode > 400 && mode < 500) break;
2360         fallthrough;
2361     case 405:
2362         test_ahash_speed("sha384", sec, generic_hash_speed_template);
2363         if (mode > 400 && mode < 500) break;
2364         fallthrough;
2365     case 406:
2366         test_ahash_speed("sha512", sec, generic_hash_speed_template);
2367         if (mode > 400 && mode < 500) break;
2368         fallthrough;
2369     case 407:
2370         test_ahash_speed("wp256", sec, generic_hash_speed_template);
2371         if (mode > 400 && mode < 500) break;
2372         fallthrough;
2373     case 408:
2374         test_ahash_speed("wp384", sec, generic_hash_speed_template);
2375         if (mode > 400 && mode < 500) break;
2376         fallthrough;
2377     case 409:
2378         test_ahash_speed("wp512", sec, generic_hash_speed_template);
2379         if (mode > 400 && mode < 500) break;
2380         fallthrough;
2381     case 413:
2382         test_ahash_speed("sha224", sec, generic_hash_speed_template);
2383         if (mode > 400 && mode < 500) break;
2384         fallthrough;
2385     case 414:
2386         test_ahash_speed("xxhash64", sec, generic_hash_speed_template);
2387         if (mode > 400 && mode < 500) break;
2388         fallthrough;
2389     case 415:
2390         test_ahash_speed("rmd160", sec, generic_hash_speed_template);
2391         if (mode > 400 && mode < 500) break;
2392         fallthrough;
2393     case 417:
2394         test_ahash_speed("blake2b-512", sec, generic_hash_speed_template);
2395         if (mode > 400 && mode < 500) break;
2396         fallthrough;
2397     case 418:
2398         test_ahash_speed("sha3-224", sec, generic_hash_speed_template);
2399         if (mode > 400 && mode < 500) break;
2400         fallthrough;
2401     case 419:
2402         test_ahash_speed("sha3-256", sec, generic_hash_speed_template);
2403         if (mode > 400 && mode < 500) break;
2404         fallthrough;
2405     case 420:
2406         test_ahash_speed("sha3-384", sec, generic_hash_speed_template);
2407         if (mode > 400 && mode < 500) break;
2408         fallthrough;
2409     case 421:
2410         test_ahash_speed("sha3-512", sec, generic_hash_speed_template);
2411         if (mode > 400 && mode < 500) break;
2412         fallthrough;
2413     case 422:
2414         test_ahash_speed("sm3", sec, generic_hash_speed_template);
2415         if (mode > 400 && mode < 500) break;
2416         fallthrough;
2417     case 499:
2418         break;
2419
2420     case 500:
2421         test_acipher_speed("ecb(aes)", ENCRYPT, sec, NULL, 0,
2422                    speed_template_16_24_32);
2423         test_acipher_speed("ecb(aes)", DECRYPT, sec, NULL, 0,
2424                    speed_template_16_24_32);
2425         test_acipher_speed("cbc(aes)", ENCRYPT, sec, NULL, 0,
2426                    speed_template_16_24_32);
2427         test_acipher_speed("cbc(aes)", DECRYPT, sec, NULL, 0,
2428                    speed_template_16_24_32);
2429         test_acipher_speed("lrw(aes)", ENCRYPT, sec, NULL, 0,
2430                    speed_template_32_40_48);
2431         test_acipher_speed("lrw(aes)", DECRYPT, sec, NULL, 0,
2432                    speed_template_32_40_48);
2433         test_acipher_speed("xts(aes)", ENCRYPT, sec, NULL, 0,
2434                    speed_template_32_64);
2435         test_acipher_speed("xts(aes)", DECRYPT, sec, NULL, 0,
2436                    speed_template_32_64);
2437         test_acipher_speed("cts(cbc(aes))", ENCRYPT, sec, NULL, 0,
2438                    speed_template_16_24_32);
2439         test_acipher_speed("cts(cbc(aes))", DECRYPT, sec, NULL, 0,
2440                    speed_template_16_24_32);
2441         test_acipher_speed("ctr(aes)", ENCRYPT, sec, NULL, 0,
2442                    speed_template_16_24_32);
2443         test_acipher_speed("ctr(aes)", DECRYPT, sec, NULL, 0,
2444                    speed_template_16_24_32);
2445         test_acipher_speed("cfb(aes)", ENCRYPT, sec, NULL, 0,
2446                    speed_template_16_24_32);
2447         test_acipher_speed("cfb(aes)", DECRYPT, sec, NULL, 0,
2448                    speed_template_16_24_32);
2449         test_acipher_speed("ofb(aes)", ENCRYPT, sec, NULL, 0,
2450                    speed_template_16_24_32);
2451         test_acipher_speed("ofb(aes)", DECRYPT, sec, NULL, 0,
2452                    speed_template_16_24_32);
2453         test_acipher_speed("rfc3686(ctr(aes))", ENCRYPT, sec, NULL, 0,
2454                    speed_template_20_28_36);
2455         test_acipher_speed("rfc3686(ctr(aes))", DECRYPT, sec, NULL, 0,
2456                    speed_template_20_28_36);
2457         break;
2458
2459     case 501:
2460         test_acipher_speed("ecb(des3_ede)", ENCRYPT, sec,
2461                    des3_speed_template, DES3_SPEED_VECTORS,
2462                    speed_template_24);
2463         test_acipher_speed("ecb(des3_ede)", DECRYPT, sec,
2464                    des3_speed_template, DES3_SPEED_VECTORS,
2465                    speed_template_24);
2466         test_acipher_speed("cbc(des3_ede)", ENCRYPT, sec,
2467                    des3_speed_template, DES3_SPEED_VECTORS,
2468                    speed_template_24);
2469         test_acipher_speed("cbc(des3_ede)", DECRYPT, sec,
2470                    des3_speed_template, DES3_SPEED_VECTORS,
2471                    speed_template_24);
2472         test_acipher_speed("cfb(des3_ede)", ENCRYPT, sec,
2473                    des3_speed_template, DES3_SPEED_VECTORS,
2474                    speed_template_24);
2475         test_acipher_speed("cfb(des3_ede)", DECRYPT, sec,
2476                    des3_speed_template, DES3_SPEED_VECTORS,
2477                    speed_template_24);
2478         test_acipher_speed("ofb(des3_ede)", ENCRYPT, sec,
2479                    des3_speed_template, DES3_SPEED_VECTORS,
2480                    speed_template_24);
2481         test_acipher_speed("ofb(des3_ede)", DECRYPT, sec,
2482                    des3_speed_template, DES3_SPEED_VECTORS,
2483                    speed_template_24);
2484         break;
2485
2486     case 502:
2487         test_acipher_speed("ecb(des)", ENCRYPT, sec, NULL, 0,
2488                    speed_template_8);
2489         test_acipher_speed("ecb(des)", DECRYPT, sec, NULL, 0,
2490                    speed_template_8);
2491         test_acipher_speed("cbc(des)", ENCRYPT, sec, NULL, 0,
2492                    speed_template_8);
2493         test_acipher_speed("cbc(des)", DECRYPT, sec, NULL, 0,
2494                    speed_template_8);
2495         test_acipher_speed("cfb(des)", ENCRYPT, sec, NULL, 0,
2496                    speed_template_8);
2497         test_acipher_speed("cfb(des)", DECRYPT, sec, NULL, 0,
2498                    speed_template_8);
2499         test_acipher_speed("ofb(des)", ENCRYPT, sec, NULL, 0,
2500                    speed_template_8);
2501         test_acipher_speed("ofb(des)", DECRYPT, sec, NULL, 0,
2502                    speed_template_8);
2503         break;
2504
2505     case 503:
2506         test_acipher_speed("ecb(serpent)", ENCRYPT, sec, NULL, 0,
2507                    speed_template_16_32);
2508         test_acipher_speed("ecb(serpent)", DECRYPT, sec, NULL, 0,
2509                    speed_template_16_32);
2510         test_acipher_speed("cbc(serpent)", ENCRYPT, sec, NULL, 0,
2511                    speed_template_16_32);
2512         test_acipher_speed("cbc(serpent)", DECRYPT, sec, NULL, 0,
2513                    speed_template_16_32);
2514         test_acipher_speed("ctr(serpent)", ENCRYPT, sec, NULL, 0,
2515                    speed_template_16_32);
2516         test_acipher_speed("ctr(serpent)", DECRYPT, sec, NULL, 0,
2517                    speed_template_16_32);
2518         test_acipher_speed("lrw(serpent)", ENCRYPT, sec, NULL, 0,
2519                    speed_template_32_48);
2520         test_acipher_speed("lrw(serpent)", DECRYPT, sec, NULL, 0,
2521                    speed_template_32_48);
2522         test_acipher_speed("xts(serpent)", ENCRYPT, sec, NULL, 0,
2523                    speed_template_32_64);
2524         test_acipher_speed("xts(serpent)", DECRYPT, sec, NULL, 0,
2525                    speed_template_32_64);
2526         break;
2527
2528     case 504:
2529         test_acipher_speed("ecb(twofish)", ENCRYPT, sec, NULL, 0,
2530                    speed_template_16_24_32);
2531         test_acipher_speed("ecb(twofish)", DECRYPT, sec, NULL, 0,
2532                    speed_template_16_24_32);
2533         test_acipher_speed("cbc(twofish)", ENCRYPT, sec, NULL, 0,
2534                    speed_template_16_24_32);
2535         test_acipher_speed("cbc(twofish)", DECRYPT, sec, NULL, 0,
2536                    speed_template_16_24_32);
2537         test_acipher_speed("ctr(twofish)", ENCRYPT, sec, NULL, 0,
2538                    speed_template_16_24_32);
2539         test_acipher_speed("ctr(twofish)", DECRYPT, sec, NULL, 0,
2540                    speed_template_16_24_32);
2541         test_acipher_speed("lrw(twofish)", ENCRYPT, sec, NULL, 0,
2542                    speed_template_32_40_48);
2543         test_acipher_speed("lrw(twofish)", DECRYPT, sec, NULL, 0,
2544                    speed_template_32_40_48);
2545         test_acipher_speed("xts(twofish)", ENCRYPT, sec, NULL, 0,
2546                    speed_template_32_48_64);
2547         test_acipher_speed("xts(twofish)", DECRYPT, sec, NULL, 0,
2548                    speed_template_32_48_64);
2549         break;
2550
2551     case 505:
2552         test_acipher_speed("ecb(arc4)", ENCRYPT, sec, NULL, 0,
2553                    speed_template_8);
2554         break;
2555
2556     case 506:
2557         test_acipher_speed("ecb(cast5)", ENCRYPT, sec, NULL, 0,
2558                    speed_template_8_16);
2559         test_acipher_speed("ecb(cast5)", DECRYPT, sec, NULL, 0,
2560                    speed_template_8_16);
2561         test_acipher_speed("cbc(cast5)", ENCRYPT, sec, NULL, 0,
2562                    speed_template_8_16);
2563         test_acipher_speed("cbc(cast5)", DECRYPT, sec, NULL, 0,
2564                    speed_template_8_16);
2565         test_acipher_speed("ctr(cast5)", ENCRYPT, sec, NULL, 0,
2566                    speed_template_8_16);
2567         test_acipher_speed("ctr(cast5)", DECRYPT, sec, NULL, 0,
2568                    speed_template_8_16);
2569         break;
2570
2571     case 507:
2572         test_acipher_speed("ecb(cast6)", ENCRYPT, sec, NULL, 0,
2573                    speed_template_16_32);
2574         test_acipher_speed("ecb(cast6)", DECRYPT, sec, NULL, 0,
2575                    speed_template_16_32);
2576         test_acipher_speed("cbc(cast6)", ENCRYPT, sec, NULL, 0,
2577                    speed_template_16_32);
2578         test_acipher_speed("cbc(cast6)", DECRYPT, sec, NULL, 0,
2579                    speed_template_16_32);
2580         test_acipher_speed("ctr(cast6)", ENCRYPT, sec, NULL, 0,
2581                    speed_template_16_32);
2582         test_acipher_speed("ctr(cast6)", DECRYPT, sec, NULL, 0,
2583                    speed_template_16_32);
2584         test_acipher_speed("lrw(cast6)", ENCRYPT, sec, NULL, 0,
2585                    speed_template_32_48);
2586         test_acipher_speed("lrw(cast6)", DECRYPT, sec, NULL, 0,
2587                    speed_template_32_48);
2588         test_acipher_speed("xts(cast6)", ENCRYPT, sec, NULL, 0,
2589                    speed_template_32_64);
2590         test_acipher_speed("xts(cast6)", DECRYPT, sec, NULL, 0,
2591                    speed_template_32_64);
2592         break;
2593
2594     case 508:
2595         test_acipher_speed("ecb(camellia)", ENCRYPT, sec, NULL, 0,
2596                    speed_template_16_32);
2597         test_acipher_speed("ecb(camellia)", DECRYPT, sec, NULL, 0,
2598                    speed_template_16_32);
2599         test_acipher_speed("cbc(camellia)", ENCRYPT, sec, NULL, 0,
2600                    speed_template_16_32);
2601         test_acipher_speed("cbc(camellia)", DECRYPT, sec, NULL, 0,
2602                    speed_template_16_32);
2603         test_acipher_speed("ctr(camellia)", ENCRYPT, sec, NULL, 0,
2604                    speed_template_16_32);
2605         test_acipher_speed("ctr(camellia)", DECRYPT, sec, NULL, 0,
2606                    speed_template_16_32);
2607         test_acipher_speed("lrw(camellia)", ENCRYPT, sec, NULL, 0,
2608                    speed_template_32_48);
2609         test_acipher_speed("lrw(camellia)", DECRYPT, sec, NULL, 0,
2610                    speed_template_32_48);
2611         test_acipher_speed("xts(camellia)", ENCRYPT, sec, NULL, 0,
2612                    speed_template_32_64);
2613         test_acipher_speed("xts(camellia)", DECRYPT, sec, NULL, 0,
2614                    speed_template_32_64);
2615         break;
2616
2617     case 509:
2618         test_acipher_speed("ecb(blowfish)", ENCRYPT, sec, NULL, 0,
2619                    speed_template_8_32);
2620         test_acipher_speed("ecb(blowfish)", DECRYPT, sec, NULL, 0,
2621                    speed_template_8_32);
2622         test_acipher_speed("cbc(blowfish)", ENCRYPT, sec, NULL, 0,
2623                    speed_template_8_32);
2624         test_acipher_speed("cbc(blowfish)", DECRYPT, sec, NULL, 0,
2625                    speed_template_8_32);
2626         test_acipher_speed("ctr(blowfish)", ENCRYPT, sec, NULL, 0,
2627                    speed_template_8_32);
2628         test_acipher_speed("ctr(blowfish)", DECRYPT, sec, NULL, 0,
2629                    speed_template_8_32);
2630         break;
2631
2632     case 518:
2633         test_acipher_speed("ecb(sm4)", ENCRYPT, sec, NULL, 0,
2634                 speed_template_16);
2635         test_acipher_speed("ecb(sm4)", DECRYPT, sec, NULL, 0,
2636                 speed_template_16);
2637         test_acipher_speed("cbc(sm4)", ENCRYPT, sec, NULL, 0,
2638                 speed_template_16);
2639         test_acipher_speed("cbc(sm4)", DECRYPT, sec, NULL, 0,
2640                 speed_template_16);
2641         test_acipher_speed("cfb(sm4)", ENCRYPT, sec, NULL, 0,
2642                 speed_template_16);
2643         test_acipher_speed("cfb(sm4)", DECRYPT, sec, NULL, 0,
2644                 speed_template_16);
2645         test_acipher_speed("ctr(sm4)", ENCRYPT, sec, NULL, 0,
2646                 speed_template_16);
2647         test_acipher_speed("ctr(sm4)", DECRYPT, sec, NULL, 0,
2648                 speed_template_16);
2649         break;
2650
2651     case 600:
2652         test_mb_skcipher_speed("ecb(aes)", ENCRYPT, sec, NULL, 0,
2653                        speed_template_16_24_32, num_mb);
2654         test_mb_skcipher_speed("ecb(aes)", DECRYPT, sec, NULL, 0,
2655                        speed_template_16_24_32, num_mb);
2656         test_mb_skcipher_speed("cbc(aes)", ENCRYPT, sec, NULL, 0,
2657                        speed_template_16_24_32, num_mb);
2658         test_mb_skcipher_speed("cbc(aes)", DECRYPT, sec, NULL, 0,
2659                        speed_template_16_24_32, num_mb);
2660         test_mb_skcipher_speed("lrw(aes)", ENCRYPT, sec, NULL, 0,
2661                        speed_template_32_40_48, num_mb);
2662         test_mb_skcipher_speed("lrw(aes)", DECRYPT, sec, NULL, 0,
2663                        speed_template_32_40_48, num_mb);
2664         test_mb_skcipher_speed("xts(aes)", ENCRYPT, sec, NULL, 0,
2665                        speed_template_32_64, num_mb);
2666         test_mb_skcipher_speed("xts(aes)", DECRYPT, sec, NULL, 0,
2667                        speed_template_32_64, num_mb);
2668         test_mb_skcipher_speed("cts(cbc(aes))", ENCRYPT, sec, NULL, 0,
2669                        speed_template_16_24_32, num_mb);
2670         test_mb_skcipher_speed("cts(cbc(aes))", DECRYPT, sec, NULL, 0,
2671                        speed_template_16_24_32, num_mb);
2672         test_mb_skcipher_speed("ctr(aes)", ENCRYPT, sec, NULL, 0,
2673                        speed_template_16_24_32, num_mb);
2674         test_mb_skcipher_speed("ctr(aes)", DECRYPT, sec, NULL, 0,
2675                        speed_template_16_24_32, num_mb);
2676         test_mb_skcipher_speed("cfb(aes)", ENCRYPT, sec, NULL, 0,
2677                        speed_template_16_24_32, num_mb);
2678         test_mb_skcipher_speed("cfb(aes)", DECRYPT, sec, NULL, 0,
2679                        speed_template_16_24_32, num_mb);
2680         test_mb_skcipher_speed("ofb(aes)", ENCRYPT, sec, NULL, 0,
2681                        speed_template_16_24_32, num_mb);
2682         test_mb_skcipher_speed("ofb(aes)", DECRYPT, sec, NULL, 0,
2683                        speed_template_16_24_32, num_mb);
2684         test_mb_skcipher_speed("rfc3686(ctr(aes))", ENCRYPT, sec, NULL,
2685                        0, speed_template_20_28_36, num_mb);
2686         test_mb_skcipher_speed("rfc3686(ctr(aes))", DECRYPT, sec, NULL,
2687                        0, speed_template_20_28_36, num_mb);
2688         break;
2689
2690     case 601:
2691         test_mb_skcipher_speed("ecb(des3_ede)", ENCRYPT, sec,
2692                        des3_speed_template, DES3_SPEED_VECTORS,
2693                        speed_template_24, num_mb);
2694         test_mb_skcipher_speed("ecb(des3_ede)", DECRYPT, sec,
2695                        des3_speed_template, DES3_SPEED_VECTORS,
2696                        speed_template_24, num_mb);
2697         test_mb_skcipher_speed("cbc(des3_ede)", ENCRYPT, sec,
2698                        des3_speed_template, DES3_SPEED_VECTORS,
2699                        speed_template_24, num_mb);
2700         test_mb_skcipher_speed("cbc(des3_ede)", DECRYPT, sec,
2701                        des3_speed_template, DES3_SPEED_VECTORS,
2702                        speed_template_24, num_mb);
2703         test_mb_skcipher_speed("cfb(des3_ede)", ENCRYPT, sec,
2704                        des3_speed_template, DES3_SPEED_VECTORS,
2705                        speed_template_24, num_mb);
2706         test_mb_skcipher_speed("cfb(des3_ede)", DECRYPT, sec,
2707                        des3_speed_template, DES3_SPEED_VECTORS,
2708                        speed_template_24, num_mb);
2709         test_mb_skcipher_speed("ofb(des3_ede)", ENCRYPT, sec,
2710                        des3_speed_template, DES3_SPEED_VECTORS,
2711                        speed_template_24, num_mb);
2712         test_mb_skcipher_speed("ofb(des3_ede)", DECRYPT, sec,
2713                        des3_speed_template, DES3_SPEED_VECTORS,
2714                        speed_template_24, num_mb);
2715         break;
2716
2717     case 602:
2718         test_mb_skcipher_speed("ecb(des)", ENCRYPT, sec, NULL, 0,
2719                        speed_template_8, num_mb);
2720         test_mb_skcipher_speed("ecb(des)", DECRYPT, sec, NULL, 0,
2721                        speed_template_8, num_mb);
2722         test_mb_skcipher_speed("cbc(des)", ENCRYPT, sec, NULL, 0,
2723                        speed_template_8, num_mb);
2724         test_mb_skcipher_speed("cbc(des)", DECRYPT, sec, NULL, 0,
2725                        speed_template_8, num_mb);
2726         test_mb_skcipher_speed("cfb(des)", ENCRYPT, sec, NULL, 0,
2727                        speed_template_8, num_mb);
2728         test_mb_skcipher_speed("cfb(des)", DECRYPT, sec, NULL, 0,
2729                        speed_template_8, num_mb);
2730         test_mb_skcipher_speed("ofb(des)", ENCRYPT, sec, NULL, 0,
2731                        speed_template_8, num_mb);
2732         test_mb_skcipher_speed("ofb(des)", DECRYPT, sec, NULL, 0,
2733                        speed_template_8, num_mb);
2734         break;
2735
2736     case 603:
2737         test_mb_skcipher_speed("ecb(serpent)", ENCRYPT, sec, NULL, 0,
2738                        speed_template_16_32, num_mb);
2739         test_mb_skcipher_speed("ecb(serpent)", DECRYPT, sec, NULL, 0,
2740                        speed_template_16_32, num_mb);
2741         test_mb_skcipher_speed("cbc(serpent)", ENCRYPT, sec, NULL, 0,
2742                        speed_template_16_32, num_mb);
2743         test_mb_skcipher_speed("cbc(serpent)", DECRYPT, sec, NULL, 0,
2744                        speed_template_16_32, num_mb);
2745         test_mb_skcipher_speed("ctr(serpent)", ENCRYPT, sec, NULL, 0,
2746                        speed_template_16_32, num_mb);
2747         test_mb_skcipher_speed("ctr(serpent)", DECRYPT, sec, NULL, 0,
2748                        speed_template_16_32, num_mb);
2749         test_mb_skcipher_speed("lrw(serpent)", ENCRYPT, sec, NULL, 0,
2750                        speed_template_32_48, num_mb);
2751         test_mb_skcipher_speed("lrw(serpent)", DECRYPT, sec, NULL, 0,
2752                        speed_template_32_48, num_mb);
2753         test_mb_skcipher_speed("xts(serpent)", ENCRYPT, sec, NULL, 0,
2754                        speed_template_32_64, num_mb);
2755         test_mb_skcipher_speed("xts(serpent)", DECRYPT, sec, NULL, 0,
2756                        speed_template_32_64, num_mb);
2757         break;
2758
2759     case 604:
2760         test_mb_skcipher_speed("ecb(twofish)", ENCRYPT, sec, NULL, 0,
2761                        speed_template_16_24_32, num_mb);
2762         test_mb_skcipher_speed("ecb(twofish)", DECRYPT, sec, NULL, 0,
2763                        speed_template_16_24_32, num_mb);
2764         test_mb_skcipher_speed("cbc(twofish)", ENCRYPT, sec, NULL, 0,
2765                        speed_template_16_24_32, num_mb);
2766         test_mb_skcipher_speed("cbc(twofish)", DECRYPT, sec, NULL, 0,
2767                        speed_template_16_24_32, num_mb);
2768         test_mb_skcipher_speed("ctr(twofish)", ENCRYPT, sec, NULL, 0,
2769                        speed_template_16_24_32, num_mb);
2770         test_mb_skcipher_speed("ctr(twofish)", DECRYPT, sec, NULL, 0,
2771                        speed_template_16_24_32, num_mb);
2772         test_mb_skcipher_speed("lrw(twofish)", ENCRYPT, sec, NULL, 0,
2773                        speed_template_32_40_48, num_mb);
2774         test_mb_skcipher_speed("lrw(twofish)", DECRYPT, sec, NULL, 0,
2775                        speed_template_32_40_48, num_mb);
2776         test_mb_skcipher_speed("xts(twofish)", ENCRYPT, sec, NULL, 0,
2777                        speed_template_32_48_64, num_mb);
2778         test_mb_skcipher_speed("xts(twofish)", DECRYPT, sec, NULL, 0,
2779                        speed_template_32_48_64, num_mb);
2780         break;
2781
2782     case 605:
2783         test_mb_skcipher_speed("ecb(arc4)", ENCRYPT, sec, NULL, 0,
2784                        speed_template_8, num_mb);
2785         break;
2786
2787     case 606:
2788         test_mb_skcipher_speed("ecb(cast5)", ENCRYPT, sec, NULL, 0,
2789                        speed_template_8_16, num_mb);
2790         test_mb_skcipher_speed("ecb(cast5)", DECRYPT, sec, NULL, 0,
2791                        speed_template_8_16, num_mb);
2792         test_mb_skcipher_speed("cbc(cast5)", ENCRYPT, sec, NULL, 0,
2793                        speed_template_8_16, num_mb);
2794         test_mb_skcipher_speed("cbc(cast5)", DECRYPT, sec, NULL, 0,
2795                        speed_template_8_16, num_mb);
2796         test_mb_skcipher_speed("ctr(cast5)", ENCRYPT, sec, NULL, 0,
2797                        speed_template_8_16, num_mb);
2798         test_mb_skcipher_speed("ctr(cast5)", DECRYPT, sec, NULL, 0,
2799                        speed_template_8_16, num_mb);
2800         break;
2801
2802     case 607:
2803         test_mb_skcipher_speed("ecb(cast6)", ENCRYPT, sec, NULL, 0,
2804                        speed_template_16_32, num_mb);
2805         test_mb_skcipher_speed("ecb(cast6)", DECRYPT, sec, NULL, 0,
2806                        speed_template_16_32, num_mb);
2807         test_mb_skcipher_speed("cbc(cast6)", ENCRYPT, sec, NULL, 0,
2808                        speed_template_16_32, num_mb);
2809         test_mb_skcipher_speed("cbc(cast6)", DECRYPT, sec, NULL, 0,
2810                        speed_template_16_32, num_mb);
2811         test_mb_skcipher_speed("ctr(cast6)", ENCRYPT, sec, NULL, 0,
2812                        speed_template_16_32, num_mb);
2813         test_mb_skcipher_speed("ctr(cast6)", DECRYPT, sec, NULL, 0,
2814                        speed_template_16_32, num_mb);
2815         test_mb_skcipher_speed("lrw(cast6)", ENCRYPT, sec, NULL, 0,
2816                        speed_template_32_48, num_mb);
2817         test_mb_skcipher_speed("lrw(cast6)", DECRYPT, sec, NULL, 0,
2818                        speed_template_32_48, num_mb);
2819         test_mb_skcipher_speed("xts(cast6)", ENCRYPT, sec, NULL, 0,
2820                        speed_template_32_64, num_mb);
2821         test_mb_skcipher_speed("xts(cast6)", DECRYPT, sec, NULL, 0,
2822                        speed_template_32_64, num_mb);
2823         break;
2824
2825     case 608:
2826         test_mb_skcipher_speed("ecb(camellia)", ENCRYPT, sec, NULL, 0,
2827                        speed_template_16_32, num_mb);
2828         test_mb_skcipher_speed("ecb(camellia)", DECRYPT, sec, NULL, 0,
2829                        speed_template_16_32, num_mb);
2830         test_mb_skcipher_speed("cbc(camellia)", ENCRYPT, sec, NULL, 0,
2831                        speed_template_16_32, num_mb);
2832         test_mb_skcipher_speed("cbc(camellia)", DECRYPT, sec, NULL, 0,
2833                        speed_template_16_32, num_mb);
2834         test_mb_skcipher_speed("ctr(camellia)", ENCRYPT, sec, NULL, 0,
2835                        speed_template_16_32, num_mb);
2836         test_mb_skcipher_speed("ctr(camellia)", DECRYPT, sec, NULL, 0,
2837                        speed_template_16_32, num_mb);
2838         test_mb_skcipher_speed("lrw(camellia)", ENCRYPT, sec, NULL, 0,
2839                        speed_template_32_48, num_mb);
2840         test_mb_skcipher_speed("lrw(camellia)", DECRYPT, sec, NULL, 0,
2841                        speed_template_32_48, num_mb);
2842         test_mb_skcipher_speed("xts(camellia)", ENCRYPT, sec, NULL, 0,
2843                        speed_template_32_64, num_mb);
2844         test_mb_skcipher_speed("xts(camellia)", DECRYPT, sec, NULL, 0,
2845                        speed_template_32_64, num_mb);
2846         break;
2847
2848     case 609:
2849         test_mb_skcipher_speed("ecb(blowfish)", ENCRYPT, sec, NULL, 0,
2850                        speed_template_8_32, num_mb);
2851         test_mb_skcipher_speed("ecb(blowfish)", DECRYPT, sec, NULL, 0,
2852                        speed_template_8_32, num_mb);
2853         test_mb_skcipher_speed("cbc(blowfish)", ENCRYPT, sec, NULL, 0,
2854                        speed_template_8_32, num_mb);
2855         test_mb_skcipher_speed("cbc(blowfish)", DECRYPT, sec, NULL, 0,
2856                        speed_template_8_32, num_mb);
2857         test_mb_skcipher_speed("ctr(blowfish)", ENCRYPT, sec, NULL, 0,
2858                        speed_template_8_32, num_mb);
2859         test_mb_skcipher_speed("ctr(blowfish)", DECRYPT, sec, NULL, 0,
2860                        speed_template_8_32, num_mb);
2861         break;
2862
2863     case 1000:
2864         test_available();
2865         break;
2866     }
2867
2868     return ret;
2869 }
2870
2871 static int __init tcrypt_mod_init(void)
2872 {
2873     int err = -ENOMEM;
2874     int i;
2875
2876     for (i = 0; i < TVMEMSIZE; i++) {
2877         tvmem[i] = (void *)__get_free_page(GFP_KERNEL);
2878         if (!tvmem[i])
2879             goto err_free_tv;
2880     }
2881
2882     err = do_test(alg, type, mask, mode, num_mb);
2883
2884     if (err) {
2885         printk(KERN_ERR "tcrypt: one or more tests failed!\n");
2886         goto err_free_tv;
2887     } else {
2888         pr_debug("all tests passed\n");
2889     }
2890
2891     /* We intentionaly return -EAGAIN to prevent keeping the module,
2892      * unless we're running in fips mode. It does all its work from
2893      * init() and doesn't offer any runtime functionality, but in
2894      * the fips case, checking for a successful load is helpful.
2895      * => we don't need it in the memory, do we?
2896      *                                        -- mludvig
2897      */
2898     if (!fips_enabled)
2899         err = -EAGAIN;
2900
2901 err_free_tv:
2902     for (i = 0; i < TVMEMSIZE && tvmem[i]; i++)
2903         free_page((unsigned long)tvmem[i]);
2904
2905     return err;
2906 }
2907
2908 /*
2909  * If an init function is provided, an exit function must also be provided
2910  * to allow module unload.
2911  */
2912 static void __exit tcrypt_mod_fini(void) { }
2913
2914 late_initcall(tcrypt_mod_init);
2915 module_exit(tcrypt_mod_fini);
2916
2917 module_param(alg, charp, 0);
2918 module_param(type, uint, 0);
2919 module_param(mask, uint, 0);
2920 module_param(mode, int, 0);
2921 module_param(sec, uint, 0);
2922 MODULE_PARM_DESC(sec, "Length in seconds of speed tests "
2923               "(defaults to zero which uses CPU cycles instead)");
2924 module_param(num_mb, uint, 0000);
2925 MODULE_PARM_DESC(num_mb, "Number of concurrent requests to be used in mb speed tests (defaults to 8)");
2926 module_param(klen, uint, 0);
2927 MODULE_PARM_DESC(klen, "Key length (defaults to 0)");
2928
2929 MODULE_LICENSE("GPL");
2930 MODULE_DESCRIPTION("Quick & dirty crypto testing module");
2931 MODULE_AUTHOR("James Morris <jmorris@intercode.com.au>");