Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * ARM NEON accelerated ChaCha and XChaCha stream ciphers,
0004  * including ChaCha20 (RFC7539)
0005  *
0006  * Copyright (C) 2016-2019 Linaro, Ltd. <ard.biesheuvel@linaro.org>
0007  * Copyright (C) 2015 Martin Willi
0008  */
0009 
0010 #include <crypto/algapi.h>
0011 #include <crypto/internal/chacha.h>
0012 #include <crypto/internal/simd.h>
0013 #include <crypto/internal/skcipher.h>
0014 #include <linux/jump_label.h>
0015 #include <linux/kernel.h>
0016 #include <linux/module.h>
0017 
0018 #include <asm/cputype.h>
0019 #include <asm/hwcap.h>
0020 #include <asm/neon.h>
0021 #include <asm/simd.h>
0022 
0023 asmlinkage void chacha_block_xor_neon(const u32 *state, u8 *dst, const u8 *src,
0024                       int nrounds);
0025 asmlinkage void chacha_4block_xor_neon(const u32 *state, u8 *dst, const u8 *src,
0026                        int nrounds, unsigned int nbytes);
0027 asmlinkage void hchacha_block_arm(const u32 *state, u32 *out, int nrounds);
0028 asmlinkage void hchacha_block_neon(const u32 *state, u32 *out, int nrounds);
0029 
0030 asmlinkage void chacha_doarm(u8 *dst, const u8 *src, unsigned int bytes,
0031                  const u32 *state, int nrounds);
0032 
0033 static __ro_after_init DEFINE_STATIC_KEY_FALSE(use_neon);
0034 
0035 static inline bool neon_usable(void)
0036 {
0037     return static_branch_likely(&use_neon) && crypto_simd_usable();
0038 }
0039 
0040 static void chacha_doneon(u32 *state, u8 *dst, const u8 *src,
0041               unsigned int bytes, int nrounds)
0042 {
0043     u8 buf[CHACHA_BLOCK_SIZE];
0044 
0045     while (bytes > CHACHA_BLOCK_SIZE) {
0046         unsigned int l = min(bytes, CHACHA_BLOCK_SIZE * 4U);
0047 
0048         chacha_4block_xor_neon(state, dst, src, nrounds, l);
0049         bytes -= l;
0050         src += l;
0051         dst += l;
0052         state[12] += DIV_ROUND_UP(l, CHACHA_BLOCK_SIZE);
0053     }
0054     if (bytes) {
0055         const u8 *s = src;
0056         u8 *d = dst;
0057 
0058         if (bytes != CHACHA_BLOCK_SIZE)
0059             s = d = memcpy(buf, src, bytes);
0060         chacha_block_xor_neon(state, d, s, nrounds);
0061         if (d != dst)
0062             memcpy(dst, buf, bytes);
0063         state[12]++;
0064     }
0065 }
0066 
0067 void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds)
0068 {
0069     if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon_usable()) {
0070         hchacha_block_arm(state, stream, nrounds);
0071     } else {
0072         kernel_neon_begin();
0073         hchacha_block_neon(state, stream, nrounds);
0074         kernel_neon_end();
0075     }
0076 }
0077 EXPORT_SYMBOL(hchacha_block_arch);
0078 
0079 void chacha_init_arch(u32 *state, const u32 *key, const u8 *iv)
0080 {
0081     chacha_init_generic(state, key, iv);
0082 }
0083 EXPORT_SYMBOL(chacha_init_arch);
0084 
0085 void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes,
0086                int nrounds)
0087 {
0088     if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon_usable() ||
0089         bytes <= CHACHA_BLOCK_SIZE) {
0090         chacha_doarm(dst, src, bytes, state, nrounds);
0091         state[12] += DIV_ROUND_UP(bytes, CHACHA_BLOCK_SIZE);
0092         return;
0093     }
0094 
0095     do {
0096         unsigned int todo = min_t(unsigned int, bytes, SZ_4K);
0097 
0098         kernel_neon_begin();
0099         chacha_doneon(state, dst, src, todo, nrounds);
0100         kernel_neon_end();
0101 
0102         bytes -= todo;
0103         src += todo;
0104         dst += todo;
0105     } while (bytes);
0106 }
0107 EXPORT_SYMBOL(chacha_crypt_arch);
0108 
0109 static int chacha_stream_xor(struct skcipher_request *req,
0110                  const struct chacha_ctx *ctx, const u8 *iv,
0111                  bool neon)
0112 {
0113     struct skcipher_walk walk;
0114     u32 state[16];
0115     int err;
0116 
0117     err = skcipher_walk_virt(&walk, req, false);
0118 
0119     chacha_init_generic(state, ctx->key, iv);
0120 
0121     while (walk.nbytes > 0) {
0122         unsigned int nbytes = walk.nbytes;
0123 
0124         if (nbytes < walk.total)
0125             nbytes = round_down(nbytes, walk.stride);
0126 
0127         if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon) {
0128             chacha_doarm(walk.dst.virt.addr, walk.src.virt.addr,
0129                      nbytes, state, ctx->nrounds);
0130             state[12] += DIV_ROUND_UP(nbytes, CHACHA_BLOCK_SIZE);
0131         } else {
0132             kernel_neon_begin();
0133             chacha_doneon(state, walk.dst.virt.addr,
0134                       walk.src.virt.addr, nbytes, ctx->nrounds);
0135             kernel_neon_end();
0136         }
0137         err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
0138     }
0139 
0140     return err;
0141 }
0142 
0143 static int do_chacha(struct skcipher_request *req, bool neon)
0144 {
0145     struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
0146     struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
0147 
0148     return chacha_stream_xor(req, ctx, req->iv, neon);
0149 }
0150 
0151 static int chacha_arm(struct skcipher_request *req)
0152 {
0153     return do_chacha(req, false);
0154 }
0155 
0156 static int chacha_neon(struct skcipher_request *req)
0157 {
0158     return do_chacha(req, neon_usable());
0159 }
0160 
0161 static int do_xchacha(struct skcipher_request *req, bool neon)
0162 {
0163     struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
0164     struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
0165     struct chacha_ctx subctx;
0166     u32 state[16];
0167     u8 real_iv[16];
0168 
0169     chacha_init_generic(state, ctx->key, req->iv);
0170 
0171     if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon) {
0172         hchacha_block_arm(state, subctx.key, ctx->nrounds);
0173     } else {
0174         kernel_neon_begin();
0175         hchacha_block_neon(state, subctx.key, ctx->nrounds);
0176         kernel_neon_end();
0177     }
0178     subctx.nrounds = ctx->nrounds;
0179 
0180     memcpy(&real_iv[0], req->iv + 24, 8);
0181     memcpy(&real_iv[8], req->iv + 16, 8);
0182     return chacha_stream_xor(req, &subctx, real_iv, neon);
0183 }
0184 
0185 static int xchacha_arm(struct skcipher_request *req)
0186 {
0187     return do_xchacha(req, false);
0188 }
0189 
0190 static int xchacha_neon(struct skcipher_request *req)
0191 {
0192     return do_xchacha(req, neon_usable());
0193 }
0194 
0195 static struct skcipher_alg arm_algs[] = {
0196     {
0197         .base.cra_name      = "chacha20",
0198         .base.cra_driver_name   = "chacha20-arm",
0199         .base.cra_priority  = 200,
0200         .base.cra_blocksize = 1,
0201         .base.cra_ctxsize   = sizeof(struct chacha_ctx),
0202         .base.cra_module    = THIS_MODULE,
0203 
0204         .min_keysize        = CHACHA_KEY_SIZE,
0205         .max_keysize        = CHACHA_KEY_SIZE,
0206         .ivsize         = CHACHA_IV_SIZE,
0207         .chunksize      = CHACHA_BLOCK_SIZE,
0208         .setkey         = chacha20_setkey,
0209         .encrypt        = chacha_arm,
0210         .decrypt        = chacha_arm,
0211     }, {
0212         .base.cra_name      = "xchacha20",
0213         .base.cra_driver_name   = "xchacha20-arm",
0214         .base.cra_priority  = 200,
0215         .base.cra_blocksize = 1,
0216         .base.cra_ctxsize   = sizeof(struct chacha_ctx),
0217         .base.cra_module    = THIS_MODULE,
0218 
0219         .min_keysize        = CHACHA_KEY_SIZE,
0220         .max_keysize        = CHACHA_KEY_SIZE,
0221         .ivsize         = XCHACHA_IV_SIZE,
0222         .chunksize      = CHACHA_BLOCK_SIZE,
0223         .setkey         = chacha20_setkey,
0224         .encrypt        = xchacha_arm,
0225         .decrypt        = xchacha_arm,
0226     }, {
0227         .base.cra_name      = "xchacha12",
0228         .base.cra_driver_name   = "xchacha12-arm",
0229         .base.cra_priority  = 200,
0230         .base.cra_blocksize = 1,
0231         .base.cra_ctxsize   = sizeof(struct chacha_ctx),
0232         .base.cra_module    = THIS_MODULE,
0233 
0234         .min_keysize        = CHACHA_KEY_SIZE,
0235         .max_keysize        = CHACHA_KEY_SIZE,
0236         .ivsize         = XCHACHA_IV_SIZE,
0237         .chunksize      = CHACHA_BLOCK_SIZE,
0238         .setkey         = chacha12_setkey,
0239         .encrypt        = xchacha_arm,
0240         .decrypt        = xchacha_arm,
0241     },
0242 };
0243 
0244 static struct skcipher_alg neon_algs[] = {
0245     {
0246         .base.cra_name      = "chacha20",
0247         .base.cra_driver_name   = "chacha20-neon",
0248         .base.cra_priority  = 300,
0249         .base.cra_blocksize = 1,
0250         .base.cra_ctxsize   = sizeof(struct chacha_ctx),
0251         .base.cra_module    = THIS_MODULE,
0252 
0253         .min_keysize        = CHACHA_KEY_SIZE,
0254         .max_keysize        = CHACHA_KEY_SIZE,
0255         .ivsize         = CHACHA_IV_SIZE,
0256         .chunksize      = CHACHA_BLOCK_SIZE,
0257         .walksize       = 4 * CHACHA_BLOCK_SIZE,
0258         .setkey         = chacha20_setkey,
0259         .encrypt        = chacha_neon,
0260         .decrypt        = chacha_neon,
0261     }, {
0262         .base.cra_name      = "xchacha20",
0263         .base.cra_driver_name   = "xchacha20-neon",
0264         .base.cra_priority  = 300,
0265         .base.cra_blocksize = 1,
0266         .base.cra_ctxsize   = sizeof(struct chacha_ctx),
0267         .base.cra_module    = THIS_MODULE,
0268 
0269         .min_keysize        = CHACHA_KEY_SIZE,
0270         .max_keysize        = CHACHA_KEY_SIZE,
0271         .ivsize         = XCHACHA_IV_SIZE,
0272         .chunksize      = CHACHA_BLOCK_SIZE,
0273         .walksize       = 4 * CHACHA_BLOCK_SIZE,
0274         .setkey         = chacha20_setkey,
0275         .encrypt        = xchacha_neon,
0276         .decrypt        = xchacha_neon,
0277     }, {
0278         .base.cra_name      = "xchacha12",
0279         .base.cra_driver_name   = "xchacha12-neon",
0280         .base.cra_priority  = 300,
0281         .base.cra_blocksize = 1,
0282         .base.cra_ctxsize   = sizeof(struct chacha_ctx),
0283         .base.cra_module    = THIS_MODULE,
0284 
0285         .min_keysize        = CHACHA_KEY_SIZE,
0286         .max_keysize        = CHACHA_KEY_SIZE,
0287         .ivsize         = XCHACHA_IV_SIZE,
0288         .chunksize      = CHACHA_BLOCK_SIZE,
0289         .walksize       = 4 * CHACHA_BLOCK_SIZE,
0290         .setkey         = chacha12_setkey,
0291         .encrypt        = xchacha_neon,
0292         .decrypt        = xchacha_neon,
0293     }
0294 };
0295 
0296 static int __init chacha_simd_mod_init(void)
0297 {
0298     int err = 0;
0299 
0300     if (IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER)) {
0301         err = crypto_register_skciphers(arm_algs, ARRAY_SIZE(arm_algs));
0302         if (err)
0303             return err;
0304     }
0305 
0306     if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && (elf_hwcap & HWCAP_NEON)) {
0307         int i;
0308 
0309         switch (read_cpuid_part()) {
0310         case ARM_CPU_PART_CORTEX_A7:
0311         case ARM_CPU_PART_CORTEX_A5:
0312             /*
0313              * The Cortex-A7 and Cortex-A5 do not perform well with
0314              * the NEON implementation but do incredibly with the
0315              * scalar one and use less power.
0316              */
0317             for (i = 0; i < ARRAY_SIZE(neon_algs); i++)
0318                 neon_algs[i].base.cra_priority = 0;
0319             break;
0320         default:
0321             static_branch_enable(&use_neon);
0322         }
0323 
0324         if (IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER)) {
0325             err = crypto_register_skciphers(neon_algs, ARRAY_SIZE(neon_algs));
0326             if (err)
0327                 crypto_unregister_skciphers(arm_algs, ARRAY_SIZE(arm_algs));
0328         }
0329     }
0330     return err;
0331 }
0332 
0333 static void __exit chacha_simd_mod_fini(void)
0334 {
0335     if (IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER)) {
0336         crypto_unregister_skciphers(arm_algs, ARRAY_SIZE(arm_algs));
0337         if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && (elf_hwcap & HWCAP_NEON))
0338             crypto_unregister_skciphers(neon_algs, ARRAY_SIZE(neon_algs));
0339     }
0340 }
0341 
0342 module_init(chacha_simd_mod_init);
0343 module_exit(chacha_simd_mod_fini);
0344 
0345 MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (scalar and NEON accelerated)");
0346 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
0347 MODULE_LICENSE("GPL v2");
0348 MODULE_ALIAS_CRYPTO("chacha20");
0349 MODULE_ALIAS_CRYPTO("chacha20-arm");
0350 MODULE_ALIAS_CRYPTO("xchacha20");
0351 MODULE_ALIAS_CRYPTO("xchacha20-arm");
0352 MODULE_ALIAS_CRYPTO("xchacha12");
0353 MODULE_ALIAS_CRYPTO("xchacha12-arm");
0354 #ifdef CONFIG_KERNEL_MODE_NEON
0355 MODULE_ALIAS_CRYPTO("chacha20-neon");
0356 MODULE_ALIAS_CRYPTO("xchacha20-neon");
0357 MODULE_ALIAS_CRYPTO("xchacha12-neon");
0358 #endif
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!