Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Accelerated GHASH implementation with ARMv8 PMULL instructions.
0004  *
0005  * Copyright (C) 2014 - 2018 Linaro Ltd. <ard.biesheuvel@linaro.org>
0006  */
0007 
0008 #include <asm/neon.h>
0009 #include <asm/simd.h>
0010 #include <asm/unaligned.h>
0011 #include <crypto/aes.h>
0012 #include <crypto/algapi.h>
0013 #include <crypto/b128ops.h>
0014 #include <crypto/gf128mul.h>
0015 #include <crypto/internal/aead.h>
0016 #include <crypto/internal/hash.h>
0017 #include <crypto/internal/simd.h>
0018 #include <crypto/internal/skcipher.h>
0019 #include <crypto/scatterwalk.h>
0020 #include <linux/cpufeature.h>
0021 #include <linux/crypto.h>
0022 #include <linux/module.h>
0023 
0024 MODULE_DESCRIPTION("GHASH and AES-GCM using ARMv8 Crypto Extensions");
0025 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
0026 MODULE_LICENSE("GPL v2");
0027 MODULE_ALIAS_CRYPTO("ghash");
0028 
0029 #define GHASH_BLOCK_SIZE    16
0030 #define GHASH_DIGEST_SIZE   16
0031 #define GCM_IV_SIZE     12
0032 
0033 struct ghash_key {
0034     be128           k;
0035     u64         h[][2];
0036 };
0037 
0038 struct ghash_desc_ctx {
0039     u64 digest[GHASH_DIGEST_SIZE/sizeof(u64)];
0040     u8 buf[GHASH_BLOCK_SIZE];
0041     u32 count;
0042 };
0043 
0044 struct gcm_aes_ctx {
0045     struct crypto_aes_ctx   aes_key;
0046     struct ghash_key    ghash_key;
0047 };
0048 
0049 asmlinkage void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src,
0050                        u64 const h[][2], const char *head);
0051 
0052 asmlinkage void pmull_ghash_update_p8(int blocks, u64 dg[], const char *src,
0053                       u64 const h[][2], const char *head);
0054 
0055 asmlinkage void pmull_gcm_encrypt(int bytes, u8 dst[], const u8 src[],
0056                   u64 const h[][2], u64 dg[], u8 ctr[],
0057                   u32 const rk[], int rounds, u8 tag[]);
0058 asmlinkage int pmull_gcm_decrypt(int bytes, u8 dst[], const u8 src[],
0059                  u64 const h[][2], u64 dg[], u8 ctr[],
0060                  u32 const rk[], int rounds, const u8 l[],
0061                  const u8 tag[], u64 authsize);
0062 
0063 static int ghash_init(struct shash_desc *desc)
0064 {
0065     struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);
0066 
0067     *ctx = (struct ghash_desc_ctx){};
0068     return 0;
0069 }
0070 
0071 static void ghash_do_update(int blocks, u64 dg[], const char *src,
0072                 struct ghash_key *key, const char *head)
0073 {
0074     be128 dst = { cpu_to_be64(dg[1]), cpu_to_be64(dg[0]) };
0075 
0076     do {
0077         const u8 *in = src;
0078 
0079         if (head) {
0080             in = head;
0081             blocks++;
0082             head = NULL;
0083         } else {
0084             src += GHASH_BLOCK_SIZE;
0085         }
0086 
0087         crypto_xor((u8 *)&dst, in, GHASH_BLOCK_SIZE);
0088         gf128mul_lle(&dst, &key->k);
0089     } while (--blocks);
0090 
0091     dg[0] = be64_to_cpu(dst.b);
0092     dg[1] = be64_to_cpu(dst.a);
0093 }
0094 
0095 static __always_inline
0096 void ghash_do_simd_update(int blocks, u64 dg[], const char *src,
0097               struct ghash_key *key, const char *head,
0098               void (*simd_update)(int blocks, u64 dg[],
0099                           const char *src,
0100                           u64 const h[][2],
0101                           const char *head))
0102 {
0103     if (likely(crypto_simd_usable())) {
0104         kernel_neon_begin();
0105         simd_update(blocks, dg, src, key->h, head);
0106         kernel_neon_end();
0107     } else {
0108         ghash_do_update(blocks, dg, src, key, head);
0109     }
0110 }
0111 
0112 /* avoid hogging the CPU for too long */
0113 #define MAX_BLOCKS  (SZ_64K / GHASH_BLOCK_SIZE)
0114 
0115 static int ghash_update(struct shash_desc *desc, const u8 *src,
0116             unsigned int len)
0117 {
0118     struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);
0119     unsigned int partial = ctx->count % GHASH_BLOCK_SIZE;
0120 
0121     ctx->count += len;
0122 
0123     if ((partial + len) >= GHASH_BLOCK_SIZE) {
0124         struct ghash_key *key = crypto_shash_ctx(desc->tfm);
0125         int blocks;
0126 
0127         if (partial) {
0128             int p = GHASH_BLOCK_SIZE - partial;
0129 
0130             memcpy(ctx->buf + partial, src, p);
0131             src += p;
0132             len -= p;
0133         }
0134 
0135         blocks = len / GHASH_BLOCK_SIZE;
0136         len %= GHASH_BLOCK_SIZE;
0137 
0138         do {
0139             int chunk = min(blocks, MAX_BLOCKS);
0140 
0141             ghash_do_simd_update(chunk, ctx->digest, src, key,
0142                          partial ? ctx->buf : NULL,
0143                          pmull_ghash_update_p8);
0144 
0145             blocks -= chunk;
0146             src += chunk * GHASH_BLOCK_SIZE;
0147             partial = 0;
0148         } while (unlikely(blocks > 0));
0149     }
0150     if (len)
0151         memcpy(ctx->buf + partial, src, len);
0152     return 0;
0153 }
0154 
0155 static int ghash_final(struct shash_desc *desc, u8 *dst)
0156 {
0157     struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);
0158     unsigned int partial = ctx->count % GHASH_BLOCK_SIZE;
0159 
0160     if (partial) {
0161         struct ghash_key *key = crypto_shash_ctx(desc->tfm);
0162 
0163         memset(ctx->buf + partial, 0, GHASH_BLOCK_SIZE - partial);
0164 
0165         ghash_do_simd_update(1, ctx->digest, ctx->buf, key, NULL,
0166                      pmull_ghash_update_p8);
0167     }
0168     put_unaligned_be64(ctx->digest[1], dst);
0169     put_unaligned_be64(ctx->digest[0], dst + 8);
0170 
0171     memzero_explicit(ctx, sizeof(*ctx));
0172     return 0;
0173 }
0174 
0175 static void ghash_reflect(u64 h[], const be128 *k)
0176 {
0177     u64 carry = be64_to_cpu(k->a) & BIT(63) ? 1 : 0;
0178 
0179     h[0] = (be64_to_cpu(k->b) << 1) | carry;
0180     h[1] = (be64_to_cpu(k->a) << 1) | (be64_to_cpu(k->b) >> 63);
0181 
0182     if (carry)
0183         h[1] ^= 0xc200000000000000UL;
0184 }
0185 
0186 static int ghash_setkey(struct crypto_shash *tfm,
0187             const u8 *inkey, unsigned int keylen)
0188 {
0189     struct ghash_key *key = crypto_shash_ctx(tfm);
0190 
0191     if (keylen != GHASH_BLOCK_SIZE)
0192         return -EINVAL;
0193 
0194     /* needed for the fallback */
0195     memcpy(&key->k, inkey, GHASH_BLOCK_SIZE);
0196 
0197     ghash_reflect(key->h[0], &key->k);
0198     return 0;
0199 }
0200 
0201 static struct shash_alg ghash_alg = {
0202     .base.cra_name      = "ghash",
0203     .base.cra_driver_name   = "ghash-neon",
0204     .base.cra_priority  = 150,
0205     .base.cra_blocksize = GHASH_BLOCK_SIZE,
0206     .base.cra_ctxsize   = sizeof(struct ghash_key) + sizeof(u64[2]),
0207     .base.cra_module    = THIS_MODULE,
0208 
0209     .digestsize     = GHASH_DIGEST_SIZE,
0210     .init           = ghash_init,
0211     .update         = ghash_update,
0212     .final          = ghash_final,
0213     .setkey         = ghash_setkey,
0214     .descsize       = sizeof(struct ghash_desc_ctx),
0215 };
0216 
0217 static int num_rounds(struct crypto_aes_ctx *ctx)
0218 {
0219     /*
0220      * # of rounds specified by AES:
0221      * 128 bit key      10 rounds
0222      * 192 bit key      12 rounds
0223      * 256 bit key      14 rounds
0224      * => n byte key    => 6 + (n/4) rounds
0225      */
0226     return 6 + ctx->key_length / 4;
0227 }
0228 
0229 static int gcm_setkey(struct crypto_aead *tfm, const u8 *inkey,
0230               unsigned int keylen)
0231 {
0232     struct gcm_aes_ctx *ctx = crypto_aead_ctx(tfm);
0233     u8 key[GHASH_BLOCK_SIZE];
0234     be128 h;
0235     int ret;
0236 
0237     ret = aes_expandkey(&ctx->aes_key, inkey, keylen);
0238     if (ret)
0239         return -EINVAL;
0240 
0241     aes_encrypt(&ctx->aes_key, key, (u8[AES_BLOCK_SIZE]){});
0242 
0243     /* needed for the fallback */
0244     memcpy(&ctx->ghash_key.k, key, GHASH_BLOCK_SIZE);
0245 
0246     ghash_reflect(ctx->ghash_key.h[0], &ctx->ghash_key.k);
0247 
0248     h = ctx->ghash_key.k;
0249     gf128mul_lle(&h, &ctx->ghash_key.k);
0250     ghash_reflect(ctx->ghash_key.h[1], &h);
0251 
0252     gf128mul_lle(&h, &ctx->ghash_key.k);
0253     ghash_reflect(ctx->ghash_key.h[2], &h);
0254 
0255     gf128mul_lle(&h, &ctx->ghash_key.k);
0256     ghash_reflect(ctx->ghash_key.h[3], &h);
0257 
0258     return 0;
0259 }
0260 
0261 static int gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
0262 {
0263     switch (authsize) {
0264     case 4:
0265     case 8:
0266     case 12 ... 16:
0267         break;
0268     default:
0269         return -EINVAL;
0270     }
0271     return 0;
0272 }
0273 
0274 static void gcm_update_mac(u64 dg[], const u8 *src, int count, u8 buf[],
0275                int *buf_count, struct gcm_aes_ctx *ctx)
0276 {
0277     if (*buf_count > 0) {
0278         int buf_added = min(count, GHASH_BLOCK_SIZE - *buf_count);
0279 
0280         memcpy(&buf[*buf_count], src, buf_added);
0281 
0282         *buf_count += buf_added;
0283         src += buf_added;
0284         count -= buf_added;
0285     }
0286 
0287     if (count >= GHASH_BLOCK_SIZE || *buf_count == GHASH_BLOCK_SIZE) {
0288         int blocks = count / GHASH_BLOCK_SIZE;
0289 
0290         ghash_do_simd_update(blocks, dg, src, &ctx->ghash_key,
0291                      *buf_count ? buf : NULL,
0292                      pmull_ghash_update_p64);
0293 
0294         src += blocks * GHASH_BLOCK_SIZE;
0295         count %= GHASH_BLOCK_SIZE;
0296         *buf_count = 0;
0297     }
0298 
0299     if (count > 0) {
0300         memcpy(buf, src, count);
0301         *buf_count = count;
0302     }
0303 }
0304 
0305 static void gcm_calculate_auth_mac(struct aead_request *req, u64 dg[])
0306 {
0307     struct crypto_aead *aead = crypto_aead_reqtfm(req);
0308     struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead);
0309     u8 buf[GHASH_BLOCK_SIZE];
0310     struct scatter_walk walk;
0311     u32 len = req->assoclen;
0312     int buf_count = 0;
0313 
0314     scatterwalk_start(&walk, req->src);
0315 
0316     do {
0317         u32 n = scatterwalk_clamp(&walk, len);
0318         u8 *p;
0319 
0320         if (!n) {
0321             scatterwalk_start(&walk, sg_next(walk.sg));
0322             n = scatterwalk_clamp(&walk, len);
0323         }
0324         p = scatterwalk_map(&walk);
0325 
0326         gcm_update_mac(dg, p, n, buf, &buf_count, ctx);
0327         len -= n;
0328 
0329         scatterwalk_unmap(p);
0330         scatterwalk_advance(&walk, n);
0331         scatterwalk_done(&walk, 0, len);
0332     } while (len);
0333 
0334     if (buf_count) {
0335         memset(&buf[buf_count], 0, GHASH_BLOCK_SIZE - buf_count);
0336         ghash_do_simd_update(1, dg, buf, &ctx->ghash_key, NULL,
0337                      pmull_ghash_update_p64);
0338     }
0339 }
0340 
0341 static int gcm_encrypt(struct aead_request *req)
0342 {
0343     struct crypto_aead *aead = crypto_aead_reqtfm(req);
0344     struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead);
0345     int nrounds = num_rounds(&ctx->aes_key);
0346     struct skcipher_walk walk;
0347     u8 buf[AES_BLOCK_SIZE];
0348     u8 iv[AES_BLOCK_SIZE];
0349     u64 dg[2] = {};
0350     be128 lengths;
0351     u8 *tag;
0352     int err;
0353 
0354     lengths.a = cpu_to_be64(req->assoclen * 8);
0355     lengths.b = cpu_to_be64(req->cryptlen * 8);
0356 
0357     if (req->assoclen)
0358         gcm_calculate_auth_mac(req, dg);
0359 
0360     memcpy(iv, req->iv, GCM_IV_SIZE);
0361     put_unaligned_be32(2, iv + GCM_IV_SIZE);
0362 
0363     err = skcipher_walk_aead_encrypt(&walk, req, false);
0364 
0365     do {
0366         const u8 *src = walk.src.virt.addr;
0367         u8 *dst = walk.dst.virt.addr;
0368         int nbytes = walk.nbytes;
0369 
0370         tag = (u8 *)&lengths;
0371 
0372         if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE)) {
0373             src = dst = memcpy(buf + sizeof(buf) - nbytes,
0374                        src, nbytes);
0375         } else if (nbytes < walk.total) {
0376             nbytes &= ~(AES_BLOCK_SIZE - 1);
0377             tag = NULL;
0378         }
0379 
0380         kernel_neon_begin();
0381         pmull_gcm_encrypt(nbytes, dst, src, ctx->ghash_key.h,
0382                   dg, iv, ctx->aes_key.key_enc, nrounds,
0383                   tag);
0384         kernel_neon_end();
0385 
0386         if (unlikely(!nbytes))
0387             break;
0388 
0389         if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE))
0390             memcpy(walk.dst.virt.addr,
0391                    buf + sizeof(buf) - nbytes, nbytes);
0392 
0393         err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
0394     } while (walk.nbytes);
0395 
0396     if (err)
0397         return err;
0398 
0399     /* copy authtag to end of dst */
0400     scatterwalk_map_and_copy(tag, req->dst, req->assoclen + req->cryptlen,
0401                  crypto_aead_authsize(aead), 1);
0402 
0403     return 0;
0404 }
0405 
0406 static int gcm_decrypt(struct aead_request *req)
0407 {
0408     struct crypto_aead *aead = crypto_aead_reqtfm(req);
0409     struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead);
0410     unsigned int authsize = crypto_aead_authsize(aead);
0411     int nrounds = num_rounds(&ctx->aes_key);
0412     struct skcipher_walk walk;
0413     u8 otag[AES_BLOCK_SIZE];
0414     u8 buf[AES_BLOCK_SIZE];
0415     u8 iv[AES_BLOCK_SIZE];
0416     u64 dg[2] = {};
0417     be128 lengths;
0418     u8 *tag;
0419     int ret;
0420     int err;
0421 
0422     lengths.a = cpu_to_be64(req->assoclen * 8);
0423     lengths.b = cpu_to_be64((req->cryptlen - authsize) * 8);
0424 
0425     if (req->assoclen)
0426         gcm_calculate_auth_mac(req, dg);
0427 
0428     memcpy(iv, req->iv, GCM_IV_SIZE);
0429     put_unaligned_be32(2, iv + GCM_IV_SIZE);
0430 
0431     scatterwalk_map_and_copy(otag, req->src,
0432                  req->assoclen + req->cryptlen - authsize,
0433                  authsize, 0);
0434 
0435     err = skcipher_walk_aead_decrypt(&walk, req, false);
0436 
0437     do {
0438         const u8 *src = walk.src.virt.addr;
0439         u8 *dst = walk.dst.virt.addr;
0440         int nbytes = walk.nbytes;
0441 
0442         tag = (u8 *)&lengths;
0443 
0444         if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE)) {
0445             src = dst = memcpy(buf + sizeof(buf) - nbytes,
0446                        src, nbytes);
0447         } else if (nbytes < walk.total) {
0448             nbytes &= ~(AES_BLOCK_SIZE - 1);
0449             tag = NULL;
0450         }
0451 
0452         kernel_neon_begin();
0453         ret = pmull_gcm_decrypt(nbytes, dst, src, ctx->ghash_key.h,
0454                     dg, iv, ctx->aes_key.key_enc,
0455                     nrounds, tag, otag, authsize);
0456         kernel_neon_end();
0457 
0458         if (unlikely(!nbytes))
0459             break;
0460 
0461         if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE))
0462             memcpy(walk.dst.virt.addr,
0463                    buf + sizeof(buf) - nbytes, nbytes);
0464 
0465         err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
0466     } while (walk.nbytes);
0467 
0468     if (err)
0469         return err;
0470 
0471     return ret ? -EBADMSG : 0;
0472 }
0473 
0474 static struct aead_alg gcm_aes_alg = {
0475     .ivsize         = GCM_IV_SIZE,
0476     .chunksize      = AES_BLOCK_SIZE,
0477     .maxauthsize        = AES_BLOCK_SIZE,
0478     .setkey         = gcm_setkey,
0479     .setauthsize        = gcm_setauthsize,
0480     .encrypt        = gcm_encrypt,
0481     .decrypt        = gcm_decrypt,
0482 
0483     .base.cra_name      = "gcm(aes)",
0484     .base.cra_driver_name   = "gcm-aes-ce",
0485     .base.cra_priority  = 300,
0486     .base.cra_blocksize = 1,
0487     .base.cra_ctxsize   = sizeof(struct gcm_aes_ctx) +
0488                   4 * sizeof(u64[2]),
0489     .base.cra_module    = THIS_MODULE,
0490 };
0491 
0492 static int __init ghash_ce_mod_init(void)
0493 {
0494     if (!cpu_have_named_feature(ASIMD))
0495         return -ENODEV;
0496 
0497     if (cpu_have_named_feature(PMULL))
0498         return crypto_register_aead(&gcm_aes_alg);
0499 
0500     return crypto_register_shash(&ghash_alg);
0501 }
0502 
0503 static void __exit ghash_ce_mod_exit(void)
0504 {
0505     if (cpu_have_named_feature(PMULL))
0506         crypto_unregister_aead(&gcm_aes_alg);
0507     else
0508         crypto_unregister_shash(&ghash_alg);
0509 }
0510 
0511 static const struct cpu_feature ghash_cpu_feature[] = {
0512     { cpu_feature(PMULL) }, { }
0513 };
0514 MODULE_DEVICE_TABLE(cpu, ghash_cpu_feature);
0515 
0516 module_init(ghash_ce_mod_init);
0517 module_exit(ghash_ce_mod_exit);
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!