x86/crypto/aegis128-aesni-glue.c

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  * The AEGIS-128 Authenticated-Encryption Algorithm
0004  *   Glue for AES-NI + SSE2 implementation
0005  *
0006  * Copyright (c) 2017-2018 Ondrej Mosnacek <omosnacek@gmail.com>
0007  * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
0008  */
0009
0010 #include <crypto/internal/aead.h>
0011 #include <crypto/internal/simd.h>
0012 #include <crypto/internal/skcipher.h>
0013 #include <crypto/scatterwalk.h>
0014 #include <linux/module.h>
0015 #include <asm/fpu/api.h>
0016 #include <asm/cpu_device_id.h>
0017
0018 #define AEGIS128_BLOCK_ALIGN 16
0019 #define AEGIS128_BLOCK_SIZE 16
0020 #define AEGIS128_NONCE_SIZE 16
0021 #define AEGIS128_STATE_BLOCKS 5
0022 #define AEGIS128_KEY_SIZE 16
0023 #define AEGIS128_MIN_AUTH_SIZE 8
0024 #define AEGIS128_MAX_AUTH_SIZE 16
0025
0026 asmlinkage void crypto_aegis128_aesni_init(void *state, void *key, void *iv);
0027
0028 asmlinkage void crypto_aegis128_aesni_ad(
0029         void *state, unsigned int length, const void *data);
0030
0031 asmlinkage void crypto_aegis128_aesni_enc(
0032         void *state, unsigned int length, const void *src, void *dst);
0033
0034 asmlinkage void crypto_aegis128_aesni_dec(
0035         void *state, unsigned int length, const void *src, void *dst);
0036
0037 asmlinkage void crypto_aegis128_aesni_enc_tail(
0038         void *state, unsigned int length, const void *src, void *dst);
0039
0040 asmlinkage void crypto_aegis128_aesni_dec_tail(
0041         void *state, unsigned int length, const void *src, void *dst);
0042
0043 asmlinkage void crypto_aegis128_aesni_final(
0044         void *state, void *tag_xor, unsigned int cryptlen,
0045         unsigned int assoclen);
0046
0047 struct aegis_block {
0048     u8 bytes[AEGIS128_BLOCK_SIZE] __aligned(AEGIS128_BLOCK_ALIGN);
0049 };
0050
0051 struct aegis_state {
0052     struct aegis_block blocks[AEGIS128_STATE_BLOCKS];
0053 };
0054
0055 struct aegis_ctx {
0056     struct aegis_block key;
0057 };
0058
0059 struct aegis_crypt_ops {
0060     int (*skcipher_walk_init)(struct skcipher_walk *walk,
0061                   struct aead_request *req, bool atomic);
0062
0063     void (*crypt_blocks)(void *state, unsigned int length, const void *src,
0064                  void *dst);
0065     void (*crypt_tail)(void *state, unsigned int length, const void *src,
0066                void *dst);
0067 };
0068
0069 static void crypto_aegis128_aesni_process_ad(
0070         struct aegis_state *state, struct scatterlist *sg_src,
0071         unsigned int assoclen)
0072 {
0073     struct scatter_walk walk;
0074     struct aegis_block buf;
0075     unsigned int pos = 0;
0076
0077     scatterwalk_start(&walk, sg_src);
0078     while (assoclen != 0) {
0079         unsigned int size = scatterwalk_clamp(&walk, assoclen);
0080         unsigned int left = size;
0081         void *mapped = scatterwalk_map(&walk);
0082         const u8 *src = (const u8 *)mapped;
0083
0084         if (pos + size >= AEGIS128_BLOCK_SIZE) {
0085             if (pos > 0) {
0086                 unsigned int fill = AEGIS128_BLOCK_SIZE - pos;
0087                 memcpy(buf.bytes + pos, src, fill);
0088                 crypto_aegis128_aesni_ad(state,
0089                              AEGIS128_BLOCK_SIZE,
0090                              buf.bytes);
0091                 pos = 0;
0092                 left -= fill;
0093                 src += fill;
0094             }
0095
0096             crypto_aegis128_aesni_ad(state, left, src);
0097
0098             src += left & ~(AEGIS128_BLOCK_SIZE - 1);
0099             left &= AEGIS128_BLOCK_SIZE - 1;
0100         }
0101
0102         memcpy(buf.bytes + pos, src, left);
0103         pos += left;
0104         assoclen -= size;
0105
0106         scatterwalk_unmap(mapped);
0107         scatterwalk_advance(&walk, size);
0108         scatterwalk_done(&walk, 0, assoclen);
0109     }
0110
0111     if (pos > 0) {
0112         memset(buf.bytes + pos, 0, AEGIS128_BLOCK_SIZE - pos);
0113         crypto_aegis128_aesni_ad(state, AEGIS128_BLOCK_SIZE, buf.bytes);
0114     }
0115 }
0116
0117 static void crypto_aegis128_aesni_process_crypt(
0118         struct aegis_state *state, struct skcipher_walk *walk,
0119         const struct aegis_crypt_ops *ops)
0120 {
0121     while (walk->nbytes >= AEGIS128_BLOCK_SIZE) {
0122         ops->crypt_blocks(state,
0123                   round_down(walk->nbytes, AEGIS128_BLOCK_SIZE),
0124                   walk->src.virt.addr, walk->dst.virt.addr);
0125         skcipher_walk_done(walk, walk->nbytes % AEGIS128_BLOCK_SIZE);
0126     }
0127
0128     if (walk->nbytes) {
0129         ops->crypt_tail(state, walk->nbytes, walk->src.virt.addr,
0130                 walk->dst.virt.addr);
0131         skcipher_walk_done(walk, 0);
0132     }
0133 }
0134
0135 static struct aegis_ctx *crypto_aegis128_aesni_ctx(struct crypto_aead *aead)
0136 {
0137     u8 *ctx = crypto_aead_ctx(aead);
0138     ctx = PTR_ALIGN(ctx, __alignof__(struct aegis_ctx));
0139     return (void *)ctx;
0140 }
0141
0142 static int crypto_aegis128_aesni_setkey(struct crypto_aead *aead, const u8 *key,
0143                     unsigned int keylen)
0144 {
0145     struct aegis_ctx *ctx = crypto_aegis128_aesni_ctx(aead);
0146
0147     if (keylen != AEGIS128_KEY_SIZE)
0148         return -EINVAL;
0149
0150     memcpy(ctx->key.bytes, key, AEGIS128_KEY_SIZE);
0151
0152     return 0;
0153 }
0154
0155 static int crypto_aegis128_aesni_setauthsize(struct crypto_aead *tfm,
0156                         unsigned int authsize)
0157 {
0158     if (authsize > AEGIS128_MAX_AUTH_SIZE)
0159         return -EINVAL;
0160     if (authsize < AEGIS128_MIN_AUTH_SIZE)
0161         return -EINVAL;
0162     return 0;
0163 }
0164
0165 static void crypto_aegis128_aesni_crypt(struct aead_request *req,
0166                     struct aegis_block *tag_xor,
0167                     unsigned int cryptlen,
0168                     const struct aegis_crypt_ops *ops)
0169 {
0170     struct crypto_aead *tfm = crypto_aead_reqtfm(req);
0171     struct aegis_ctx *ctx = crypto_aegis128_aesni_ctx(tfm);
0172     struct skcipher_walk walk;
0173     struct aegis_state state;
0174
0175     ops->skcipher_walk_init(&walk, req, true);
0176
0177     kernel_fpu_begin();
0178
0179     crypto_aegis128_aesni_init(&state, ctx->key.bytes, req->iv);
0180     crypto_aegis128_aesni_process_ad(&state, req->src, req->assoclen);
0181     crypto_aegis128_aesni_process_crypt(&state, &walk, ops);
0182     crypto_aegis128_aesni_final(&state, tag_xor, req->assoclen, cryptlen);
0183
0184     kernel_fpu_end();
0185 }
0186
0187 static int crypto_aegis128_aesni_encrypt(struct aead_request *req)
0188 {
0189     static const struct aegis_crypt_ops OPS = {
0190         .skcipher_walk_init = skcipher_walk_aead_encrypt,
0191         .crypt_blocks = crypto_aegis128_aesni_enc,
0192         .crypt_tail = crypto_aegis128_aesni_enc_tail,
0193     };
0194
0195     struct crypto_aead *tfm = crypto_aead_reqtfm(req);
0196     struct aegis_block tag = {};
0197     unsigned int authsize = crypto_aead_authsize(tfm);
0198     unsigned int cryptlen = req->cryptlen;
0199
0200     crypto_aegis128_aesni_crypt(req, &tag, cryptlen, &OPS);
0201
0202     scatterwalk_map_and_copy(tag.bytes, req->dst,
0203                  req->assoclen + cryptlen, authsize, 1);
0204     return 0;
0205 }
0206
0207 static int crypto_aegis128_aesni_decrypt(struct aead_request *req)
0208 {
0209     static const struct aegis_block zeros = {};
0210
0211     static const struct aegis_crypt_ops OPS = {
0212         .skcipher_walk_init = skcipher_walk_aead_decrypt,
0213         .crypt_blocks = crypto_aegis128_aesni_dec,
0214         .crypt_tail = crypto_aegis128_aesni_dec_tail,
0215     };
0216
0217     struct crypto_aead *tfm = crypto_aead_reqtfm(req);
0218     struct aegis_block tag;
0219     unsigned int authsize = crypto_aead_authsize(tfm);
0220     unsigned int cryptlen = req->cryptlen - authsize;
0221
0222     scatterwalk_map_and_copy(tag.bytes, req->src,
0223                  req->assoclen + cryptlen, authsize, 0);
0224
0225     crypto_aegis128_aesni_crypt(req, &tag, cryptlen, &OPS);
0226
0227     return crypto_memneq(tag.bytes, zeros.bytes, authsize) ? -EBADMSG : 0;
0228 }
0229
0230 static int crypto_aegis128_aesni_init_tfm(struct crypto_aead *aead)
0231 {
0232     return 0;
0233 }
0234
0235 static void crypto_aegis128_aesni_exit_tfm(struct crypto_aead *aead)
0236 {
0237 }
0238
0239 static struct aead_alg crypto_aegis128_aesni_alg = {
0240     .setkey = crypto_aegis128_aesni_setkey,
0241     .setauthsize = crypto_aegis128_aesni_setauthsize,
0242     .encrypt = crypto_aegis128_aesni_encrypt,
0243     .decrypt = crypto_aegis128_aesni_decrypt,
0244     .init = crypto_aegis128_aesni_init_tfm,
0245     .exit = crypto_aegis128_aesni_exit_tfm,
0246
0247     .ivsize = AEGIS128_NONCE_SIZE,
0248     .maxauthsize = AEGIS128_MAX_AUTH_SIZE,
0249     .chunksize = AEGIS128_BLOCK_SIZE,
0250
0251     .base = {
0252         .cra_flags = CRYPTO_ALG_INTERNAL,
0253         .cra_blocksize = 1,
0254         .cra_ctxsize = sizeof(struct aegis_ctx) +
0255                    __alignof__(struct aegis_ctx),
0256         .cra_alignmask = 0,
0257         .cra_priority = 400,
0258
0259         .cra_name = "__aegis128",
0260         .cra_driver_name = "__aegis128-aesni",
0261
0262         .cra_module = THIS_MODULE,
0263     }
0264 };
0265
0266 static struct simd_aead_alg *simd_alg;
0267
0268 static int __init crypto_aegis128_aesni_module_init(void)
0269 {
0270     if (!boot_cpu_has(X86_FEATURE_XMM2) ||
0271         !boot_cpu_has(X86_FEATURE_AES) ||
0272         !cpu_has_xfeatures(XFEATURE_MASK_SSE, NULL))
0273         return -ENODEV;
0274
0275     return simd_register_aeads_compat(&crypto_aegis128_aesni_alg, 1,
0276                       &simd_alg);
0277 }
0278
0279 static void __exit crypto_aegis128_aesni_module_exit(void)
0280 {
0281     simd_unregister_aeads(&crypto_aegis128_aesni_alg, 1, &simd_alg);
0282 }
0283
0284 module_init(crypto_aegis128_aesni_module_init);
0285 module_exit(crypto_aegis128_aesni_module_exit);
0286
0287 MODULE_LICENSE("GPL");
0288 MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>");
0289 MODULE_DESCRIPTION("AEGIS-128 AEAD algorithm -- AESNI+SSE2 implementation");
0290 MODULE_ALIAS_CRYPTO("aegis128");
0291 MODULE_ALIAS_CRYPTO("aegis128-aesni");