Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0 OR MIT
0002 /*
0003  * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
0004  *
0005  * This is an implementation of the ChaCha20Poly1305 AEAD construction.
0006  *
0007  * Information: https://tools.ietf.org/html/rfc8439
0008  */
0009 
0010 #include <crypto/algapi.h>
0011 #include <crypto/chacha20poly1305.h>
0012 #include <crypto/chacha.h>
0013 #include <crypto/poly1305.h>
0014 #include <crypto/scatterwalk.h>
0015 
0016 #include <asm/unaligned.h>
0017 #include <linux/kernel.h>
0018 #include <linux/init.h>
0019 #include <linux/mm.h>
0020 #include <linux/module.h>
0021 
0022 #define CHACHA_KEY_WORDS    (CHACHA_KEY_SIZE / sizeof(u32))
0023 
0024 static void chacha_load_key(u32 *k, const u8 *in)
0025 {
0026     k[0] = get_unaligned_le32(in);
0027     k[1] = get_unaligned_le32(in + 4);
0028     k[2] = get_unaligned_le32(in + 8);
0029     k[3] = get_unaligned_le32(in + 12);
0030     k[4] = get_unaligned_le32(in + 16);
0031     k[5] = get_unaligned_le32(in + 20);
0032     k[6] = get_unaligned_le32(in + 24);
0033     k[7] = get_unaligned_le32(in + 28);
0034 }
0035 
0036 static void xchacha_init(u32 *chacha_state, const u8 *key, const u8 *nonce)
0037 {
0038     u32 k[CHACHA_KEY_WORDS];
0039     u8 iv[CHACHA_IV_SIZE];
0040 
0041     memset(iv, 0, 8);
0042     memcpy(iv + 8, nonce + 16, 8);
0043 
0044     chacha_load_key(k, key);
0045 
0046     /* Compute the subkey given the original key and first 128 nonce bits */
0047     chacha_init(chacha_state, k, nonce);
0048     hchacha_block(chacha_state, k, 20);
0049 
0050     chacha_init(chacha_state, k, iv);
0051 
0052     memzero_explicit(k, sizeof(k));
0053     memzero_explicit(iv, sizeof(iv));
0054 }
0055 
0056 static void
0057 __chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
0058                const u8 *ad, const size_t ad_len, u32 *chacha_state)
0059 {
0060     const u8 *pad0 = page_address(ZERO_PAGE(0));
0061     struct poly1305_desc_ctx poly1305_state;
0062     union {
0063         u8 block0[POLY1305_KEY_SIZE];
0064         __le64 lens[2];
0065     } b;
0066 
0067     chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
0068     poly1305_init(&poly1305_state, b.block0);
0069 
0070     poly1305_update(&poly1305_state, ad, ad_len);
0071     if (ad_len & 0xf)
0072         poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
0073 
0074     chacha20_crypt(chacha_state, dst, src, src_len);
0075 
0076     poly1305_update(&poly1305_state, dst, src_len);
0077     if (src_len & 0xf)
0078         poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf));
0079 
0080     b.lens[0] = cpu_to_le64(ad_len);
0081     b.lens[1] = cpu_to_le64(src_len);
0082     poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));
0083 
0084     poly1305_final(&poly1305_state, dst + src_len);
0085 
0086     memzero_explicit(chacha_state, CHACHA_STATE_WORDS * sizeof(u32));
0087     memzero_explicit(&b, sizeof(b));
0088 }
0089 
0090 void chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
0091                   const u8 *ad, const size_t ad_len,
0092                   const u64 nonce,
0093                   const u8 key[CHACHA20POLY1305_KEY_SIZE])
0094 {
0095     u32 chacha_state[CHACHA_STATE_WORDS];
0096     u32 k[CHACHA_KEY_WORDS];
0097     __le64 iv[2];
0098 
0099     chacha_load_key(k, key);
0100 
0101     iv[0] = 0;
0102     iv[1] = cpu_to_le64(nonce);
0103 
0104     chacha_init(chacha_state, k, (u8 *)iv);
0105     __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state);
0106 
0107     memzero_explicit(iv, sizeof(iv));
0108     memzero_explicit(k, sizeof(k));
0109 }
0110 EXPORT_SYMBOL(chacha20poly1305_encrypt);
0111 
0112 void xchacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
0113                    const u8 *ad, const size_t ad_len,
0114                    const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE],
0115                    const u8 key[CHACHA20POLY1305_KEY_SIZE])
0116 {
0117     u32 chacha_state[CHACHA_STATE_WORDS];
0118 
0119     xchacha_init(chacha_state, key, nonce);
0120     __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state);
0121 }
0122 EXPORT_SYMBOL(xchacha20poly1305_encrypt);
0123 
0124 static bool
0125 __chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
0126                const u8 *ad, const size_t ad_len, u32 *chacha_state)
0127 {
0128     const u8 *pad0 = page_address(ZERO_PAGE(0));
0129     struct poly1305_desc_ctx poly1305_state;
0130     size_t dst_len;
0131     int ret;
0132     union {
0133         u8 block0[POLY1305_KEY_SIZE];
0134         u8 mac[POLY1305_DIGEST_SIZE];
0135         __le64 lens[2];
0136     } b;
0137 
0138     if (unlikely(src_len < POLY1305_DIGEST_SIZE))
0139         return false;
0140 
0141     chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
0142     poly1305_init(&poly1305_state, b.block0);
0143 
0144     poly1305_update(&poly1305_state, ad, ad_len);
0145     if (ad_len & 0xf)
0146         poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
0147 
0148     dst_len = src_len - POLY1305_DIGEST_SIZE;
0149     poly1305_update(&poly1305_state, src, dst_len);
0150     if (dst_len & 0xf)
0151         poly1305_update(&poly1305_state, pad0, 0x10 - (dst_len & 0xf));
0152 
0153     b.lens[0] = cpu_to_le64(ad_len);
0154     b.lens[1] = cpu_to_le64(dst_len);
0155     poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));
0156 
0157     poly1305_final(&poly1305_state, b.mac);
0158 
0159     ret = crypto_memneq(b.mac, src + dst_len, POLY1305_DIGEST_SIZE);
0160     if (likely(!ret))
0161         chacha20_crypt(chacha_state, dst, src, dst_len);
0162 
0163     memzero_explicit(&b, sizeof(b));
0164 
0165     return !ret;
0166 }
0167 
0168 bool chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
0169                   const u8 *ad, const size_t ad_len,
0170                   const u64 nonce,
0171                   const u8 key[CHACHA20POLY1305_KEY_SIZE])
0172 {
0173     u32 chacha_state[CHACHA_STATE_WORDS];
0174     u32 k[CHACHA_KEY_WORDS];
0175     __le64 iv[2];
0176     bool ret;
0177 
0178     chacha_load_key(k, key);
0179 
0180     iv[0] = 0;
0181     iv[1] = cpu_to_le64(nonce);
0182 
0183     chacha_init(chacha_state, k, (u8 *)iv);
0184     ret = __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len,
0185                      chacha_state);
0186 
0187     memzero_explicit(chacha_state, sizeof(chacha_state));
0188     memzero_explicit(iv, sizeof(iv));
0189     memzero_explicit(k, sizeof(k));
0190     return ret;
0191 }
0192 EXPORT_SYMBOL(chacha20poly1305_decrypt);
0193 
0194 bool xchacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
0195                    const u8 *ad, const size_t ad_len,
0196                    const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE],
0197                    const u8 key[CHACHA20POLY1305_KEY_SIZE])
0198 {
0199     u32 chacha_state[CHACHA_STATE_WORDS];
0200 
0201     xchacha_init(chacha_state, key, nonce);
0202     return __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len,
0203                       chacha_state);
0204 }
0205 EXPORT_SYMBOL(xchacha20poly1305_decrypt);
0206 
0207 static
0208 bool chacha20poly1305_crypt_sg_inplace(struct scatterlist *src,
0209                        const size_t src_len,
0210                        const u8 *ad, const size_t ad_len,
0211                        const u64 nonce,
0212                        const u8 key[CHACHA20POLY1305_KEY_SIZE],
0213                        int encrypt)
0214 {
0215     const u8 *pad0 = page_address(ZERO_PAGE(0));
0216     struct poly1305_desc_ctx poly1305_state;
0217     u32 chacha_state[CHACHA_STATE_WORDS];
0218     struct sg_mapping_iter miter;
0219     size_t partial = 0;
0220     unsigned int flags;
0221     bool ret = true;
0222     int sl;
0223     union {
0224         struct {
0225             u32 k[CHACHA_KEY_WORDS];
0226             __le64 iv[2];
0227         };
0228         u8 block0[POLY1305_KEY_SIZE];
0229         u8 chacha_stream[CHACHA_BLOCK_SIZE];
0230         struct {
0231             u8 mac[2][POLY1305_DIGEST_SIZE];
0232         };
0233         __le64 lens[2];
0234     } b __aligned(16);
0235 
0236     if (WARN_ON(src_len > INT_MAX))
0237         return false;
0238 
0239     chacha_load_key(b.k, key);
0240 
0241     b.iv[0] = 0;
0242     b.iv[1] = cpu_to_le64(nonce);
0243 
0244     chacha_init(chacha_state, b.k, (u8 *)b.iv);
0245     chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
0246     poly1305_init(&poly1305_state, b.block0);
0247 
0248     if (unlikely(ad_len)) {
0249         poly1305_update(&poly1305_state, ad, ad_len);
0250         if (ad_len & 0xf)
0251             poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
0252     }
0253 
0254     flags = SG_MITER_TO_SG | SG_MITER_ATOMIC;
0255 
0256     sg_miter_start(&miter, src, sg_nents(src), flags);
0257 
0258     for (sl = src_len; sl > 0 && sg_miter_next(&miter); sl -= miter.length) {
0259         u8 *addr = miter.addr;
0260         size_t length = min_t(size_t, sl, miter.length);
0261 
0262         if (!encrypt)
0263             poly1305_update(&poly1305_state, addr, length);
0264 
0265         if (unlikely(partial)) {
0266             size_t l = min(length, CHACHA_BLOCK_SIZE - partial);
0267 
0268             crypto_xor(addr, b.chacha_stream + partial, l);
0269             partial = (partial + l) & (CHACHA_BLOCK_SIZE - 1);
0270 
0271             addr += l;
0272             length -= l;
0273         }
0274 
0275         if (likely(length >= CHACHA_BLOCK_SIZE || length == sl)) {
0276             size_t l = length;
0277 
0278             if (unlikely(length < sl))
0279                 l &= ~(CHACHA_BLOCK_SIZE - 1);
0280             chacha20_crypt(chacha_state, addr, addr, l);
0281             addr += l;
0282             length -= l;
0283         }
0284 
0285         if (unlikely(length > 0)) {
0286             chacha20_crypt(chacha_state, b.chacha_stream, pad0,
0287                        CHACHA_BLOCK_SIZE);
0288             crypto_xor(addr, b.chacha_stream, length);
0289             partial = length;
0290         }
0291 
0292         if (encrypt)
0293             poly1305_update(&poly1305_state, miter.addr,
0294                     min_t(size_t, sl, miter.length));
0295     }
0296 
0297     if (src_len & 0xf)
0298         poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf));
0299 
0300     b.lens[0] = cpu_to_le64(ad_len);
0301     b.lens[1] = cpu_to_le64(src_len);
0302     poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));
0303 
0304     if (likely(sl <= -POLY1305_DIGEST_SIZE)) {
0305         if (encrypt) {
0306             poly1305_final(&poly1305_state,
0307                        miter.addr + miter.length + sl);
0308             ret = true;
0309         } else {
0310             poly1305_final(&poly1305_state, b.mac[0]);
0311             ret = !crypto_memneq(b.mac[0],
0312                          miter.addr + miter.length + sl,
0313                          POLY1305_DIGEST_SIZE);
0314         }
0315     }
0316 
0317     sg_miter_stop(&miter);
0318 
0319     if (unlikely(sl > -POLY1305_DIGEST_SIZE)) {
0320         poly1305_final(&poly1305_state, b.mac[1]);
0321         scatterwalk_map_and_copy(b.mac[encrypt], src, src_len,
0322                      sizeof(b.mac[1]), encrypt);
0323         ret = encrypt ||
0324               !crypto_memneq(b.mac[0], b.mac[1], POLY1305_DIGEST_SIZE);
0325     }
0326 
0327     memzero_explicit(chacha_state, sizeof(chacha_state));
0328     memzero_explicit(&b, sizeof(b));
0329 
0330     return ret;
0331 }
0332 
0333 bool chacha20poly1305_encrypt_sg_inplace(struct scatterlist *src, size_t src_len,
0334                      const u8 *ad, const size_t ad_len,
0335                      const u64 nonce,
0336                      const u8 key[CHACHA20POLY1305_KEY_SIZE])
0337 {
0338     return chacha20poly1305_crypt_sg_inplace(src, src_len, ad, ad_len,
0339                          nonce, key, 1);
0340 }
0341 EXPORT_SYMBOL(chacha20poly1305_encrypt_sg_inplace);
0342 
0343 bool chacha20poly1305_decrypt_sg_inplace(struct scatterlist *src, size_t src_len,
0344                      const u8 *ad, const size_t ad_len,
0345                      const u64 nonce,
0346                      const u8 key[CHACHA20POLY1305_KEY_SIZE])
0347 {
0348     if (unlikely(src_len < POLY1305_DIGEST_SIZE))
0349         return false;
0350 
0351     return chacha20poly1305_crypt_sg_inplace(src,
0352                          src_len - POLY1305_DIGEST_SIZE,
0353                          ad, ad_len, nonce, key, 0);
0354 }
0355 EXPORT_SYMBOL(chacha20poly1305_decrypt_sg_inplace);
0356 
0357 static int __init chacha20poly1305_init(void)
0358 {
0359     if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) &&
0360         WARN_ON(!chacha20poly1305_selftest()))
0361         return -ENODEV;
0362     return 0;
0363 }
0364 
0365 static void __exit chacha20poly1305_exit(void)
0366 {
0367 }
0368 
0369 module_init(chacha20poly1305_init);
0370 module_exit(chacha20poly1305_exit);
0371 MODULE_LICENSE("GPL v2");
0372 MODULE_DESCRIPTION("ChaCha20Poly1305 AEAD construction");
0373 MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>");
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!