lib/crypto/poly1305-donna64.c

0001 // SPDX-License-Identifier: GPL-2.0 OR MIT
0002 /*
0003  * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
0004  *
0005  * This is based in part on Andrew Moon's poly1305-donna, which is in the
0006  * public domain.
0007  */
0008
0009 #include <linux/kernel.h>
0010 #include <asm/unaligned.h>
0011 #include <crypto/internal/poly1305.h>
0012
0013 typedef __uint128_t u128;
0014
0015 void poly1305_core_setkey(struct poly1305_core_key *key,
0016               const u8 raw_key[POLY1305_BLOCK_SIZE])
0017 {
0018     u64 t0, t1;
0019
0020     /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */
0021     t0 = get_unaligned_le64(&raw_key[0]);
0022     t1 = get_unaligned_le64(&raw_key[8]);
0023
0024     key->key.r64[0] = t0 & 0xffc0fffffffULL;
0025     key->key.r64[1] = ((t0 >> 44) | (t1 << 20)) & 0xfffffc0ffffULL;
0026     key->key.r64[2] = ((t1 >> 24)) & 0x00ffffffc0fULL;
0027
0028     /* s = 20*r */
0029     key->precomputed_s.r64[0] = key->key.r64[1] * 20;
0030     key->precomputed_s.r64[1] = key->key.r64[2] * 20;
0031 }
0032 EXPORT_SYMBOL(poly1305_core_setkey);
0033
0034 void poly1305_core_blocks(struct poly1305_state *state,
0035               const struct poly1305_core_key *key, const void *src,
0036               unsigned int nblocks, u32 hibit)
0037 {
0038     const u8 *input = src;
0039     u64 hibit64;
0040     u64 r0, r1, r2;
0041     u64 s1, s2;
0042     u64 h0, h1, h2;
0043     u64 c;
0044     u128 d0, d1, d2, d;
0045
0046     if (!nblocks)
0047         return;
0048
0049     hibit64 = ((u64)hibit) << 40;
0050
0051     r0 = key->key.r64[0];
0052     r1 = key->key.r64[1];
0053     r2 = key->key.r64[2];
0054
0055     h0 = state->h64[0];
0056     h1 = state->h64[1];
0057     h2 = state->h64[2];
0058
0059     s1 = key->precomputed_s.r64[0];
0060     s2 = key->precomputed_s.r64[1];
0061
0062     do {
0063         u64 t0, t1;
0064
0065         /* h += m[i] */
0066         t0 = get_unaligned_le64(&input[0]);
0067         t1 = get_unaligned_le64(&input[8]);
0068
0069         h0 += t0 & 0xfffffffffffULL;
0070         h1 += ((t0 >> 44) | (t1 << 20)) & 0xfffffffffffULL;
0071         h2 += (((t1 >> 24)) & 0x3ffffffffffULL) | hibit64;
0072
0073         /* h *= r */
0074         d0 = (u128)h0 * r0;
0075         d = (u128)h1 * s2;
0076         d0 += d;
0077         d = (u128)h2 * s1;
0078         d0 += d;
0079         d1 = (u128)h0 * r1;
0080         d = (u128)h1 * r0;
0081         d1 += d;
0082         d = (u128)h2 * s2;
0083         d1 += d;
0084         d2 = (u128)h0 * r2;
0085         d = (u128)h1 * r1;
0086         d2 += d;
0087         d = (u128)h2 * r0;
0088         d2 += d;
0089
0090         /* (partial) h %= p */
0091         c = (u64)(d0 >> 44);
0092         h0 = (u64)d0 & 0xfffffffffffULL;
0093         d1 += c;
0094         c = (u64)(d1 >> 44);
0095         h1 = (u64)d1 & 0xfffffffffffULL;
0096         d2 += c;
0097         c = (u64)(d2 >> 42);
0098         h2 = (u64)d2 & 0x3ffffffffffULL;
0099         h0 += c * 5;
0100         c = h0 >> 44;
0101         h0 = h0 & 0xfffffffffffULL;
0102         h1 += c;
0103
0104         input += POLY1305_BLOCK_SIZE;
0105     } while (--nblocks);
0106
0107     state->h64[0] = h0;
0108     state->h64[1] = h1;
0109     state->h64[2] = h2;
0110 }
0111 EXPORT_SYMBOL(poly1305_core_blocks);
0112
0113 void poly1305_core_emit(const struct poly1305_state *state, const u32 nonce[4],
0114             void *dst)
0115 {
0116     u8 *mac = dst;
0117     u64 h0, h1, h2, c;
0118     u64 g0, g1, g2;
0119     u64 t0, t1;
0120
0121     /* fully carry h */
0122     h0 = state->h64[0];
0123     h1 = state->h64[1];
0124     h2 = state->h64[2];
0125
0126     c = h1 >> 44;
0127     h1 &= 0xfffffffffffULL;
0128     h2 += c;
0129     c = h2 >> 42;
0130     h2 &= 0x3ffffffffffULL;
0131     h0 += c * 5;
0132     c = h0 >> 44;
0133     h0 &= 0xfffffffffffULL;
0134     h1 += c;
0135     c = h1 >> 44;
0136     h1 &= 0xfffffffffffULL;
0137     h2 += c;
0138     c = h2 >> 42;
0139     h2 &= 0x3ffffffffffULL;
0140     h0 += c * 5;
0141     c = h0 >> 44;
0142     h0 &= 0xfffffffffffULL;
0143     h1 += c;
0144
0145     /* compute h + -p */
0146     g0 = h0 + 5;
0147     c  = g0 >> 44;
0148     g0 &= 0xfffffffffffULL;
0149     g1 = h1 + c;
0150     c  = g1 >> 44;
0151     g1 &= 0xfffffffffffULL;
0152     g2 = h2 + c - (1ULL << 42);
0153
0154     /* select h if h < p, or h + -p if h >= p */
0155     c = (g2 >> ((sizeof(u64) * 8) - 1)) - 1;
0156     g0 &= c;
0157     g1 &= c;
0158     g2 &= c;
0159     c  = ~c;
0160     h0 = (h0 & c) | g0;
0161     h1 = (h1 & c) | g1;
0162     h2 = (h2 & c) | g2;
0163
0164     if (likely(nonce)) {
0165         /* h = (h + nonce) */
0166         t0 = ((u64)nonce[1] << 32) | nonce[0];
0167         t1 = ((u64)nonce[3] << 32) | nonce[2];
0168
0169         h0 += t0 & 0xfffffffffffULL;
0170         c = h0 >> 44;
0171         h0 &= 0xfffffffffffULL;
0172         h1 += (((t0 >> 44) | (t1 << 20)) & 0xfffffffffffULL) + c;
0173         c = h1 >> 44;
0174         h1 &= 0xfffffffffffULL;
0175         h2 += (((t1 >> 24)) & 0x3ffffffffffULL) + c;
0176         h2 &= 0x3ffffffffffULL;
0177     }
0178
0179     /* mac = h % (2^128) */
0180     h0 = h0 | (h1 << 44);
0181     h1 = (h1 >> 20) | (h2 << 24);
0182
0183     put_unaligned_le64(h0, &mac[0]);
0184     put_unaligned_le64(h1, &mac[8]);
0185 }
0186 EXPORT_SYMBOL(poly1305_core_emit);