lib/crypto/aes.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * Copyright (C) 2017-2019 Linaro Ltd <ard.biesheuvel@linaro.org>
0004  */
0005
0006 #include <crypto/aes.h>
0007 #include <linux/crypto.h>
0008 #include <linux/module.h>
0009 #include <asm/unaligned.h>
0010
0011 /*
0012  * Emit the sbox as volatile const to prevent the compiler from doing
0013  * constant folding on sbox references involving fixed indexes.
0014  */
0015 static volatile const u8 __cacheline_aligned aes_sbox[] = {
0016     0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5,
0017     0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76,
0018     0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0,
0019     0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0,
0020     0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc,
0021     0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15,
0022     0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a,
0023     0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75,
0024     0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0,
0025     0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84,
0026     0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b,
0027     0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf,
0028     0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85,
0029     0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8,
0030     0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5,
0031     0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2,
0032     0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17,
0033     0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73,
0034     0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88,
0035     0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb,
0036     0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c,
0037     0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79,
0038     0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9,
0039     0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08,
0040     0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6,
0041     0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a,
0042     0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e,
0043     0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e,
0044     0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94,
0045     0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf,
0046     0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68,
0047     0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16,
0048 };
0049
0050 static volatile const u8 __cacheline_aligned aes_inv_sbox[] = {
0051     0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38,
0052     0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb,
0053     0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87,
0054     0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb,
0055     0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d,
0056     0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e,
0057     0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2,
0058     0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25,
0059     0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16,
0060     0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92,
0061     0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda,
0062     0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84,
0063     0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a,
0064     0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06,
0065     0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02,
0066     0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b,
0067     0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea,
0068     0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73,
0069     0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85,
0070     0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e,
0071     0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89,
0072     0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b,
0073     0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20,
0074     0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4,
0075     0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31,
0076     0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f,
0077     0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d,
0078     0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef,
0079     0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0,
0080     0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61,
0081     0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26,
0082     0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d,
0083 };
0084
0085 extern const u8 crypto_aes_sbox[256] __alias(aes_sbox);
0086 extern const u8 crypto_aes_inv_sbox[256] __alias(aes_inv_sbox);
0087
0088 EXPORT_SYMBOL(crypto_aes_sbox);
0089 EXPORT_SYMBOL(crypto_aes_inv_sbox);
0090
0091 static u32 mul_by_x(u32 w)
0092 {
0093     u32 x = w & 0x7f7f7f7f;
0094     u32 y = w & 0x80808080;
0095
0096     /* multiply by polynomial 'x' (0b10) in GF(2^8) */
0097     return (x << 1) ^ (y >> 7) * 0x1b;
0098 }
0099
0100 static u32 mul_by_x2(u32 w)
0101 {
0102     u32 x = w & 0x3f3f3f3f;
0103     u32 y = w & 0x80808080;
0104     u32 z = w & 0x40404040;
0105
0106     /* multiply by polynomial 'x^2' (0b100) in GF(2^8) */
0107     return (x << 2) ^ (y >> 7) * 0x36 ^ (z >> 6) * 0x1b;
0108 }
0109
0110 static u32 mix_columns(u32 x)
0111 {
0112     /*
0113      * Perform the following matrix multiplication in GF(2^8)
0114      *
0115      * | 0x2 0x3 0x1 0x1 |   | x[0] |
0116      * | 0x1 0x2 0x3 0x1 |   | x[1] |
0117      * | 0x1 0x1 0x2 0x3 | x | x[2] |
0118      * | 0x3 0x1 0x1 0x2 |   | x[3] |
0119      */
0120     u32 y = mul_by_x(x) ^ ror32(x, 16);
0121
0122     return y ^ ror32(x ^ y, 8);
0123 }
0124
0125 static u32 inv_mix_columns(u32 x)
0126 {
0127     /*
0128      * Perform the following matrix multiplication in GF(2^8)
0129      *
0130      * | 0xe 0xb 0xd 0x9 |   | x[0] |
0131      * | 0x9 0xe 0xb 0xd |   | x[1] |
0132      * | 0xd 0x9 0xe 0xb | x | x[2] |
0133      * | 0xb 0xd 0x9 0xe |   | x[3] |
0134      *
0135      * which can conveniently be reduced to
0136      *
0137      * | 0x2 0x3 0x1 0x1 |   | 0x5 0x0 0x4 0x0 |   | x[0] |
0138      * | 0x1 0x2 0x3 0x1 |   | 0x0 0x5 0x0 0x4 |   | x[1] |
0139      * | 0x1 0x1 0x2 0x3 | x | 0x4 0x0 0x5 0x0 | x | x[2] |
0140      * | 0x3 0x1 0x1 0x2 |   | 0x0 0x4 0x0 0x5 |   | x[3] |
0141      */
0142     u32 y = mul_by_x2(x);
0143
0144     return mix_columns(x ^ y ^ ror32(y, 16));
0145 }
0146
0147 static __always_inline u32 subshift(u32 in[], int pos)
0148 {
0149     return (aes_sbox[in[pos] & 0xff]) ^
0150            (aes_sbox[(in[(pos + 1) % 4] >>  8) & 0xff] <<  8) ^
0151            (aes_sbox[(in[(pos + 2) % 4] >> 16) & 0xff] << 16) ^
0152            (aes_sbox[(in[(pos + 3) % 4] >> 24) & 0xff] << 24);
0153 }
0154
0155 static __always_inline u32 inv_subshift(u32 in[], int pos)
0156 {
0157     return (aes_inv_sbox[in[pos] & 0xff]) ^
0158            (aes_inv_sbox[(in[(pos + 3) % 4] >>  8) & 0xff] <<  8) ^
0159            (aes_inv_sbox[(in[(pos + 2) % 4] >> 16) & 0xff] << 16) ^
0160            (aes_inv_sbox[(in[(pos + 1) % 4] >> 24) & 0xff] << 24);
0161 }
0162
0163 static u32 subw(u32 in)
0164 {
0165     return (aes_sbox[in & 0xff]) ^
0166            (aes_sbox[(in >>  8) & 0xff] <<  8) ^
0167            (aes_sbox[(in >> 16) & 0xff] << 16) ^
0168            (aes_sbox[(in >> 24) & 0xff] << 24);
0169 }
0170
0171 /**
0172  * aes_expandkey - Expands the AES key as described in FIPS-197
0173  * @ctx:    The location where the computed key will be stored.
0174  * @in_key: The supplied key.
0175  * @key_len:    The length of the supplied key.
0176  *
0177  * Returns 0 on success. The function fails only if an invalid key size (or
0178  * pointer) is supplied.
0179  * The expanded key size is 240 bytes (max of 14 rounds with a unique 16 bytes
0180  * key schedule plus a 16 bytes key which is used before the first round).
0181  * The decryption key is prepared for the "Equivalent Inverse Cipher" as
0182  * described in FIPS-197. The first slot (16 bytes) of each key (enc or dec) is
0183  * for the initial combination, the second slot for the first round and so on.
0184  */
0185 int aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,
0186           unsigned int key_len)
0187 {
0188     u32 kwords = key_len / sizeof(u32);
0189     u32 rc, i, j;
0190     int err;
0191
0192     err = aes_check_keylen(key_len);
0193     if (err)
0194         return err;
0195
0196     ctx->key_length = key_len;
0197
0198     for (i = 0; i < kwords; i++)
0199         ctx->key_enc[i] = get_unaligned_le32(in_key + i * sizeof(u32));
0200
0201     for (i = 0, rc = 1; i < 10; i++, rc = mul_by_x(rc)) {
0202         u32 *rki = ctx->key_enc + (i * kwords);
0203         u32 *rko = rki + kwords;
0204
0205         rko[0] = ror32(subw(rki[kwords - 1]), 8) ^ rc ^ rki[0];
0206         rko[1] = rko[0] ^ rki[1];
0207         rko[2] = rko[1] ^ rki[2];
0208         rko[3] = rko[2] ^ rki[3];
0209
0210         if (key_len == AES_KEYSIZE_192) {
0211             if (i >= 7)
0212                 break;
0213             rko[4] = rko[3] ^ rki[4];
0214             rko[5] = rko[4] ^ rki[5];
0215         } else if (key_len == AES_KEYSIZE_256) {
0216             if (i >= 6)
0217                 break;
0218             rko[4] = subw(rko[3]) ^ rki[4];
0219             rko[5] = rko[4] ^ rki[5];
0220             rko[6] = rko[5] ^ rki[6];
0221             rko[7] = rko[6] ^ rki[7];
0222         }
0223     }
0224
0225     /*
0226      * Generate the decryption keys for the Equivalent Inverse Cipher.
0227      * This involves reversing the order of the round keys, and applying
0228      * the Inverse Mix Columns transformation to all but the first and
0229      * the last one.
0230      */
0231     ctx->key_dec[0] = ctx->key_enc[key_len + 24];
0232     ctx->key_dec[1] = ctx->key_enc[key_len + 25];
0233     ctx->key_dec[2] = ctx->key_enc[key_len + 26];
0234     ctx->key_dec[3] = ctx->key_enc[key_len + 27];
0235
0236     for (i = 4, j = key_len + 20; j > 0; i += 4, j -= 4) {
0237         ctx->key_dec[i]     = inv_mix_columns(ctx->key_enc[j]);
0238         ctx->key_dec[i + 1] = inv_mix_columns(ctx->key_enc[j + 1]);
0239         ctx->key_dec[i + 2] = inv_mix_columns(ctx->key_enc[j + 2]);
0240         ctx->key_dec[i + 3] = inv_mix_columns(ctx->key_enc[j + 3]);
0241     }
0242
0243     ctx->key_dec[i]     = ctx->key_enc[0];
0244     ctx->key_dec[i + 1] = ctx->key_enc[1];
0245     ctx->key_dec[i + 2] = ctx->key_enc[2];
0246     ctx->key_dec[i + 3] = ctx->key_enc[3];
0247
0248     return 0;
0249 }
0250 EXPORT_SYMBOL(aes_expandkey);
0251
0252 /**
0253  * aes_encrypt - Encrypt a single AES block
0254  * @ctx:    Context struct containing the key schedule
0255  * @out:    Buffer to store the ciphertext
0256  * @in:     Buffer containing the plaintext
0257  */
0258 void aes_encrypt(const struct crypto_aes_ctx *ctx, u8 *out, const u8 *in)
0259 {
0260     const u32 *rkp = ctx->key_enc + 4;
0261     int rounds = 6 + ctx->key_length / 4;
0262     u32 st0[4], st1[4];
0263     int round;
0264
0265     st0[0] = ctx->key_enc[0] ^ get_unaligned_le32(in);
0266     st0[1] = ctx->key_enc[1] ^ get_unaligned_le32(in + 4);
0267     st0[2] = ctx->key_enc[2] ^ get_unaligned_le32(in + 8);
0268     st0[3] = ctx->key_enc[3] ^ get_unaligned_le32(in + 12);
0269
0270     /*
0271      * Force the compiler to emit data independent Sbox references,
0272      * by xoring the input with Sbox values that are known to add up
0273      * to zero. This pulls the entire Sbox into the D-cache before any
0274      * data dependent lookups are done.
0275      */
0276     st0[0] ^= aes_sbox[ 0] ^ aes_sbox[ 64] ^ aes_sbox[134] ^ aes_sbox[195];
0277     st0[1] ^= aes_sbox[16] ^ aes_sbox[ 82] ^ aes_sbox[158] ^ aes_sbox[221];
0278     st0[2] ^= aes_sbox[32] ^ aes_sbox[ 96] ^ aes_sbox[160] ^ aes_sbox[234];
0279     st0[3] ^= aes_sbox[48] ^ aes_sbox[112] ^ aes_sbox[186] ^ aes_sbox[241];
0280
0281     for (round = 0;; round += 2, rkp += 8) {
0282         st1[0] = mix_columns(subshift(st0, 0)) ^ rkp[0];
0283         st1[1] = mix_columns(subshift(st0, 1)) ^ rkp[1];
0284         st1[2] = mix_columns(subshift(st0, 2)) ^ rkp[2];
0285         st1[3] = mix_columns(subshift(st0, 3)) ^ rkp[3];
0286
0287         if (round == rounds - 2)
0288             break;
0289
0290         st0[0] = mix_columns(subshift(st1, 0)) ^ rkp[4];
0291         st0[1] = mix_columns(subshift(st1, 1)) ^ rkp[5];
0292         st0[2] = mix_columns(subshift(st1, 2)) ^ rkp[6];
0293         st0[3] = mix_columns(subshift(st1, 3)) ^ rkp[7];
0294     }
0295
0296     put_unaligned_le32(subshift(st1, 0) ^ rkp[4], out);
0297     put_unaligned_le32(subshift(st1, 1) ^ rkp[5], out + 4);
0298     put_unaligned_le32(subshift(st1, 2) ^ rkp[6], out + 8);
0299     put_unaligned_le32(subshift(st1, 3) ^ rkp[7], out + 12);
0300 }
0301 EXPORT_SYMBOL(aes_encrypt);
0302
0303 /**
0304  * aes_decrypt - Decrypt a single AES block
0305  * @ctx:    Context struct containing the key schedule
0306  * @out:    Buffer to store the plaintext
0307  * @in:     Buffer containing the ciphertext
0308  */
0309 void aes_decrypt(const struct crypto_aes_ctx *ctx, u8 *out, const u8 *in)
0310 {
0311     const u32 *rkp = ctx->key_dec + 4;
0312     int rounds = 6 + ctx->key_length / 4;
0313     u32 st0[4], st1[4];
0314     int round;
0315
0316     st0[0] = ctx->key_dec[0] ^ get_unaligned_le32(in);
0317     st0[1] = ctx->key_dec[1] ^ get_unaligned_le32(in + 4);
0318     st0[2] = ctx->key_dec[2] ^ get_unaligned_le32(in + 8);
0319     st0[3] = ctx->key_dec[3] ^ get_unaligned_le32(in + 12);
0320
0321     /*
0322      * Force the compiler to emit data independent Sbox references,
0323      * by xoring the input with Sbox values that are known to add up
0324      * to zero. This pulls the entire Sbox into the D-cache before any
0325      * data dependent lookups are done.
0326      */
0327     st0[0] ^= aes_inv_sbox[ 0] ^ aes_inv_sbox[ 64] ^ aes_inv_sbox[129] ^ aes_inv_sbox[200];
0328     st0[1] ^= aes_inv_sbox[16] ^ aes_inv_sbox[ 83] ^ aes_inv_sbox[150] ^ aes_inv_sbox[212];
0329     st0[2] ^= aes_inv_sbox[32] ^ aes_inv_sbox[ 96] ^ aes_inv_sbox[160] ^ aes_inv_sbox[236];
0330     st0[3] ^= aes_inv_sbox[48] ^ aes_inv_sbox[112] ^ aes_inv_sbox[187] ^ aes_inv_sbox[247];
0331
0332     for (round = 0;; round += 2, rkp += 8) {
0333         st1[0] = inv_mix_columns(inv_subshift(st0, 0)) ^ rkp[0];
0334         st1[1] = inv_mix_columns(inv_subshift(st0, 1)) ^ rkp[1];
0335         st1[2] = inv_mix_columns(inv_subshift(st0, 2)) ^ rkp[2];
0336         st1[3] = inv_mix_columns(inv_subshift(st0, 3)) ^ rkp[3];
0337
0338         if (round == rounds - 2)
0339             break;
0340
0341         st0[0] = inv_mix_columns(inv_subshift(st1, 0)) ^ rkp[4];
0342         st0[1] = inv_mix_columns(inv_subshift(st1, 1)) ^ rkp[5];
0343         st0[2] = inv_mix_columns(inv_subshift(st1, 2)) ^ rkp[6];
0344         st0[3] = inv_mix_columns(inv_subshift(st1, 3)) ^ rkp[7];
0345     }
0346
0347     put_unaligned_le32(inv_subshift(st1, 0) ^ rkp[4], out);
0348     put_unaligned_le32(inv_subshift(st1, 1) ^ rkp[5], out + 4);
0349     put_unaligned_le32(inv_subshift(st1, 2) ^ rkp[6], out + 8);
0350     put_unaligned_le32(inv_subshift(st1, 3) ^ rkp[7], out + 12);
0351 }
0352 EXPORT_SYMBOL(aes_decrypt);
0353
0354 MODULE_DESCRIPTION("Generic AES library");
0355 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
0356 MODULE_LICENSE("GPL v2");