arm64/crypto/aes-ce.S

0001 /* SPDX-License-Identifier: GPL-2.0-only */
0002 /*
0003  * linux/arch/arm64/crypto/aes-ce.S - AES cipher for ARMv8 with
0004  *                                    Crypto Extensions
0005  *
0006  * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
0007  */
0008
0009 #include <linux/linkage.h>
0010 #include <asm/assembler.h>
0011
0012 #define AES_FUNC_START(func)        SYM_FUNC_START(ce_ ## func)
0013 #define AES_FUNC_END(func)      SYM_FUNC_END(ce_ ## func)
0014
0015     .arch       armv8-a+crypto
0016
0017     xtsmask     .req    v16
0018     cbciv       .req    v16
0019     vctr        .req    v16
0020
0021     .macro      xts_reload_mask, tmp
0022     .endm
0023
0024     .macro      xts_cts_skip_tw, reg, lbl
0025     .endm
0026
0027     /* preload all round keys */
0028     .macro      load_round_keys, rounds, rk
0029     cmp     \rounds, #12
0030     blo     2222f       /* 128 bits */
0031     beq     1111f       /* 192 bits */
0032     ld1     {v17.4s-v18.4s}, [\rk], #32
0033 1111:   ld1     {v19.4s-v20.4s}, [\rk], #32
0034 2222:   ld1     {v21.4s-v24.4s}, [\rk], #64
0035     ld1     {v25.4s-v28.4s}, [\rk], #64
0036     ld1     {v29.4s-v31.4s}, [\rk]
0037     .endm
0038
0039     /* prepare for encryption with key in rk[] */
0040     .macro      enc_prepare, rounds, rk, temp
0041     mov     \temp, \rk
0042     load_round_keys \rounds, \temp
0043     .endm
0044
0045     /* prepare for encryption (again) but with new key in rk[] */
0046     .macro      enc_switch_key, rounds, rk, temp
0047     mov     \temp, \rk
0048     load_round_keys \rounds, \temp
0049     .endm
0050
0051     /* prepare for decryption with key in rk[] */
0052     .macro      dec_prepare, rounds, rk, temp
0053     mov     \temp, \rk
0054     load_round_keys \rounds, \temp
0055     .endm
0056
0057     .macro      do_enc_Nx, de, mc, k, i0, i1, i2, i3, i4
0058     aes\de      \i0\().16b, \k\().16b
0059     aes\mc      \i0\().16b, \i0\().16b
0060     .ifnb       \i1
0061     aes\de      \i1\().16b, \k\().16b
0062     aes\mc      \i1\().16b, \i1\().16b
0063     .ifnb       \i3
0064     aes\de      \i2\().16b, \k\().16b
0065     aes\mc      \i2\().16b, \i2\().16b
0066     aes\de      \i3\().16b, \k\().16b
0067     aes\mc      \i3\().16b, \i3\().16b
0068     .ifnb       \i4
0069     aes\de      \i4\().16b, \k\().16b
0070     aes\mc      \i4\().16b, \i4\().16b
0071     .endif
0072     .endif
0073     .endif
0074     .endm
0075
0076     /* up to 5 interleaved encryption rounds with the same round key */
0077     .macro      round_Nx, enc, k, i0, i1, i2, i3, i4
0078     .ifc        \enc, e
0079     do_enc_Nx   e, mc, \k, \i0, \i1, \i2, \i3, \i4
0080     .else
0081     do_enc_Nx   d, imc, \k, \i0, \i1, \i2, \i3, \i4
0082     .endif
0083     .endm
0084
0085     /* up to 5 interleaved final rounds */
0086     .macro      fin_round_Nx, de, k, k2, i0, i1, i2, i3, i4
0087     aes\de      \i0\().16b, \k\().16b
0088     .ifnb       \i1
0089     aes\de      \i1\().16b, \k\().16b
0090     .ifnb       \i3
0091     aes\de      \i2\().16b, \k\().16b
0092     aes\de      \i3\().16b, \k\().16b
0093     .ifnb       \i4
0094     aes\de      \i4\().16b, \k\().16b
0095     .endif
0096     .endif
0097     .endif
0098     eor     \i0\().16b, \i0\().16b, \k2\().16b
0099     .ifnb       \i1
0100     eor     \i1\().16b, \i1\().16b, \k2\().16b
0101     .ifnb       \i3
0102     eor     \i2\().16b, \i2\().16b, \k2\().16b
0103     eor     \i3\().16b, \i3\().16b, \k2\().16b
0104     .ifnb       \i4
0105     eor     \i4\().16b, \i4\().16b, \k2\().16b
0106     .endif
0107     .endif
0108     .endif
0109     .endm
0110
0111     /* up to 5 interleaved blocks */
0112     .macro      do_block_Nx, enc, rounds, i0, i1, i2, i3, i4
0113     cmp     \rounds, #12
0114     blo     2222f       /* 128 bits */
0115     beq     1111f       /* 192 bits */
0116     round_Nx    \enc, v17, \i0, \i1, \i2, \i3, \i4
0117     round_Nx    \enc, v18, \i0, \i1, \i2, \i3, \i4
0118 1111:   round_Nx    \enc, v19, \i0, \i1, \i2, \i3, \i4
0119     round_Nx    \enc, v20, \i0, \i1, \i2, \i3, \i4
0120 2222:   .irp        key, v21, v22, v23, v24, v25, v26, v27, v28, v29
0121     round_Nx    \enc, \key, \i0, \i1, \i2, \i3, \i4
0122     .endr
0123     fin_round_Nx    \enc, v30, v31, \i0, \i1, \i2, \i3, \i4
0124     .endm
0125
0126     .macro      encrypt_block, in, rounds, t0, t1, t2
0127     do_block_Nx e, \rounds, \in
0128     .endm
0129
0130     .macro      encrypt_block4x, i0, i1, i2, i3, rounds, t0, t1, t2
0131     do_block_Nx e, \rounds, \i0, \i1, \i2, \i3
0132     .endm
0133
0134     .macro      encrypt_block5x, i0, i1, i2, i3, i4, rounds, t0, t1, t2
0135     do_block_Nx e, \rounds, \i0, \i1, \i2, \i3, \i4
0136     .endm
0137
0138     .macro      decrypt_block, in, rounds, t0, t1, t2
0139     do_block_Nx d, \rounds, \in
0140     .endm
0141
0142     .macro      decrypt_block4x, i0, i1, i2, i3, rounds, t0, t1, t2
0143     do_block_Nx d, \rounds, \i0, \i1, \i2, \i3
0144     .endm
0145
0146     .macro      decrypt_block5x, i0, i1, i2, i3, i4, rounds, t0, t1, t2
0147     do_block_Nx d, \rounds, \i0, \i1, \i2, \i3, \i4
0148     .endm
0149
0150 #define MAX_STRIDE  5
0151
0152 #include "aes-modes.S"