arm64/crypto/aes-cipher-core.S

0001 /* SPDX-License-Identifier: GPL-2.0-only */
0002 /*
0003  * Scalar AES core transform
0004  *
0005  * Copyright (C) 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
0006  */
0007
0008 #include <linux/linkage.h>
0009 #include <asm/assembler.h>
0010 #include <asm/cache.h>
0011
0012     .text
0013
0014     rk      .req    x0
0015     out     .req    x1
0016     in      .req    x2
0017     rounds      .req    x3
0018     tt      .req    x2
0019
0020     .macro      __pair1, sz, op, reg0, reg1, in0, in1e, in1d, shift
0021     .ifc        \op\shift, b0
0022     ubfiz       \reg0, \in0, #2, #8
0023     ubfiz       \reg1, \in1e, #2, #8
0024     .else
0025     ubfx        \reg0, \in0, #\shift, #8
0026     ubfx        \reg1, \in1e, #\shift, #8
0027     .endif
0028
0029     /*
0030      * AArch64 cannot do byte size indexed loads from a table containing
0031      * 32-bit quantities, i.e., 'ldrb w12, [tt, w12, uxtw #2]' is not a
0032      * valid instruction. So perform the shift explicitly first for the
0033      * high bytes (the low byte is shifted implicitly by using ubfiz rather
0034      * than ubfx above)
0035      */
0036     .ifnc       \op, b
0037     ldr     \reg0, [tt, \reg0, uxtw #2]
0038     ldr     \reg1, [tt, \reg1, uxtw #2]
0039     .else
0040     .if     \shift > 0
0041     lsl     \reg0, \reg0, #2
0042     lsl     \reg1, \reg1, #2
0043     .endif
0044     ldrb        \reg0, [tt, \reg0, uxtw]
0045     ldrb        \reg1, [tt, \reg1, uxtw]
0046     .endif
0047     .endm
0048
0049     .macro      __pair0, sz, op, reg0, reg1, in0, in1e, in1d, shift
0050     ubfx        \reg0, \in0, #\shift, #8
0051     ubfx        \reg1, \in1d, #\shift, #8
0052     ldr\op      \reg0, [tt, \reg0, uxtw #\sz]
0053     ldr\op      \reg1, [tt, \reg1, uxtw #\sz]
0054     .endm
0055
0056     .macro      __hround, out0, out1, in0, in1, in2, in3, t0, t1, enc, sz, op
0057     ldp     \out0, \out1, [rk], #8
0058
0059     __pair\enc  \sz, \op, w12, w13, \in0, \in1, \in3, 0
0060     __pair\enc  \sz, \op, w14, w15, \in1, \in2, \in0, 8
0061     __pair\enc  \sz, \op, w16, w17, \in2, \in3, \in1, 16
0062     __pair\enc  \sz, \op, \t0, \t1, \in3, \in0, \in2, 24
0063
0064     eor     \out0, \out0, w12
0065     eor     \out1, \out1, w13
0066     eor     \out0, \out0, w14, ror #24
0067     eor     \out1, \out1, w15, ror #24
0068     eor     \out0, \out0, w16, ror #16
0069     eor     \out1, \out1, w17, ror #16
0070     eor     \out0, \out0, \t0, ror #8
0071     eor     \out1, \out1, \t1, ror #8
0072     .endm
0073
0074     .macro      fround, out0, out1, out2, out3, in0, in1, in2, in3, sz=2, op
0075     __hround    \out0, \out1, \in0, \in1, \in2, \in3, \out2, \out3, 1, \sz, \op
0076     __hround    \out2, \out3, \in2, \in3, \in0, \in1, \in1, \in2, 1, \sz, \op
0077     .endm
0078
0079     .macro      iround, out0, out1, out2, out3, in0, in1, in2, in3, sz=2, op
0080     __hround    \out0, \out1, \in0, \in3, \in2, \in1, \out2, \out3, 0, \sz, \op
0081     __hround    \out2, \out3, \in2, \in1, \in0, \in3, \in1, \in0, 0, \sz, \op
0082     .endm
0083
0084     .macro      do_crypt, round, ttab, ltab, bsz
0085     ldp     w4, w5, [in]
0086     ldp     w6, w7, [in, #8]
0087     ldp     w8, w9, [rk], #16
0088     ldp     w10, w11, [rk, #-8]
0089
0090 CPU_BE( rev     w4, w4      )
0091 CPU_BE( rev     w5, w5      )
0092 CPU_BE( rev     w6, w6      )
0093 CPU_BE( rev     w7, w7      )
0094
0095     eor     w4, w4, w8
0096     eor     w5, w5, w9
0097     eor     w6, w6, w10
0098     eor     w7, w7, w11
0099
0100     adr_l       tt, \ttab
0101
0102     tbnz        rounds, #1, 1f
0103
0104 0:  \round      w8, w9, w10, w11, w4, w5, w6, w7
0105     \round      w4, w5, w6, w7, w8, w9, w10, w11
0106
0107 1:  subs        rounds, rounds, #4
0108     \round      w8, w9, w10, w11, w4, w5, w6, w7
0109     b.ls        3f
0110 2:  \round      w4, w5, w6, w7, w8, w9, w10, w11
0111     b       0b
0112 3:  adr_l       tt, \ltab
0113     \round      w4, w5, w6, w7, w8, w9, w10, w11, \bsz, b
0114
0115 CPU_BE( rev     w4, w4      )
0116 CPU_BE( rev     w5, w5      )
0117 CPU_BE( rev     w6, w6      )
0118 CPU_BE( rev     w7, w7      )
0119
0120     stp     w4, w5, [out]
0121     stp     w6, w7, [out, #8]
0122     ret
0123     .endm
0124
0125 SYM_FUNC_START(__aes_arm64_encrypt)
0126     do_crypt    fround, crypto_ft_tab, crypto_ft_tab + 1, 2
0127 SYM_FUNC_END(__aes_arm64_encrypt)
0128
0129     .align      5
0130 SYM_FUNC_START(__aes_arm64_decrypt)
0131     do_crypt    iround, crypto_it_tab, crypto_aes_inv_sbox, 0
0132 SYM_FUNC_END(__aes_arm64_decrypt)