arm/crypto/aes-cipher-core.S

0001 /* SPDX-License-Identifier: GPL-2.0-only */
0002 /*
0003  * Scalar AES core transform
0004  *
0005  * Copyright (C) 2017 Linaro Ltd.
0006  * Author: Ard Biesheuvel <ard.biesheuvel@linaro.org>
0007  */
0008
0009 #include <linux/linkage.h>
0010 #include <asm/assembler.h>
0011 #include <asm/cache.h>
0012
0013     .text
0014     .align      5
0015
0016     rk      .req    r0
0017     rounds      .req    r1
0018     in      .req    r2
0019     out     .req    r3
0020     ttab        .req    ip
0021
0022     t0      .req    lr
0023     t1      .req    r2
0024     t2      .req    r3
0025
0026     .macro      __select, out, in, idx
0027     .if     __LINUX_ARM_ARCH__ < 7
0028     and     \out, \in, #0xff << (8 * \idx)
0029     .else
0030     ubfx        \out, \in, #(8 * \idx), #8
0031     .endif
0032     .endm
0033
0034     .macro      __load, out, in, idx, sz, op
0035     .if     __LINUX_ARM_ARCH__ < 7 && \idx > 0
0036     ldr\op      \out, [ttab, \in, lsr #(8 * \idx) - \sz]
0037     .else
0038     ldr\op      \out, [ttab, \in, lsl #\sz]
0039     .endif
0040     .endm
0041
0042     .macro      __hround, out0, out1, in0, in1, in2, in3, t3, t4, enc, sz, op, oldcpsr
0043     __select    \out0, \in0, 0
0044     __select    t0, \in1, 1
0045     __load      \out0, \out0, 0, \sz, \op
0046     __load      t0, t0, 1, \sz, \op
0047
0048     .if     \enc
0049     __select    \out1, \in1, 0
0050     __select    t1, \in2, 1
0051     .else
0052     __select    \out1, \in3, 0
0053     __select    t1, \in0, 1
0054     .endif
0055     __load      \out1, \out1, 0, \sz, \op
0056     __select    t2, \in2, 2
0057     __load      t1, t1, 1, \sz, \op
0058     __load      t2, t2, 2, \sz, \op
0059
0060     eor     \out0, \out0, t0, ror #24
0061
0062     __select    t0, \in3, 3
0063     .if     \enc
0064     __select    \t3, \in3, 2
0065     __select    \t4, \in0, 3
0066     .else
0067     __select    \t3, \in1, 2
0068     __select    \t4, \in2, 3
0069     .endif
0070     __load      \t3, \t3, 2, \sz, \op
0071     __load      t0, t0, 3, \sz, \op
0072     __load      \t4, \t4, 3, \sz, \op
0073
0074     .ifnb       \oldcpsr
0075     /*
0076      * This is the final round and we're done with all data-dependent table
0077      * lookups, so we can safely re-enable interrupts.
0078      */
0079     restore_irqs    \oldcpsr
0080     .endif
0081
0082     eor     \out1, \out1, t1, ror #24
0083     eor     \out0, \out0, t2, ror #16
0084     ldm     rk!, {t1, t2}
0085     eor     \out1, \out1, \t3, ror #16
0086     eor     \out0, \out0, t0, ror #8
0087     eor     \out1, \out1, \t4, ror #8
0088     eor     \out0, \out0, t1
0089     eor     \out1, \out1, t2
0090     .endm
0091
0092     .macro      fround, out0, out1, out2, out3, in0, in1, in2, in3, sz=2, op, oldcpsr
0093     __hround    \out0, \out1, \in0, \in1, \in2, \in3, \out2, \out3, 1, \sz, \op
0094     __hround    \out2, \out3, \in2, \in3, \in0, \in1, \in1, \in2, 1, \sz, \op, \oldcpsr
0095     .endm
0096
0097     .macro      iround, out0, out1, out2, out3, in0, in1, in2, in3, sz=2, op, oldcpsr
0098     __hround    \out0, \out1, \in0, \in3, \in2, \in1, \out2, \out3, 0, \sz, \op
0099     __hround    \out2, \out3, \in2, \in1, \in0, \in3, \in1, \in0, 0, \sz, \op, \oldcpsr
0100     .endm
0101
0102     .macro      do_crypt, round, ttab, ltab, bsz
0103     push        {r3-r11, lr}
0104
0105     // Load keys first, to reduce latency in case they're not cached yet.
0106     ldm     rk!, {r8-r11}
0107
0108     ldr     r4, [in]
0109     ldr     r5, [in, #4]
0110     ldr     r6, [in, #8]
0111     ldr     r7, [in, #12]
0112
0113 #ifdef CONFIG_CPU_BIG_ENDIAN
0114     rev_l       r4, t0
0115     rev_l       r5, t0
0116     rev_l       r6, t0
0117     rev_l       r7, t0
0118 #endif
0119
0120     eor     r4, r4, r8
0121     eor     r5, r5, r9
0122     eor     r6, r6, r10
0123     eor     r7, r7, r11
0124
0125     mov_l       ttab, \ttab
0126     /*
0127      * Disable interrupts and prefetch the 1024-byte 'ft' or 'it' table into
0128      * L1 cache, assuming cacheline size >= 32.  This is a hardening measure
0129      * intended to make cache-timing attacks more difficult.  They may not
0130      * be fully prevented, however; see the paper
0131      * https://cr.yp.to/antiforgery/cachetiming-20050414.pdf
0132      * ("Cache-timing attacks on AES") for a discussion of the many
0133      * difficulties involved in writing truly constant-time AES software.
0134      */
0135      save_and_disable_irqs  t0
0136     .set        i, 0
0137     .rept       1024 / 128
0138     ldr     r8, [ttab, #i + 0]
0139     ldr     r9, [ttab, #i + 32]
0140     ldr     r10, [ttab, #i + 64]
0141     ldr     r11, [ttab, #i + 96]
0142     .set        i, i + 128
0143     .endr
0144     push        {t0}        // oldcpsr
0145
0146     tst     rounds, #2
0147     bne     1f
0148
0149 0:  \round      r8, r9, r10, r11, r4, r5, r6, r7
0150     \round      r4, r5, r6, r7, r8, r9, r10, r11
0151
0152 1:  subs        rounds, rounds, #4
0153     \round      r8, r9, r10, r11, r4, r5, r6, r7
0154     bls     2f
0155     \round      r4, r5, r6, r7, r8, r9, r10, r11
0156     b       0b
0157
0158 2:  .ifb        \ltab
0159     add     ttab, ttab, #1
0160     .else
0161     mov_l       ttab, \ltab
0162     // Prefetch inverse S-box for final round; see explanation above
0163     .set        i, 0
0164     .rept       256 / 64
0165     ldr     t0, [ttab, #i + 0]
0166     ldr     t1, [ttab, #i + 32]
0167     .set        i, i + 64
0168     .endr
0169     .endif
0170
0171     pop     {rounds}    // oldcpsr
0172     \round      r4, r5, r6, r7, r8, r9, r10, r11, \bsz, b, rounds
0173
0174 #ifdef CONFIG_CPU_BIG_ENDIAN
0175     rev_l       r4, t0
0176     rev_l       r5, t0
0177     rev_l       r6, t0
0178     rev_l       r7, t0
0179 #endif
0180
0181     ldr     out, [sp]
0182
0183     str     r4, [out]
0184     str     r5, [out, #4]
0185     str     r6, [out, #8]
0186     str     r7, [out, #12]
0187
0188     pop     {r3-r11, pc}
0189
0190     .align      3
0191     .ltorg
0192     .endm
0193
0194 ENTRY(__aes_arm_encrypt)
0195     do_crypt    fround, crypto_ft_tab,, 2
0196 ENDPROC(__aes_arm_encrypt)
0197
0198     .align      5
0199 ENTRY(__aes_arm_decrypt)
0200     do_crypt    iround, crypto_it_tab, crypto_aes_inv_sbox, 0
0201 ENDPROC(__aes_arm_decrypt)