arm64/net/bpf_jit.h

0001 /* SPDX-License-Identifier: GPL-2.0-only */
0002 /*
0003  * BPF JIT compiler for ARM64
0004  *
0005  * Copyright (C) 2014-2016 Zi Shen Lim <zlim.lnx@gmail.com>
0006  */
0007 #ifndef _BPF_JIT_H
0008 #define _BPF_JIT_H
0009
0010 #include <asm/insn.h>
0011
0012 /* 5-bit Register Operand */
0013 #define A64_R(x)    AARCH64_INSN_REG_##x
0014 #define A64_FP      AARCH64_INSN_REG_FP
0015 #define A64_LR      AARCH64_INSN_REG_LR
0016 #define A64_ZR      AARCH64_INSN_REG_ZR
0017 #define A64_SP      AARCH64_INSN_REG_SP
0018
0019 #define A64_VARIANT(sf) \
0020     ((sf) ? AARCH64_INSN_VARIANT_64BIT : AARCH64_INSN_VARIANT_32BIT)
0021
0022 /* Compare & branch (immediate) */
0023 #define A64_COMP_BRANCH(sf, Rt, offset, type) \
0024     aarch64_insn_gen_comp_branch_imm(0, offset, Rt, A64_VARIANT(sf), \
0025         AARCH64_INSN_BRANCH_COMP_##type)
0026 #define A64_CBZ(sf, Rt, imm19) A64_COMP_BRANCH(sf, Rt, (imm19) << 2, ZERO)
0027 #define A64_CBNZ(sf, Rt, imm19) A64_COMP_BRANCH(sf, Rt, (imm19) << 2, NONZERO)
0028
0029 /* Conditional branch (immediate) */
0030 #define A64_COND_BRANCH(cond, offset) \
0031     aarch64_insn_gen_cond_branch_imm(0, offset, cond)
0032 #define A64_COND_EQ AARCH64_INSN_COND_EQ /* == */
0033 #define A64_COND_NE AARCH64_INSN_COND_NE /* != */
0034 #define A64_COND_CS AARCH64_INSN_COND_CS /* unsigned >= */
0035 #define A64_COND_HI AARCH64_INSN_COND_HI /* unsigned > */
0036 #define A64_COND_LS AARCH64_INSN_COND_LS /* unsigned <= */
0037 #define A64_COND_CC AARCH64_INSN_COND_CC /* unsigned < */
0038 #define A64_COND_GE AARCH64_INSN_COND_GE /* signed >= */
0039 #define A64_COND_GT AARCH64_INSN_COND_GT /* signed > */
0040 #define A64_COND_LE AARCH64_INSN_COND_LE /* signed <= */
0041 #define A64_COND_LT AARCH64_INSN_COND_LT /* signed < */
0042 #define A64_B_(cond, imm19) A64_COND_BRANCH(cond, (imm19) << 2)
0043
0044 /* Unconditional branch (immediate) */
0045 #define A64_BRANCH(offset, type) aarch64_insn_gen_branch_imm(0, offset, \
0046     AARCH64_INSN_BRANCH_##type)
0047 #define A64_B(imm26)  A64_BRANCH((imm26) << 2, NOLINK)
0048 #define A64_BL(imm26) A64_BRANCH((imm26) << 2, LINK)
0049
0050 /* Unconditional branch (register) */
0051 #define A64_BR(Rn)  aarch64_insn_gen_branch_reg(Rn, AARCH64_INSN_BRANCH_NOLINK)
0052 #define A64_BLR(Rn) aarch64_insn_gen_branch_reg(Rn, AARCH64_INSN_BRANCH_LINK)
0053 #define A64_RET(Rn) aarch64_insn_gen_branch_reg(Rn, AARCH64_INSN_BRANCH_RETURN)
0054
0055 /* Load/store register (register offset) */
0056 #define A64_LS_REG(Rt, Rn, Rm, size, type) \
0057     aarch64_insn_gen_load_store_reg(Rt, Rn, Rm, \
0058         AARCH64_INSN_SIZE_##size, \
0059         AARCH64_INSN_LDST_##type##_REG_OFFSET)
0060 #define A64_STRB(Wt, Xn, Xm)  A64_LS_REG(Wt, Xn, Xm, 8, STORE)
0061 #define A64_LDRB(Wt, Xn, Xm)  A64_LS_REG(Wt, Xn, Xm, 8, LOAD)
0062 #define A64_STRH(Wt, Xn, Xm)  A64_LS_REG(Wt, Xn, Xm, 16, STORE)
0063 #define A64_LDRH(Wt, Xn, Xm)  A64_LS_REG(Wt, Xn, Xm, 16, LOAD)
0064 #define A64_STR32(Wt, Xn, Xm) A64_LS_REG(Wt, Xn, Xm, 32, STORE)
0065 #define A64_LDR32(Wt, Xn, Xm) A64_LS_REG(Wt, Xn, Xm, 32, LOAD)
0066 #define A64_STR64(Xt, Xn, Xm) A64_LS_REG(Xt, Xn, Xm, 64, STORE)
0067 #define A64_LDR64(Xt, Xn, Xm) A64_LS_REG(Xt, Xn, Xm, 64, LOAD)
0068
0069 /* Load/store register (immediate offset) */
0070 #define A64_LS_IMM(Rt, Rn, imm, size, type) \
0071     aarch64_insn_gen_load_store_imm(Rt, Rn, imm, \
0072         AARCH64_INSN_SIZE_##size, \
0073         AARCH64_INSN_LDST_##type##_IMM_OFFSET)
0074 #define A64_STRBI(Wt, Xn, imm)  A64_LS_IMM(Wt, Xn, imm, 8, STORE)
0075 #define A64_LDRBI(Wt, Xn, imm)  A64_LS_IMM(Wt, Xn, imm, 8, LOAD)
0076 #define A64_STRHI(Wt, Xn, imm)  A64_LS_IMM(Wt, Xn, imm, 16, STORE)
0077 #define A64_LDRHI(Wt, Xn, imm)  A64_LS_IMM(Wt, Xn, imm, 16, LOAD)
0078 #define A64_STR32I(Wt, Xn, imm) A64_LS_IMM(Wt, Xn, imm, 32, STORE)
0079 #define A64_LDR32I(Wt, Xn, imm) A64_LS_IMM(Wt, Xn, imm, 32, LOAD)
0080 #define A64_STR64I(Xt, Xn, imm) A64_LS_IMM(Xt, Xn, imm, 64, STORE)
0081 #define A64_LDR64I(Xt, Xn, imm) A64_LS_IMM(Xt, Xn, imm, 64, LOAD)
0082
0083 /* LDR (literal) */
0084 #define A64_LDR32LIT(Wt, offset) \
0085     aarch64_insn_gen_load_literal(0, offset, Wt, false)
0086 #define A64_LDR64LIT(Xt, offset) \
0087     aarch64_insn_gen_load_literal(0, offset, Xt, true)
0088
0089 /* Load/store register pair */
0090 #define A64_LS_PAIR(Rt, Rt2, Rn, offset, ls, type) \
0091     aarch64_insn_gen_load_store_pair(Rt, Rt2, Rn, offset, \
0092         AARCH64_INSN_VARIANT_64BIT, \
0093         AARCH64_INSN_LDST_##ls##_PAIR_##type)
0094 /* Rn -= 16; Rn[0] = Rt; Rn[8] = Rt2; */
0095 #define A64_PUSH(Rt, Rt2, Rn) A64_LS_PAIR(Rt, Rt2, Rn, -16, STORE, PRE_INDEX)
0096 /* Rt = Rn[0]; Rt2 = Rn[8]; Rn += 16; */
0097 #define A64_POP(Rt, Rt2, Rn)  A64_LS_PAIR(Rt, Rt2, Rn, 16, LOAD, POST_INDEX)
0098
0099 /* Load/store exclusive */
0100 #define A64_SIZE(sf) \
0101     ((sf) ? AARCH64_INSN_SIZE_64 : AARCH64_INSN_SIZE_32)
0102 #define A64_LSX(sf, Rt, Rn, Rs, type) \
0103     aarch64_insn_gen_load_store_ex(Rt, Rn, Rs, A64_SIZE(sf), \
0104                        AARCH64_INSN_LDST_##type)
0105 /* Rt = [Rn]; (atomic) */
0106 #define A64_LDXR(sf, Rt, Rn) \
0107     A64_LSX(sf, Rt, Rn, A64_ZR, LOAD_EX)
0108 /* [Rn] = Rt; (atomic) Rs = [state] */
0109 #define A64_STXR(sf, Rt, Rn, Rs) \
0110     A64_LSX(sf, Rt, Rn, Rs, STORE_EX)
0111 /* [Rn] = Rt (store release); (atomic) Rs = [state] */
0112 #define A64_STLXR(sf, Rt, Rn, Rs) \
0113     aarch64_insn_gen_load_store_ex(Rt, Rn, Rs, A64_SIZE(sf), \
0114                        AARCH64_INSN_LDST_STORE_REL_EX)
0115
0116 /*
0117  * LSE atomics
0118  *
0119  * ST{ADD,CLR,SET,EOR} is simply encoded as an alias for
0120  * LDD{ADD,CLR,SET,EOR} with XZR as the destination register.
0121  */
0122 #define A64_ST_OP(sf, Rn, Rs, op) \
0123     aarch64_insn_gen_atomic_ld_op(A64_ZR, Rn, Rs, \
0124         A64_SIZE(sf), AARCH64_INSN_MEM_ATOMIC_##op, \
0125         AARCH64_INSN_MEM_ORDER_NONE)
0126 /* [Rn] <op>= Rs */
0127 #define A64_STADD(sf, Rn, Rs) A64_ST_OP(sf, Rn, Rs, ADD)
0128 #define A64_STCLR(sf, Rn, Rs) A64_ST_OP(sf, Rn, Rs, CLR)
0129 #define A64_STEOR(sf, Rn, Rs) A64_ST_OP(sf, Rn, Rs, EOR)
0130 #define A64_STSET(sf, Rn, Rs) A64_ST_OP(sf, Rn, Rs, SET)
0131
0132 #define A64_LD_OP_AL(sf, Rt, Rn, Rs, op) \
0133     aarch64_insn_gen_atomic_ld_op(Rt, Rn, Rs, \
0134         A64_SIZE(sf), AARCH64_INSN_MEM_ATOMIC_##op, \
0135         AARCH64_INSN_MEM_ORDER_ACQREL)
0136 /* Rt = [Rn] (load acquire); [Rn] <op>= Rs (store release) */
0137 #define A64_LDADDAL(sf, Rt, Rn, Rs) A64_LD_OP_AL(sf, Rt, Rn, Rs, ADD)
0138 #define A64_LDCLRAL(sf, Rt, Rn, Rs) A64_LD_OP_AL(sf, Rt, Rn, Rs, CLR)
0139 #define A64_LDEORAL(sf, Rt, Rn, Rs) A64_LD_OP_AL(sf, Rt, Rn, Rs, EOR)
0140 #define A64_LDSETAL(sf, Rt, Rn, Rs) A64_LD_OP_AL(sf, Rt, Rn, Rs, SET)
0141 /* Rt = [Rn] (load acquire); [Rn] = Rs (store release) */
0142 #define A64_SWPAL(sf, Rt, Rn, Rs) A64_LD_OP_AL(sf, Rt, Rn, Rs, SWP)
0143 /* Rs = CAS(Rn, Rs, Rt) (load acquire & store release) */
0144 #define A64_CASAL(sf, Rt, Rn, Rs) \
0145     aarch64_insn_gen_cas(Rt, Rn, Rs, A64_SIZE(sf), \
0146         AARCH64_INSN_MEM_ORDER_ACQREL)
0147
0148 /* Add/subtract (immediate) */
0149 #define A64_ADDSUB_IMM(sf, Rd, Rn, imm12, type) \
0150     aarch64_insn_gen_add_sub_imm(Rd, Rn, imm12, \
0151         A64_VARIANT(sf), AARCH64_INSN_ADSB_##type)
0152 /* Rd = Rn OP imm12 */
0153 #define A64_ADD_I(sf, Rd, Rn, imm12) A64_ADDSUB_IMM(sf, Rd, Rn, imm12, ADD)
0154 #define A64_SUB_I(sf, Rd, Rn, imm12) A64_ADDSUB_IMM(sf, Rd, Rn, imm12, SUB)
0155 #define A64_ADDS_I(sf, Rd, Rn, imm12) \
0156     A64_ADDSUB_IMM(sf, Rd, Rn, imm12, ADD_SETFLAGS)
0157 #define A64_SUBS_I(sf, Rd, Rn, imm12) \
0158     A64_ADDSUB_IMM(sf, Rd, Rn, imm12, SUB_SETFLAGS)
0159 /* Rn + imm12; set condition flags */
0160 #define A64_CMN_I(sf, Rn, imm12) A64_ADDS_I(sf, A64_ZR, Rn, imm12)
0161 /* Rn - imm12; set condition flags */
0162 #define A64_CMP_I(sf, Rn, imm12) A64_SUBS_I(sf, A64_ZR, Rn, imm12)
0163 /* Rd = Rn */
0164 #define A64_MOV(sf, Rd, Rn) A64_ADD_I(sf, Rd, Rn, 0)
0165
0166 /* Bitfield move */
0167 #define A64_BITFIELD(sf, Rd, Rn, immr, imms, type) \
0168     aarch64_insn_gen_bitfield(Rd, Rn, immr, imms, \
0169         A64_VARIANT(sf), AARCH64_INSN_BITFIELD_MOVE_##type)
0170 /* Signed, with sign replication to left and zeros to right */
0171 #define A64_SBFM(sf, Rd, Rn, ir, is) A64_BITFIELD(sf, Rd, Rn, ir, is, SIGNED)
0172 /* Unsigned, with zeros to left and right */
0173 #define A64_UBFM(sf, Rd, Rn, ir, is) A64_BITFIELD(sf, Rd, Rn, ir, is, UNSIGNED)
0174
0175 /* Rd = Rn << shift */
0176 #define A64_LSL(sf, Rd, Rn, shift) ({   \
0177     int sz = (sf) ? 64 : 32;    \
0178     A64_UBFM(sf, Rd, Rn, (unsigned)-(shift) % sz, sz - 1 - (shift)); \
0179 })
0180 /* Rd = Rn >> shift */
0181 #define A64_LSR(sf, Rd, Rn, shift) A64_UBFM(sf, Rd, Rn, shift, (sf) ? 63 : 31)
0182 /* Rd = Rn >> shift; signed */
0183 #define A64_ASR(sf, Rd, Rn, shift) A64_SBFM(sf, Rd, Rn, shift, (sf) ? 63 : 31)
0184
0185 /* Zero extend */
0186 #define A64_UXTH(sf, Rd, Rn) A64_UBFM(sf, Rd, Rn, 0, 15)
0187 #define A64_UXTW(sf, Rd, Rn) A64_UBFM(sf, Rd, Rn, 0, 31)
0188
0189 /* Move wide (immediate) */
0190 #define A64_MOVEW(sf, Rd, imm16, shift, type) \
0191     aarch64_insn_gen_movewide(Rd, imm16, shift, \
0192         A64_VARIANT(sf), AARCH64_INSN_MOVEWIDE_##type)
0193 /* Rd = Zeros (for MOVZ);
0194  * Rd |= imm16 << shift (where shift is {0, 16, 32, 48});
0195  * Rd = ~Rd; (for MOVN); */
0196 #define A64_MOVN(sf, Rd, imm16, shift) A64_MOVEW(sf, Rd, imm16, shift, INVERSE)
0197 #define A64_MOVZ(sf, Rd, imm16, shift) A64_MOVEW(sf, Rd, imm16, shift, ZERO)
0198 #define A64_MOVK(sf, Rd, imm16, shift) A64_MOVEW(sf, Rd, imm16, shift, KEEP)
0199
0200 /* Add/subtract (shifted register) */
0201 #define A64_ADDSUB_SREG(sf, Rd, Rn, Rm, type) \
0202     aarch64_insn_gen_add_sub_shifted_reg(Rd, Rn, Rm, 0, \
0203         A64_VARIANT(sf), AARCH64_INSN_ADSB_##type)
0204 /* Rd = Rn OP Rm */
0205 #define A64_ADD(sf, Rd, Rn, Rm)  A64_ADDSUB_SREG(sf, Rd, Rn, Rm, ADD)
0206 #define A64_SUB(sf, Rd, Rn, Rm)  A64_ADDSUB_SREG(sf, Rd, Rn, Rm, SUB)
0207 #define A64_SUBS(sf, Rd, Rn, Rm) A64_ADDSUB_SREG(sf, Rd, Rn, Rm, SUB_SETFLAGS)
0208 /* Rd = -Rm */
0209 #define A64_NEG(sf, Rd, Rm) A64_SUB(sf, Rd, A64_ZR, Rm)
0210 /* Rn - Rm; set condition flags */
0211 #define A64_CMP(sf, Rn, Rm) A64_SUBS(sf, A64_ZR, Rn, Rm)
0212
0213 /* Data-processing (1 source) */
0214 #define A64_DATA1(sf, Rd, Rn, type) aarch64_insn_gen_data1(Rd, Rn, \
0215     A64_VARIANT(sf), AARCH64_INSN_DATA1_##type)
0216 /* Rd = BSWAPx(Rn) */
0217 #define A64_REV16(sf, Rd, Rn) A64_DATA1(sf, Rd, Rn, REVERSE_16)
0218 #define A64_REV32(sf, Rd, Rn) A64_DATA1(sf, Rd, Rn, REVERSE_32)
0219 #define A64_REV64(Rd, Rn)     A64_DATA1(1, Rd, Rn, REVERSE_64)
0220
0221 /* Data-processing (2 source) */
0222 /* Rd = Rn OP Rm */
0223 #define A64_DATA2(sf, Rd, Rn, Rm, type) aarch64_insn_gen_data2(Rd, Rn, Rm, \
0224     A64_VARIANT(sf), AARCH64_INSN_DATA2_##type)
0225 #define A64_UDIV(sf, Rd, Rn, Rm) A64_DATA2(sf, Rd, Rn, Rm, UDIV)
0226 #define A64_LSLV(sf, Rd, Rn, Rm) A64_DATA2(sf, Rd, Rn, Rm, LSLV)
0227 #define A64_LSRV(sf, Rd, Rn, Rm) A64_DATA2(sf, Rd, Rn, Rm, LSRV)
0228 #define A64_ASRV(sf, Rd, Rn, Rm) A64_DATA2(sf, Rd, Rn, Rm, ASRV)
0229
0230 /* Data-processing (3 source) */
0231 /* Rd = Ra + Rn * Rm */
0232 #define A64_MADD(sf, Rd, Ra, Rn, Rm) aarch64_insn_gen_data3(Rd, Ra, Rn, Rm, \
0233     A64_VARIANT(sf), AARCH64_INSN_DATA3_MADD)
0234 /* Rd = Ra - Rn * Rm */
0235 #define A64_MSUB(sf, Rd, Ra, Rn, Rm) aarch64_insn_gen_data3(Rd, Ra, Rn, Rm, \
0236     A64_VARIANT(sf), AARCH64_INSN_DATA3_MSUB)
0237 /* Rd = Rn * Rm */
0238 #define A64_MUL(sf, Rd, Rn, Rm) A64_MADD(sf, Rd, A64_ZR, Rn, Rm)
0239
0240 /* Logical (shifted register) */
0241 #define A64_LOGIC_SREG(sf, Rd, Rn, Rm, type) \
0242     aarch64_insn_gen_logical_shifted_reg(Rd, Rn, Rm, 0, \
0243         A64_VARIANT(sf), AARCH64_INSN_LOGIC_##type)
0244 /* Rd = Rn OP Rm */
0245 #define A64_AND(sf, Rd, Rn, Rm)  A64_LOGIC_SREG(sf, Rd, Rn, Rm, AND)
0246 #define A64_ORR(sf, Rd, Rn, Rm)  A64_LOGIC_SREG(sf, Rd, Rn, Rm, ORR)
0247 #define A64_EOR(sf, Rd, Rn, Rm)  A64_LOGIC_SREG(sf, Rd, Rn, Rm, EOR)
0248 #define A64_ANDS(sf, Rd, Rn, Rm) A64_LOGIC_SREG(sf, Rd, Rn, Rm, AND_SETFLAGS)
0249 /* Rn & Rm; set condition flags */
0250 #define A64_TST(sf, Rn, Rm) A64_ANDS(sf, A64_ZR, Rn, Rm)
0251 /* Rd = ~Rm (alias of ORN with A64_ZR as Rn) */
0252 #define A64_MVN(sf, Rd, Rm)  \
0253     A64_LOGIC_SREG(sf, Rd, A64_ZR, Rm, ORN)
0254
0255 /* Logical (immediate) */
0256 #define A64_LOGIC_IMM(sf, Rd, Rn, imm, type) ({ \
0257     u64 imm64 = (sf) ? (u64)imm : (u64)(u32)imm; \
0258     aarch64_insn_gen_logical_immediate(AARCH64_INSN_LOGIC_##type, \
0259         A64_VARIANT(sf), Rn, Rd, imm64); \
0260 })
0261 /* Rd = Rn OP imm */
0262 #define A64_AND_I(sf, Rd, Rn, imm) A64_LOGIC_IMM(sf, Rd, Rn, imm, AND)
0263 #define A64_ORR_I(sf, Rd, Rn, imm) A64_LOGIC_IMM(sf, Rd, Rn, imm, ORR)
0264 #define A64_EOR_I(sf, Rd, Rn, imm) A64_LOGIC_IMM(sf, Rd, Rn, imm, EOR)
0265 #define A64_ANDS_I(sf, Rd, Rn, imm) A64_LOGIC_IMM(sf, Rd, Rn, imm, AND_SETFLAGS)
0266 /* Rn & imm; set condition flags */
0267 #define A64_TST_I(sf, Rn, imm) A64_ANDS_I(sf, A64_ZR, Rn, imm)
0268
0269 /* HINTs */
0270 #define A64_HINT(x) aarch64_insn_gen_hint(x)
0271
0272 #define A64_PACIASP A64_HINT(AARCH64_INSN_HINT_PACIASP)
0273 #define A64_AUTIASP A64_HINT(AARCH64_INSN_HINT_AUTIASP)
0274
0275 /* BTI */
0276 #define A64_BTI_C  A64_HINT(AARCH64_INSN_HINT_BTIC)
0277 #define A64_BTI_J  A64_HINT(AARCH64_INSN_HINT_BTIJ)
0278 #define A64_BTI_JC A64_HINT(AARCH64_INSN_HINT_BTIJC)
0279 #define A64_NOP    A64_HINT(AARCH64_INSN_HINT_NOP)
0280
0281 /* DMB */
0282 #define A64_DMB_ISH aarch64_insn_gen_dmb(AARCH64_INSN_MB_ISH)
0283
0284 #endif /* _BPF_JIT_H */