Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Copyright (C) 2013 Huawei Ltd.
0004  * Author: Jiang Liu <liuj97@gmail.com>
0005  *
0006  * Copyright (C) 2014-2016 Zi Shen Lim <zlim.lnx@gmail.com>
0007  */
0008 #include <linux/bitops.h>
0009 #include <linux/bug.h>
0010 #include <linux/printk.h>
0011 #include <linux/sizes.h>
0012 #include <linux/types.h>
0013 
0014 #include <asm/debug-monitors.h>
0015 #include <asm/errno.h>
0016 #include <asm/insn.h>
0017 #include <asm/kprobes.h>
0018 
0019 #define AARCH64_INSN_SF_BIT BIT(31)
0020 #define AARCH64_INSN_N_BIT  BIT(22)
0021 #define AARCH64_INSN_LSL_12 BIT(22)
0022 
0023 static const int aarch64_insn_encoding_class[] = {
0024     AARCH64_INSN_CLS_UNKNOWN,
0025     AARCH64_INSN_CLS_UNKNOWN,
0026     AARCH64_INSN_CLS_SVE,
0027     AARCH64_INSN_CLS_UNKNOWN,
0028     AARCH64_INSN_CLS_LDST,
0029     AARCH64_INSN_CLS_DP_REG,
0030     AARCH64_INSN_CLS_LDST,
0031     AARCH64_INSN_CLS_DP_FPSIMD,
0032     AARCH64_INSN_CLS_DP_IMM,
0033     AARCH64_INSN_CLS_DP_IMM,
0034     AARCH64_INSN_CLS_BR_SYS,
0035     AARCH64_INSN_CLS_BR_SYS,
0036     AARCH64_INSN_CLS_LDST,
0037     AARCH64_INSN_CLS_DP_REG,
0038     AARCH64_INSN_CLS_LDST,
0039     AARCH64_INSN_CLS_DP_FPSIMD,
0040 };
0041 
0042 enum aarch64_insn_encoding_class __kprobes aarch64_get_insn_class(u32 insn)
0043 {
0044     return aarch64_insn_encoding_class[(insn >> 25) & 0xf];
0045 }
0046 
0047 bool __kprobes aarch64_insn_is_steppable_hint(u32 insn)
0048 {
0049     if (!aarch64_insn_is_hint(insn))
0050         return false;
0051 
0052     switch (insn & 0xFE0) {
0053     case AARCH64_INSN_HINT_XPACLRI:
0054     case AARCH64_INSN_HINT_PACIA_1716:
0055     case AARCH64_INSN_HINT_PACIB_1716:
0056     case AARCH64_INSN_HINT_PACIAZ:
0057     case AARCH64_INSN_HINT_PACIASP:
0058     case AARCH64_INSN_HINT_PACIBZ:
0059     case AARCH64_INSN_HINT_PACIBSP:
0060     case AARCH64_INSN_HINT_BTI:
0061     case AARCH64_INSN_HINT_BTIC:
0062     case AARCH64_INSN_HINT_BTIJ:
0063     case AARCH64_INSN_HINT_BTIJC:
0064     case AARCH64_INSN_HINT_NOP:
0065         return true;
0066     default:
0067         return false;
0068     }
0069 }
0070 
0071 bool aarch64_insn_is_branch_imm(u32 insn)
0072 {
0073     return (aarch64_insn_is_b(insn) || aarch64_insn_is_bl(insn) ||
0074         aarch64_insn_is_tbz(insn) || aarch64_insn_is_tbnz(insn) ||
0075         aarch64_insn_is_cbz(insn) || aarch64_insn_is_cbnz(insn) ||
0076         aarch64_insn_is_bcond(insn));
0077 }
0078 
0079 bool __kprobes aarch64_insn_uses_literal(u32 insn)
0080 {
0081     /* ldr/ldrsw (literal), prfm */
0082 
0083     return aarch64_insn_is_ldr_lit(insn) ||
0084         aarch64_insn_is_ldrsw_lit(insn) ||
0085         aarch64_insn_is_adr_adrp(insn) ||
0086         aarch64_insn_is_prfm_lit(insn);
0087 }
0088 
0089 bool __kprobes aarch64_insn_is_branch(u32 insn)
0090 {
0091     /* b, bl, cb*, tb*, ret*, b.cond, br*, blr* */
0092 
0093     return aarch64_insn_is_b(insn) ||
0094         aarch64_insn_is_bl(insn) ||
0095         aarch64_insn_is_cbz(insn) ||
0096         aarch64_insn_is_cbnz(insn) ||
0097         aarch64_insn_is_tbz(insn) ||
0098         aarch64_insn_is_tbnz(insn) ||
0099         aarch64_insn_is_ret(insn) ||
0100         aarch64_insn_is_ret_auth(insn) ||
0101         aarch64_insn_is_br(insn) ||
0102         aarch64_insn_is_br_auth(insn) ||
0103         aarch64_insn_is_blr(insn) ||
0104         aarch64_insn_is_blr_auth(insn) ||
0105         aarch64_insn_is_bcond(insn);
0106 }
0107 
0108 static int __kprobes aarch64_get_imm_shift_mask(enum aarch64_insn_imm_type type,
0109                         u32 *maskp, int *shiftp)
0110 {
0111     u32 mask;
0112     int shift;
0113 
0114     switch (type) {
0115     case AARCH64_INSN_IMM_26:
0116         mask = BIT(26) - 1;
0117         shift = 0;
0118         break;
0119     case AARCH64_INSN_IMM_19:
0120         mask = BIT(19) - 1;
0121         shift = 5;
0122         break;
0123     case AARCH64_INSN_IMM_16:
0124         mask = BIT(16) - 1;
0125         shift = 5;
0126         break;
0127     case AARCH64_INSN_IMM_14:
0128         mask = BIT(14) - 1;
0129         shift = 5;
0130         break;
0131     case AARCH64_INSN_IMM_12:
0132         mask = BIT(12) - 1;
0133         shift = 10;
0134         break;
0135     case AARCH64_INSN_IMM_9:
0136         mask = BIT(9) - 1;
0137         shift = 12;
0138         break;
0139     case AARCH64_INSN_IMM_7:
0140         mask = BIT(7) - 1;
0141         shift = 15;
0142         break;
0143     case AARCH64_INSN_IMM_6:
0144     case AARCH64_INSN_IMM_S:
0145         mask = BIT(6) - 1;
0146         shift = 10;
0147         break;
0148     case AARCH64_INSN_IMM_R:
0149         mask = BIT(6) - 1;
0150         shift = 16;
0151         break;
0152     case AARCH64_INSN_IMM_N:
0153         mask = 1;
0154         shift = 22;
0155         break;
0156     default:
0157         return -EINVAL;
0158     }
0159 
0160     *maskp = mask;
0161     *shiftp = shift;
0162 
0163     return 0;
0164 }
0165 
0166 #define ADR_IMM_HILOSPLIT   2
0167 #define ADR_IMM_SIZE        SZ_2M
0168 #define ADR_IMM_LOMASK      ((1 << ADR_IMM_HILOSPLIT) - 1)
0169 #define ADR_IMM_HIMASK      ((ADR_IMM_SIZE >> ADR_IMM_HILOSPLIT) - 1)
0170 #define ADR_IMM_LOSHIFT     29
0171 #define ADR_IMM_HISHIFT     5
0172 
0173 u64 aarch64_insn_decode_immediate(enum aarch64_insn_imm_type type, u32 insn)
0174 {
0175     u32 immlo, immhi, mask;
0176     int shift;
0177 
0178     switch (type) {
0179     case AARCH64_INSN_IMM_ADR:
0180         shift = 0;
0181         immlo = (insn >> ADR_IMM_LOSHIFT) & ADR_IMM_LOMASK;
0182         immhi = (insn >> ADR_IMM_HISHIFT) & ADR_IMM_HIMASK;
0183         insn = (immhi << ADR_IMM_HILOSPLIT) | immlo;
0184         mask = ADR_IMM_SIZE - 1;
0185         break;
0186     default:
0187         if (aarch64_get_imm_shift_mask(type, &mask, &shift) < 0) {
0188             pr_err("%s: unknown immediate encoding %d\n", __func__,
0189                    type);
0190             return 0;
0191         }
0192     }
0193 
0194     return (insn >> shift) & mask;
0195 }
0196 
0197 u32 __kprobes aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type,
0198                   u32 insn, u64 imm)
0199 {
0200     u32 immlo, immhi, mask;
0201     int shift;
0202 
0203     if (insn == AARCH64_BREAK_FAULT)
0204         return AARCH64_BREAK_FAULT;
0205 
0206     switch (type) {
0207     case AARCH64_INSN_IMM_ADR:
0208         shift = 0;
0209         immlo = (imm & ADR_IMM_LOMASK) << ADR_IMM_LOSHIFT;
0210         imm >>= ADR_IMM_HILOSPLIT;
0211         immhi = (imm & ADR_IMM_HIMASK) << ADR_IMM_HISHIFT;
0212         imm = immlo | immhi;
0213         mask = ((ADR_IMM_LOMASK << ADR_IMM_LOSHIFT) |
0214             (ADR_IMM_HIMASK << ADR_IMM_HISHIFT));
0215         break;
0216     default:
0217         if (aarch64_get_imm_shift_mask(type, &mask, &shift) < 0) {
0218             pr_err("%s: unknown immediate encoding %d\n", __func__,
0219                    type);
0220             return AARCH64_BREAK_FAULT;
0221         }
0222     }
0223 
0224     /* Update the immediate field. */
0225     insn &= ~(mask << shift);
0226     insn |= (imm & mask) << shift;
0227 
0228     return insn;
0229 }
0230 
0231 u32 aarch64_insn_decode_register(enum aarch64_insn_register_type type,
0232                     u32 insn)
0233 {
0234     int shift;
0235 
0236     switch (type) {
0237     case AARCH64_INSN_REGTYPE_RT:
0238     case AARCH64_INSN_REGTYPE_RD:
0239         shift = 0;
0240         break;
0241     case AARCH64_INSN_REGTYPE_RN:
0242         shift = 5;
0243         break;
0244     case AARCH64_INSN_REGTYPE_RT2:
0245     case AARCH64_INSN_REGTYPE_RA:
0246         shift = 10;
0247         break;
0248     case AARCH64_INSN_REGTYPE_RM:
0249         shift = 16;
0250         break;
0251     default:
0252         pr_err("%s: unknown register type encoding %d\n", __func__,
0253                type);
0254         return 0;
0255     }
0256 
0257     return (insn >> shift) & GENMASK(4, 0);
0258 }
0259 
0260 static u32 aarch64_insn_encode_register(enum aarch64_insn_register_type type,
0261                     u32 insn,
0262                     enum aarch64_insn_register reg)
0263 {
0264     int shift;
0265 
0266     if (insn == AARCH64_BREAK_FAULT)
0267         return AARCH64_BREAK_FAULT;
0268 
0269     if (reg < AARCH64_INSN_REG_0 || reg > AARCH64_INSN_REG_SP) {
0270         pr_err("%s: unknown register encoding %d\n", __func__, reg);
0271         return AARCH64_BREAK_FAULT;
0272     }
0273 
0274     switch (type) {
0275     case AARCH64_INSN_REGTYPE_RT:
0276     case AARCH64_INSN_REGTYPE_RD:
0277         shift = 0;
0278         break;
0279     case AARCH64_INSN_REGTYPE_RN:
0280         shift = 5;
0281         break;
0282     case AARCH64_INSN_REGTYPE_RT2:
0283     case AARCH64_INSN_REGTYPE_RA:
0284         shift = 10;
0285         break;
0286     case AARCH64_INSN_REGTYPE_RM:
0287     case AARCH64_INSN_REGTYPE_RS:
0288         shift = 16;
0289         break;
0290     default:
0291         pr_err("%s: unknown register type encoding %d\n", __func__,
0292                type);
0293         return AARCH64_BREAK_FAULT;
0294     }
0295 
0296     insn &= ~(GENMASK(4, 0) << shift);
0297     insn |= reg << shift;
0298 
0299     return insn;
0300 }
0301 
0302 static const u32 aarch64_insn_ldst_size[] = {
0303     [AARCH64_INSN_SIZE_8] = 0,
0304     [AARCH64_INSN_SIZE_16] = 1,
0305     [AARCH64_INSN_SIZE_32] = 2,
0306     [AARCH64_INSN_SIZE_64] = 3,
0307 };
0308 
0309 static u32 aarch64_insn_encode_ldst_size(enum aarch64_insn_size_type type,
0310                      u32 insn)
0311 {
0312     u32 size;
0313 
0314     if (type < AARCH64_INSN_SIZE_8 || type > AARCH64_INSN_SIZE_64) {
0315         pr_err("%s: unknown size encoding %d\n", __func__, type);
0316         return AARCH64_BREAK_FAULT;
0317     }
0318 
0319     size = aarch64_insn_ldst_size[type];
0320     insn &= ~GENMASK(31, 30);
0321     insn |= size << 30;
0322 
0323     return insn;
0324 }
0325 
0326 static inline long label_imm_common(unsigned long pc, unsigned long addr,
0327                      long range)
0328 {
0329     long offset;
0330 
0331     if ((pc & 0x3) || (addr & 0x3)) {
0332         pr_err("%s: A64 instructions must be word aligned\n", __func__);
0333         return range;
0334     }
0335 
0336     offset = ((long)addr - (long)pc);
0337 
0338     if (offset < -range || offset >= range) {
0339         pr_err("%s: offset out of range\n", __func__);
0340         return range;
0341     }
0342 
0343     return offset;
0344 }
0345 
0346 u32 __kprobes aarch64_insn_gen_branch_imm(unsigned long pc, unsigned long addr,
0347                       enum aarch64_insn_branch_type type)
0348 {
0349     u32 insn;
0350     long offset;
0351 
0352     /*
0353      * B/BL support [-128M, 128M) offset
0354      * ARM64 virtual address arrangement guarantees all kernel and module
0355      * texts are within +/-128M.
0356      */
0357     offset = label_imm_common(pc, addr, SZ_128M);
0358     if (offset >= SZ_128M)
0359         return AARCH64_BREAK_FAULT;
0360 
0361     switch (type) {
0362     case AARCH64_INSN_BRANCH_LINK:
0363         insn = aarch64_insn_get_bl_value();
0364         break;
0365     case AARCH64_INSN_BRANCH_NOLINK:
0366         insn = aarch64_insn_get_b_value();
0367         break;
0368     default:
0369         pr_err("%s: unknown branch encoding %d\n", __func__, type);
0370         return AARCH64_BREAK_FAULT;
0371     }
0372 
0373     return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_26, insn,
0374                          offset >> 2);
0375 }
0376 
0377 u32 aarch64_insn_gen_comp_branch_imm(unsigned long pc, unsigned long addr,
0378                      enum aarch64_insn_register reg,
0379                      enum aarch64_insn_variant variant,
0380                      enum aarch64_insn_branch_type type)
0381 {
0382     u32 insn;
0383     long offset;
0384 
0385     offset = label_imm_common(pc, addr, SZ_1M);
0386     if (offset >= SZ_1M)
0387         return AARCH64_BREAK_FAULT;
0388 
0389     switch (type) {
0390     case AARCH64_INSN_BRANCH_COMP_ZERO:
0391         insn = aarch64_insn_get_cbz_value();
0392         break;
0393     case AARCH64_INSN_BRANCH_COMP_NONZERO:
0394         insn = aarch64_insn_get_cbnz_value();
0395         break;
0396     default:
0397         pr_err("%s: unknown branch encoding %d\n", __func__, type);
0398         return AARCH64_BREAK_FAULT;
0399     }
0400 
0401     switch (variant) {
0402     case AARCH64_INSN_VARIANT_32BIT:
0403         break;
0404     case AARCH64_INSN_VARIANT_64BIT:
0405         insn |= AARCH64_INSN_SF_BIT;
0406         break;
0407     default:
0408         pr_err("%s: unknown variant encoding %d\n", __func__, variant);
0409         return AARCH64_BREAK_FAULT;
0410     }
0411 
0412     insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn, reg);
0413 
0414     return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_19, insn,
0415                          offset >> 2);
0416 }
0417 
0418 u32 aarch64_insn_gen_cond_branch_imm(unsigned long pc, unsigned long addr,
0419                      enum aarch64_insn_condition cond)
0420 {
0421     u32 insn;
0422     long offset;
0423 
0424     offset = label_imm_common(pc, addr, SZ_1M);
0425 
0426     insn = aarch64_insn_get_bcond_value();
0427 
0428     if (cond < AARCH64_INSN_COND_EQ || cond > AARCH64_INSN_COND_AL) {
0429         pr_err("%s: unknown condition encoding %d\n", __func__, cond);
0430         return AARCH64_BREAK_FAULT;
0431     }
0432     insn |= cond;
0433 
0434     return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_19, insn,
0435                          offset >> 2);
0436 }
0437 
0438 u32 __kprobes aarch64_insn_gen_hint(enum aarch64_insn_hint_cr_op op)
0439 {
0440     return aarch64_insn_get_hint_value() | op;
0441 }
0442 
0443 u32 __kprobes aarch64_insn_gen_nop(void)
0444 {
0445     return aarch64_insn_gen_hint(AARCH64_INSN_HINT_NOP);
0446 }
0447 
0448 u32 aarch64_insn_gen_branch_reg(enum aarch64_insn_register reg,
0449                 enum aarch64_insn_branch_type type)
0450 {
0451     u32 insn;
0452 
0453     switch (type) {
0454     case AARCH64_INSN_BRANCH_NOLINK:
0455         insn = aarch64_insn_get_br_value();
0456         break;
0457     case AARCH64_INSN_BRANCH_LINK:
0458         insn = aarch64_insn_get_blr_value();
0459         break;
0460     case AARCH64_INSN_BRANCH_RETURN:
0461         insn = aarch64_insn_get_ret_value();
0462         break;
0463     default:
0464         pr_err("%s: unknown branch encoding %d\n", __func__, type);
0465         return AARCH64_BREAK_FAULT;
0466     }
0467 
0468     return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, reg);
0469 }
0470 
0471 u32 aarch64_insn_gen_load_store_reg(enum aarch64_insn_register reg,
0472                     enum aarch64_insn_register base,
0473                     enum aarch64_insn_register offset,
0474                     enum aarch64_insn_size_type size,
0475                     enum aarch64_insn_ldst_type type)
0476 {
0477     u32 insn;
0478 
0479     switch (type) {
0480     case AARCH64_INSN_LDST_LOAD_REG_OFFSET:
0481         insn = aarch64_insn_get_ldr_reg_value();
0482         break;
0483     case AARCH64_INSN_LDST_STORE_REG_OFFSET:
0484         insn = aarch64_insn_get_str_reg_value();
0485         break;
0486     default:
0487         pr_err("%s: unknown load/store encoding %d\n", __func__, type);
0488         return AARCH64_BREAK_FAULT;
0489     }
0490 
0491     insn = aarch64_insn_encode_ldst_size(size, insn);
0492 
0493     insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn, reg);
0494 
0495     insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
0496                         base);
0497 
0498     return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn,
0499                         offset);
0500 }
0501 
0502 u32 aarch64_insn_gen_load_store_imm(enum aarch64_insn_register reg,
0503                     enum aarch64_insn_register base,
0504                     unsigned int imm,
0505                     enum aarch64_insn_size_type size,
0506                     enum aarch64_insn_ldst_type type)
0507 {
0508     u32 insn;
0509     u32 shift;
0510 
0511     if (size < AARCH64_INSN_SIZE_8 || size > AARCH64_INSN_SIZE_64) {
0512         pr_err("%s: unknown size encoding %d\n", __func__, type);
0513         return AARCH64_BREAK_FAULT;
0514     }
0515 
0516     shift = aarch64_insn_ldst_size[size];
0517     if (imm & ~(BIT(12 + shift) - BIT(shift))) {
0518         pr_err("%s: invalid imm: %d\n", __func__, imm);
0519         return AARCH64_BREAK_FAULT;
0520     }
0521 
0522     imm >>= shift;
0523 
0524     switch (type) {
0525     case AARCH64_INSN_LDST_LOAD_IMM_OFFSET:
0526         insn = aarch64_insn_get_ldr_imm_value();
0527         break;
0528     case AARCH64_INSN_LDST_STORE_IMM_OFFSET:
0529         insn = aarch64_insn_get_str_imm_value();
0530         break;
0531     default:
0532         pr_err("%s: unknown load/store encoding %d\n", __func__, type);
0533         return AARCH64_BREAK_FAULT;
0534     }
0535 
0536     insn = aarch64_insn_encode_ldst_size(size, insn);
0537 
0538     insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn, reg);
0539 
0540     insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
0541                         base);
0542 
0543     return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_12, insn, imm);
0544 }
0545 
0546 u32 aarch64_insn_gen_load_literal(unsigned long pc, unsigned long addr,
0547                   enum aarch64_insn_register reg,
0548                   bool is64bit)
0549 {
0550     u32 insn;
0551     long offset;
0552 
0553     offset = label_imm_common(pc, addr, SZ_1M);
0554     if (offset >= SZ_1M)
0555         return AARCH64_BREAK_FAULT;
0556 
0557     insn = aarch64_insn_get_ldr_lit_value();
0558 
0559     if (is64bit)
0560         insn |= BIT(30);
0561 
0562     insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn, reg);
0563 
0564     return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_19, insn,
0565                          offset >> 2);
0566 }
0567 
0568 u32 aarch64_insn_gen_load_store_pair(enum aarch64_insn_register reg1,
0569                      enum aarch64_insn_register reg2,
0570                      enum aarch64_insn_register base,
0571                      int offset,
0572                      enum aarch64_insn_variant variant,
0573                      enum aarch64_insn_ldst_type type)
0574 {
0575     u32 insn;
0576     int shift;
0577 
0578     switch (type) {
0579     case AARCH64_INSN_LDST_LOAD_PAIR_PRE_INDEX:
0580         insn = aarch64_insn_get_ldp_pre_value();
0581         break;
0582     case AARCH64_INSN_LDST_STORE_PAIR_PRE_INDEX:
0583         insn = aarch64_insn_get_stp_pre_value();
0584         break;
0585     case AARCH64_INSN_LDST_LOAD_PAIR_POST_INDEX:
0586         insn = aarch64_insn_get_ldp_post_value();
0587         break;
0588     case AARCH64_INSN_LDST_STORE_PAIR_POST_INDEX:
0589         insn = aarch64_insn_get_stp_post_value();
0590         break;
0591     default:
0592         pr_err("%s: unknown load/store encoding %d\n", __func__, type);
0593         return AARCH64_BREAK_FAULT;
0594     }
0595 
0596     switch (variant) {
0597     case AARCH64_INSN_VARIANT_32BIT:
0598         if ((offset & 0x3) || (offset < -256) || (offset > 252)) {
0599             pr_err("%s: offset must be multiples of 4 in the range of [-256, 252] %d\n",
0600                    __func__, offset);
0601             return AARCH64_BREAK_FAULT;
0602         }
0603         shift = 2;
0604         break;
0605     case AARCH64_INSN_VARIANT_64BIT:
0606         if ((offset & 0x7) || (offset < -512) || (offset > 504)) {
0607             pr_err("%s: offset must be multiples of 8 in the range of [-512, 504] %d\n",
0608                    __func__, offset);
0609             return AARCH64_BREAK_FAULT;
0610         }
0611         shift = 3;
0612         insn |= AARCH64_INSN_SF_BIT;
0613         break;
0614     default:
0615         pr_err("%s: unknown variant encoding %d\n", __func__, variant);
0616         return AARCH64_BREAK_FAULT;
0617     }
0618 
0619     insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn,
0620                         reg1);
0621 
0622     insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT2, insn,
0623                         reg2);
0624 
0625     insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
0626                         base);
0627 
0628     return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_7, insn,
0629                          offset >> shift);
0630 }
0631 
0632 u32 aarch64_insn_gen_load_store_ex(enum aarch64_insn_register reg,
0633                    enum aarch64_insn_register base,
0634                    enum aarch64_insn_register state,
0635                    enum aarch64_insn_size_type size,
0636                    enum aarch64_insn_ldst_type type)
0637 {
0638     u32 insn;
0639 
0640     switch (type) {
0641     case AARCH64_INSN_LDST_LOAD_EX:
0642     case AARCH64_INSN_LDST_LOAD_ACQ_EX:
0643         insn = aarch64_insn_get_load_ex_value();
0644         if (type == AARCH64_INSN_LDST_LOAD_ACQ_EX)
0645             insn |= BIT(15);
0646         break;
0647     case AARCH64_INSN_LDST_STORE_EX:
0648     case AARCH64_INSN_LDST_STORE_REL_EX:
0649         insn = aarch64_insn_get_store_ex_value();
0650         if (type == AARCH64_INSN_LDST_STORE_REL_EX)
0651             insn |= BIT(15);
0652         break;
0653     default:
0654         pr_err("%s: unknown load/store exclusive encoding %d\n", __func__, type);
0655         return AARCH64_BREAK_FAULT;
0656     }
0657 
0658     insn = aarch64_insn_encode_ldst_size(size, insn);
0659 
0660     insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn,
0661                         reg);
0662 
0663     insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
0664                         base);
0665 
0666     insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT2, insn,
0667                         AARCH64_INSN_REG_ZR);
0668 
0669     return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RS, insn,
0670                         state);
0671 }
0672 
0673 #ifdef CONFIG_ARM64_LSE_ATOMICS
0674 static u32 aarch64_insn_encode_ldst_order(enum aarch64_insn_mem_order_type type,
0675                       u32 insn)
0676 {
0677     u32 order;
0678 
0679     switch (type) {
0680     case AARCH64_INSN_MEM_ORDER_NONE:
0681         order = 0;
0682         break;
0683     case AARCH64_INSN_MEM_ORDER_ACQ:
0684         order = 2;
0685         break;
0686     case AARCH64_INSN_MEM_ORDER_REL:
0687         order = 1;
0688         break;
0689     case AARCH64_INSN_MEM_ORDER_ACQREL:
0690         order = 3;
0691         break;
0692     default:
0693         pr_err("%s: unknown mem order %d\n", __func__, type);
0694         return AARCH64_BREAK_FAULT;
0695     }
0696 
0697     insn &= ~GENMASK(23, 22);
0698     insn |= order << 22;
0699 
0700     return insn;
0701 }
0702 
0703 u32 aarch64_insn_gen_atomic_ld_op(enum aarch64_insn_register result,
0704                   enum aarch64_insn_register address,
0705                   enum aarch64_insn_register value,
0706                   enum aarch64_insn_size_type size,
0707                   enum aarch64_insn_mem_atomic_op op,
0708                   enum aarch64_insn_mem_order_type order)
0709 {
0710     u32 insn;
0711 
0712     switch (op) {
0713     case AARCH64_INSN_MEM_ATOMIC_ADD:
0714         insn = aarch64_insn_get_ldadd_value();
0715         break;
0716     case AARCH64_INSN_MEM_ATOMIC_CLR:
0717         insn = aarch64_insn_get_ldclr_value();
0718         break;
0719     case AARCH64_INSN_MEM_ATOMIC_EOR:
0720         insn = aarch64_insn_get_ldeor_value();
0721         break;
0722     case AARCH64_INSN_MEM_ATOMIC_SET:
0723         insn = aarch64_insn_get_ldset_value();
0724         break;
0725     case AARCH64_INSN_MEM_ATOMIC_SWP:
0726         insn = aarch64_insn_get_swp_value();
0727         break;
0728     default:
0729         pr_err("%s: unimplemented mem atomic op %d\n", __func__, op);
0730         return AARCH64_BREAK_FAULT;
0731     }
0732 
0733     switch (size) {
0734     case AARCH64_INSN_SIZE_32:
0735     case AARCH64_INSN_SIZE_64:
0736         break;
0737     default:
0738         pr_err("%s: unimplemented size encoding %d\n", __func__, size);
0739         return AARCH64_BREAK_FAULT;
0740     }
0741 
0742     insn = aarch64_insn_encode_ldst_size(size, insn);
0743 
0744     insn = aarch64_insn_encode_ldst_order(order, insn);
0745 
0746     insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn,
0747                         result);
0748 
0749     insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
0750                         address);
0751 
0752     return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RS, insn,
0753                         value);
0754 }
0755 
0756 static u32 aarch64_insn_encode_cas_order(enum aarch64_insn_mem_order_type type,
0757                      u32 insn)
0758 {
0759     u32 order;
0760 
0761     switch (type) {
0762     case AARCH64_INSN_MEM_ORDER_NONE:
0763         order = 0;
0764         break;
0765     case AARCH64_INSN_MEM_ORDER_ACQ:
0766         order = BIT(22);
0767         break;
0768     case AARCH64_INSN_MEM_ORDER_REL:
0769         order = BIT(15);
0770         break;
0771     case AARCH64_INSN_MEM_ORDER_ACQREL:
0772         order = BIT(15) | BIT(22);
0773         break;
0774     default:
0775         pr_err("%s: unknown mem order %d\n", __func__, type);
0776         return AARCH64_BREAK_FAULT;
0777     }
0778 
0779     insn &= ~(BIT(15) | BIT(22));
0780     insn |= order;
0781 
0782     return insn;
0783 }
0784 
0785 u32 aarch64_insn_gen_cas(enum aarch64_insn_register result,
0786              enum aarch64_insn_register address,
0787              enum aarch64_insn_register value,
0788              enum aarch64_insn_size_type size,
0789              enum aarch64_insn_mem_order_type order)
0790 {
0791     u32 insn;
0792 
0793     switch (size) {
0794     case AARCH64_INSN_SIZE_32:
0795     case AARCH64_INSN_SIZE_64:
0796         break;
0797     default:
0798         pr_err("%s: unimplemented size encoding %d\n", __func__, size);
0799         return AARCH64_BREAK_FAULT;
0800     }
0801 
0802     insn = aarch64_insn_get_cas_value();
0803 
0804     insn = aarch64_insn_encode_ldst_size(size, insn);
0805 
0806     insn = aarch64_insn_encode_cas_order(order, insn);
0807 
0808     insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn,
0809                         result);
0810 
0811     insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
0812                         address);
0813 
0814     return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RS, insn,
0815                         value);
0816 }
0817 #endif
0818 
0819 static u32 aarch64_insn_encode_prfm_imm(enum aarch64_insn_prfm_type type,
0820                     enum aarch64_insn_prfm_target target,
0821                     enum aarch64_insn_prfm_policy policy,
0822                     u32 insn)
0823 {
0824     u32 imm_type = 0, imm_target = 0, imm_policy = 0;
0825 
0826     switch (type) {
0827     case AARCH64_INSN_PRFM_TYPE_PLD:
0828         break;
0829     case AARCH64_INSN_PRFM_TYPE_PLI:
0830         imm_type = BIT(0);
0831         break;
0832     case AARCH64_INSN_PRFM_TYPE_PST:
0833         imm_type = BIT(1);
0834         break;
0835     default:
0836         pr_err("%s: unknown prfm type encoding %d\n", __func__, type);
0837         return AARCH64_BREAK_FAULT;
0838     }
0839 
0840     switch (target) {
0841     case AARCH64_INSN_PRFM_TARGET_L1:
0842         break;
0843     case AARCH64_INSN_PRFM_TARGET_L2:
0844         imm_target = BIT(0);
0845         break;
0846     case AARCH64_INSN_PRFM_TARGET_L3:
0847         imm_target = BIT(1);
0848         break;
0849     default:
0850         pr_err("%s: unknown prfm target encoding %d\n", __func__, target);
0851         return AARCH64_BREAK_FAULT;
0852     }
0853 
0854     switch (policy) {
0855     case AARCH64_INSN_PRFM_POLICY_KEEP:
0856         break;
0857     case AARCH64_INSN_PRFM_POLICY_STRM:
0858         imm_policy = BIT(0);
0859         break;
0860     default:
0861         pr_err("%s: unknown prfm policy encoding %d\n", __func__, policy);
0862         return AARCH64_BREAK_FAULT;
0863     }
0864 
0865     /* In this case, imm5 is encoded into Rt field. */
0866     insn &= ~GENMASK(4, 0);
0867     insn |= imm_policy | (imm_target << 1) | (imm_type << 3);
0868 
0869     return insn;
0870 }
0871 
0872 u32 aarch64_insn_gen_prefetch(enum aarch64_insn_register base,
0873                   enum aarch64_insn_prfm_type type,
0874                   enum aarch64_insn_prfm_target target,
0875                   enum aarch64_insn_prfm_policy policy)
0876 {
0877     u32 insn = aarch64_insn_get_prfm_value();
0878 
0879     insn = aarch64_insn_encode_ldst_size(AARCH64_INSN_SIZE_64, insn);
0880 
0881     insn = aarch64_insn_encode_prfm_imm(type, target, policy, insn);
0882 
0883     insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
0884                         base);
0885 
0886     return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_12, insn, 0);
0887 }
0888 
0889 u32 aarch64_insn_gen_add_sub_imm(enum aarch64_insn_register dst,
0890                  enum aarch64_insn_register src,
0891                  int imm, enum aarch64_insn_variant variant,
0892                  enum aarch64_insn_adsb_type type)
0893 {
0894     u32 insn;
0895 
0896     switch (type) {
0897     case AARCH64_INSN_ADSB_ADD:
0898         insn = aarch64_insn_get_add_imm_value();
0899         break;
0900     case AARCH64_INSN_ADSB_SUB:
0901         insn = aarch64_insn_get_sub_imm_value();
0902         break;
0903     case AARCH64_INSN_ADSB_ADD_SETFLAGS:
0904         insn = aarch64_insn_get_adds_imm_value();
0905         break;
0906     case AARCH64_INSN_ADSB_SUB_SETFLAGS:
0907         insn = aarch64_insn_get_subs_imm_value();
0908         break;
0909     default:
0910         pr_err("%s: unknown add/sub encoding %d\n", __func__, type);
0911         return AARCH64_BREAK_FAULT;
0912     }
0913 
0914     switch (variant) {
0915     case AARCH64_INSN_VARIANT_32BIT:
0916         break;
0917     case AARCH64_INSN_VARIANT_64BIT:
0918         insn |= AARCH64_INSN_SF_BIT;
0919         break;
0920     default:
0921         pr_err("%s: unknown variant encoding %d\n", __func__, variant);
0922         return AARCH64_BREAK_FAULT;
0923     }
0924 
0925     /* We can't encode more than a 24bit value (12bit + 12bit shift) */
0926     if (imm & ~(BIT(24) - 1))
0927         goto out;
0928 
0929     /* If we have something in the top 12 bits... */
0930     if (imm & ~(SZ_4K - 1)) {
0931         /* ... and in the low 12 bits -> error */
0932         if (imm & (SZ_4K - 1))
0933             goto out;
0934 
0935         imm >>= 12;
0936         insn |= AARCH64_INSN_LSL_12;
0937     }
0938 
0939     insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
0940 
0941     insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src);
0942 
0943     return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_12, insn, imm);
0944 
0945 out:
0946     pr_err("%s: invalid immediate encoding %d\n", __func__, imm);
0947     return AARCH64_BREAK_FAULT;
0948 }
0949 
0950 u32 aarch64_insn_gen_bitfield(enum aarch64_insn_register dst,
0951                   enum aarch64_insn_register src,
0952                   int immr, int imms,
0953                   enum aarch64_insn_variant variant,
0954                   enum aarch64_insn_bitfield_type type)
0955 {
0956     u32 insn;
0957     u32 mask;
0958 
0959     switch (type) {
0960     case AARCH64_INSN_BITFIELD_MOVE:
0961         insn = aarch64_insn_get_bfm_value();
0962         break;
0963     case AARCH64_INSN_BITFIELD_MOVE_UNSIGNED:
0964         insn = aarch64_insn_get_ubfm_value();
0965         break;
0966     case AARCH64_INSN_BITFIELD_MOVE_SIGNED:
0967         insn = aarch64_insn_get_sbfm_value();
0968         break;
0969     default:
0970         pr_err("%s: unknown bitfield encoding %d\n", __func__, type);
0971         return AARCH64_BREAK_FAULT;
0972     }
0973 
0974     switch (variant) {
0975     case AARCH64_INSN_VARIANT_32BIT:
0976         mask = GENMASK(4, 0);
0977         break;
0978     case AARCH64_INSN_VARIANT_64BIT:
0979         insn |= AARCH64_INSN_SF_BIT | AARCH64_INSN_N_BIT;
0980         mask = GENMASK(5, 0);
0981         break;
0982     default:
0983         pr_err("%s: unknown variant encoding %d\n", __func__, variant);
0984         return AARCH64_BREAK_FAULT;
0985     }
0986 
0987     if (immr & ~mask) {
0988         pr_err("%s: invalid immr encoding %d\n", __func__, immr);
0989         return AARCH64_BREAK_FAULT;
0990     }
0991     if (imms & ~mask) {
0992         pr_err("%s: invalid imms encoding %d\n", __func__, imms);
0993         return AARCH64_BREAK_FAULT;
0994     }
0995 
0996     insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
0997 
0998     insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src);
0999 
1000     insn = aarch64_insn_encode_immediate(AARCH64_INSN_IMM_R, insn, immr);
1001 
1002     return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_S, insn, imms);
1003 }
1004 
1005 u32 aarch64_insn_gen_movewide(enum aarch64_insn_register dst,
1006                   int imm, int shift,
1007                   enum aarch64_insn_variant variant,
1008                   enum aarch64_insn_movewide_type type)
1009 {
1010     u32 insn;
1011 
1012     switch (type) {
1013     case AARCH64_INSN_MOVEWIDE_ZERO:
1014         insn = aarch64_insn_get_movz_value();
1015         break;
1016     case AARCH64_INSN_MOVEWIDE_KEEP:
1017         insn = aarch64_insn_get_movk_value();
1018         break;
1019     case AARCH64_INSN_MOVEWIDE_INVERSE:
1020         insn = aarch64_insn_get_movn_value();
1021         break;
1022     default:
1023         pr_err("%s: unknown movewide encoding %d\n", __func__, type);
1024         return AARCH64_BREAK_FAULT;
1025     }
1026 
1027     if (imm & ~(SZ_64K - 1)) {
1028         pr_err("%s: invalid immediate encoding %d\n", __func__, imm);
1029         return AARCH64_BREAK_FAULT;
1030     }
1031 
1032     switch (variant) {
1033     case AARCH64_INSN_VARIANT_32BIT:
1034         if (shift != 0 && shift != 16) {
1035             pr_err("%s: invalid shift encoding %d\n", __func__,
1036                    shift);
1037             return AARCH64_BREAK_FAULT;
1038         }
1039         break;
1040     case AARCH64_INSN_VARIANT_64BIT:
1041         insn |= AARCH64_INSN_SF_BIT;
1042         if (shift != 0 && shift != 16 && shift != 32 && shift != 48) {
1043             pr_err("%s: invalid shift encoding %d\n", __func__,
1044                    shift);
1045             return AARCH64_BREAK_FAULT;
1046         }
1047         break;
1048     default:
1049         pr_err("%s: unknown variant encoding %d\n", __func__, variant);
1050         return AARCH64_BREAK_FAULT;
1051     }
1052 
1053     insn |= (shift >> 4) << 21;
1054 
1055     insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
1056 
1057     return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_16, insn, imm);
1058 }
1059 
1060 u32 aarch64_insn_gen_add_sub_shifted_reg(enum aarch64_insn_register dst,
1061                      enum aarch64_insn_register src,
1062                      enum aarch64_insn_register reg,
1063                      int shift,
1064                      enum aarch64_insn_variant variant,
1065                      enum aarch64_insn_adsb_type type)
1066 {
1067     u32 insn;
1068 
1069     switch (type) {
1070     case AARCH64_INSN_ADSB_ADD:
1071         insn = aarch64_insn_get_add_value();
1072         break;
1073     case AARCH64_INSN_ADSB_SUB:
1074         insn = aarch64_insn_get_sub_value();
1075         break;
1076     case AARCH64_INSN_ADSB_ADD_SETFLAGS:
1077         insn = aarch64_insn_get_adds_value();
1078         break;
1079     case AARCH64_INSN_ADSB_SUB_SETFLAGS:
1080         insn = aarch64_insn_get_subs_value();
1081         break;
1082     default:
1083         pr_err("%s: unknown add/sub encoding %d\n", __func__, type);
1084         return AARCH64_BREAK_FAULT;
1085     }
1086 
1087     switch (variant) {
1088     case AARCH64_INSN_VARIANT_32BIT:
1089         if (shift & ~(SZ_32 - 1)) {
1090             pr_err("%s: invalid shift encoding %d\n", __func__,
1091                    shift);
1092             return AARCH64_BREAK_FAULT;
1093         }
1094         break;
1095     case AARCH64_INSN_VARIANT_64BIT:
1096         insn |= AARCH64_INSN_SF_BIT;
1097         if (shift & ~(SZ_64 - 1)) {
1098             pr_err("%s: invalid shift encoding %d\n", __func__,
1099                    shift);
1100             return AARCH64_BREAK_FAULT;
1101         }
1102         break;
1103     default:
1104         pr_err("%s: unknown variant encoding %d\n", __func__, variant);
1105         return AARCH64_BREAK_FAULT;
1106     }
1107 
1108 
1109     insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
1110 
1111     insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src);
1112 
1113     insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, reg);
1114 
1115     return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_6, insn, shift);
1116 }
1117 
1118 u32 aarch64_insn_gen_data1(enum aarch64_insn_register dst,
1119                enum aarch64_insn_register src,
1120                enum aarch64_insn_variant variant,
1121                enum aarch64_insn_data1_type type)
1122 {
1123     u32 insn;
1124 
1125     switch (type) {
1126     case AARCH64_INSN_DATA1_REVERSE_16:
1127         insn = aarch64_insn_get_rev16_value();
1128         break;
1129     case AARCH64_INSN_DATA1_REVERSE_32:
1130         insn = aarch64_insn_get_rev32_value();
1131         break;
1132     case AARCH64_INSN_DATA1_REVERSE_64:
1133         if (variant != AARCH64_INSN_VARIANT_64BIT) {
1134             pr_err("%s: invalid variant for reverse64 %d\n",
1135                    __func__, variant);
1136             return AARCH64_BREAK_FAULT;
1137         }
1138         insn = aarch64_insn_get_rev64_value();
1139         break;
1140     default:
1141         pr_err("%s: unknown data1 encoding %d\n", __func__, type);
1142         return AARCH64_BREAK_FAULT;
1143     }
1144 
1145     switch (variant) {
1146     case AARCH64_INSN_VARIANT_32BIT:
1147         break;
1148     case AARCH64_INSN_VARIANT_64BIT:
1149         insn |= AARCH64_INSN_SF_BIT;
1150         break;
1151     default:
1152         pr_err("%s: unknown variant encoding %d\n", __func__, variant);
1153         return AARCH64_BREAK_FAULT;
1154     }
1155 
1156     insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
1157 
1158     return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src);
1159 }
1160 
1161 u32 aarch64_insn_gen_data2(enum aarch64_insn_register dst,
1162                enum aarch64_insn_register src,
1163                enum aarch64_insn_register reg,
1164                enum aarch64_insn_variant variant,
1165                enum aarch64_insn_data2_type type)
1166 {
1167     u32 insn;
1168 
1169     switch (type) {
1170     case AARCH64_INSN_DATA2_UDIV:
1171         insn = aarch64_insn_get_udiv_value();
1172         break;
1173     case AARCH64_INSN_DATA2_SDIV:
1174         insn = aarch64_insn_get_sdiv_value();
1175         break;
1176     case AARCH64_INSN_DATA2_LSLV:
1177         insn = aarch64_insn_get_lslv_value();
1178         break;
1179     case AARCH64_INSN_DATA2_LSRV:
1180         insn = aarch64_insn_get_lsrv_value();
1181         break;
1182     case AARCH64_INSN_DATA2_ASRV:
1183         insn = aarch64_insn_get_asrv_value();
1184         break;
1185     case AARCH64_INSN_DATA2_RORV:
1186         insn = aarch64_insn_get_rorv_value();
1187         break;
1188     default:
1189         pr_err("%s: unknown data2 encoding %d\n", __func__, type);
1190         return AARCH64_BREAK_FAULT;
1191     }
1192 
1193     switch (variant) {
1194     case AARCH64_INSN_VARIANT_32BIT:
1195         break;
1196     case AARCH64_INSN_VARIANT_64BIT:
1197         insn |= AARCH64_INSN_SF_BIT;
1198         break;
1199     default:
1200         pr_err("%s: unknown variant encoding %d\n", __func__, variant);
1201         return AARCH64_BREAK_FAULT;
1202     }
1203 
1204     insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
1205 
1206     insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src);
1207 
1208     return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, reg);
1209 }
1210 
1211 u32 aarch64_insn_gen_data3(enum aarch64_insn_register dst,
1212                enum aarch64_insn_register src,
1213                enum aarch64_insn_register reg1,
1214                enum aarch64_insn_register reg2,
1215                enum aarch64_insn_variant variant,
1216                enum aarch64_insn_data3_type type)
1217 {
1218     u32 insn;
1219 
1220     switch (type) {
1221     case AARCH64_INSN_DATA3_MADD:
1222         insn = aarch64_insn_get_madd_value();
1223         break;
1224     case AARCH64_INSN_DATA3_MSUB:
1225         insn = aarch64_insn_get_msub_value();
1226         break;
1227     default:
1228         pr_err("%s: unknown data3 encoding %d\n", __func__, type);
1229         return AARCH64_BREAK_FAULT;
1230     }
1231 
1232     switch (variant) {
1233     case AARCH64_INSN_VARIANT_32BIT:
1234         break;
1235     case AARCH64_INSN_VARIANT_64BIT:
1236         insn |= AARCH64_INSN_SF_BIT;
1237         break;
1238     default:
1239         pr_err("%s: unknown variant encoding %d\n", __func__, variant);
1240         return AARCH64_BREAK_FAULT;
1241     }
1242 
1243     insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
1244 
1245     insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RA, insn, src);
1246 
1247     insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
1248                         reg1);
1249 
1250     return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn,
1251                         reg2);
1252 }
1253 
1254 u32 aarch64_insn_gen_logical_shifted_reg(enum aarch64_insn_register dst,
1255                      enum aarch64_insn_register src,
1256                      enum aarch64_insn_register reg,
1257                      int shift,
1258                      enum aarch64_insn_variant variant,
1259                      enum aarch64_insn_logic_type type)
1260 {
1261     u32 insn;
1262 
1263     switch (type) {
1264     case AARCH64_INSN_LOGIC_AND:
1265         insn = aarch64_insn_get_and_value();
1266         break;
1267     case AARCH64_INSN_LOGIC_BIC:
1268         insn = aarch64_insn_get_bic_value();
1269         break;
1270     case AARCH64_INSN_LOGIC_ORR:
1271         insn = aarch64_insn_get_orr_value();
1272         break;
1273     case AARCH64_INSN_LOGIC_ORN:
1274         insn = aarch64_insn_get_orn_value();
1275         break;
1276     case AARCH64_INSN_LOGIC_EOR:
1277         insn = aarch64_insn_get_eor_value();
1278         break;
1279     case AARCH64_INSN_LOGIC_EON:
1280         insn = aarch64_insn_get_eon_value();
1281         break;
1282     case AARCH64_INSN_LOGIC_AND_SETFLAGS:
1283         insn = aarch64_insn_get_ands_value();
1284         break;
1285     case AARCH64_INSN_LOGIC_BIC_SETFLAGS:
1286         insn = aarch64_insn_get_bics_value();
1287         break;
1288     default:
1289         pr_err("%s: unknown logical encoding %d\n", __func__, type);
1290         return AARCH64_BREAK_FAULT;
1291     }
1292 
1293     switch (variant) {
1294     case AARCH64_INSN_VARIANT_32BIT:
1295         if (shift & ~(SZ_32 - 1)) {
1296             pr_err("%s: invalid shift encoding %d\n", __func__,
1297                    shift);
1298             return AARCH64_BREAK_FAULT;
1299         }
1300         break;
1301     case AARCH64_INSN_VARIANT_64BIT:
1302         insn |= AARCH64_INSN_SF_BIT;
1303         if (shift & ~(SZ_64 - 1)) {
1304             pr_err("%s: invalid shift encoding %d\n", __func__,
1305                    shift);
1306             return AARCH64_BREAK_FAULT;
1307         }
1308         break;
1309     default:
1310         pr_err("%s: unknown variant encoding %d\n", __func__, variant);
1311         return AARCH64_BREAK_FAULT;
1312     }
1313 
1314 
1315     insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
1316 
1317     insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src);
1318 
1319     insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, reg);
1320 
1321     return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_6, insn, shift);
1322 }
1323 
1324 /*
1325  * MOV (register) is architecturally an alias of ORR (shifted register) where
1326  * MOV <*d>, <*m> is equivalent to ORR <*d>, <*ZR>, <*m>
1327  */
1328 u32 aarch64_insn_gen_move_reg(enum aarch64_insn_register dst,
1329                   enum aarch64_insn_register src,
1330                   enum aarch64_insn_variant variant)
1331 {
1332     return aarch64_insn_gen_logical_shifted_reg(dst, AARCH64_INSN_REG_ZR,
1333                             src, 0, variant,
1334                             AARCH64_INSN_LOGIC_ORR);
1335 }
1336 
1337 u32 aarch64_insn_gen_adr(unsigned long pc, unsigned long addr,
1338              enum aarch64_insn_register reg,
1339              enum aarch64_insn_adr_type type)
1340 {
1341     u32 insn;
1342     s32 offset;
1343 
1344     switch (type) {
1345     case AARCH64_INSN_ADR_TYPE_ADR:
1346         insn = aarch64_insn_get_adr_value();
1347         offset = addr - pc;
1348         break;
1349     case AARCH64_INSN_ADR_TYPE_ADRP:
1350         insn = aarch64_insn_get_adrp_value();
1351         offset = (addr - ALIGN_DOWN(pc, SZ_4K)) >> 12;
1352         break;
1353     default:
1354         pr_err("%s: unknown adr encoding %d\n", __func__, type);
1355         return AARCH64_BREAK_FAULT;
1356     }
1357 
1358     if (offset < -SZ_1M || offset >= SZ_1M)
1359         return AARCH64_BREAK_FAULT;
1360 
1361     insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, reg);
1362 
1363     return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_ADR, insn, offset);
1364 }
1365 
1366 /*
1367  * Decode the imm field of a branch, and return the byte offset as a
1368  * signed value (so it can be used when computing a new branch
1369  * target).
1370  */
1371 s32 aarch64_get_branch_offset(u32 insn)
1372 {
1373     s32 imm;
1374 
1375     if (aarch64_insn_is_b(insn) || aarch64_insn_is_bl(insn)) {
1376         imm = aarch64_insn_decode_immediate(AARCH64_INSN_IMM_26, insn);
1377         return (imm << 6) >> 4;
1378     }
1379 
1380     if (aarch64_insn_is_cbz(insn) || aarch64_insn_is_cbnz(insn) ||
1381         aarch64_insn_is_bcond(insn)) {
1382         imm = aarch64_insn_decode_immediate(AARCH64_INSN_IMM_19, insn);
1383         return (imm << 13) >> 11;
1384     }
1385 
1386     if (aarch64_insn_is_tbz(insn) || aarch64_insn_is_tbnz(insn)) {
1387         imm = aarch64_insn_decode_immediate(AARCH64_INSN_IMM_14, insn);
1388         return (imm << 18) >> 16;
1389     }
1390 
1391     /* Unhandled instruction */
1392     BUG();
1393 }
1394 
1395 /*
1396  * Encode the displacement of a branch in the imm field and return the
1397  * updated instruction.
1398  */
1399 u32 aarch64_set_branch_offset(u32 insn, s32 offset)
1400 {
1401     if (aarch64_insn_is_b(insn) || aarch64_insn_is_bl(insn))
1402         return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_26, insn,
1403                              offset >> 2);
1404 
1405     if (aarch64_insn_is_cbz(insn) || aarch64_insn_is_cbnz(insn) ||
1406         aarch64_insn_is_bcond(insn))
1407         return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_19, insn,
1408                              offset >> 2);
1409 
1410     if (aarch64_insn_is_tbz(insn) || aarch64_insn_is_tbnz(insn))
1411         return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_14, insn,
1412                              offset >> 2);
1413 
1414     /* Unhandled instruction */
1415     BUG();
1416 }
1417 
1418 s32 aarch64_insn_adrp_get_offset(u32 insn)
1419 {
1420     BUG_ON(!aarch64_insn_is_adrp(insn));
1421     return aarch64_insn_decode_immediate(AARCH64_INSN_IMM_ADR, insn) << 12;
1422 }
1423 
1424 u32 aarch64_insn_adrp_set_offset(u32 insn, s32 offset)
1425 {
1426     BUG_ON(!aarch64_insn_is_adrp(insn));
1427     return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_ADR, insn,
1428                         offset >> 12);
1429 }
1430 
1431 /*
1432  * Extract the Op/CR data from a msr/mrs instruction.
1433  */
1434 u32 aarch64_insn_extract_system_reg(u32 insn)
1435 {
1436     return (insn & 0x1FFFE0) >> 5;
1437 }
1438 
1439 bool aarch32_insn_is_wide(u32 insn)
1440 {
1441     return insn >= 0xe800;
1442 }
1443 
1444 /*
1445  * Macros/defines for extracting register numbers from instruction.
1446  */
1447 u32 aarch32_insn_extract_reg_num(u32 insn, int offset)
1448 {
1449     return (insn & (0xf << offset)) >> offset;
1450 }
1451 
1452 #define OPC2_MASK   0x7
1453 #define OPC2_OFFSET 5
1454 u32 aarch32_insn_mcr_extract_opc2(u32 insn)
1455 {
1456     return (insn & (OPC2_MASK << OPC2_OFFSET)) >> OPC2_OFFSET;
1457 }
1458 
1459 #define CRM_MASK    0xf
1460 u32 aarch32_insn_mcr_extract_crm(u32 insn)
1461 {
1462     return insn & CRM_MASK;
1463 }
1464 
1465 static bool range_of_ones(u64 val)
1466 {
1467     /* Doesn't handle full ones or full zeroes */
1468     u64 sval = val >> __ffs64(val);
1469 
1470     /* One of Sean Eron Anderson's bithack tricks */
1471     return ((sval + 1) & (sval)) == 0;
1472 }
1473 
1474 static u32 aarch64_encode_immediate(u64 imm,
1475                     enum aarch64_insn_variant variant,
1476                     u32 insn)
1477 {
1478     unsigned int immr, imms, n, ones, ror, esz, tmp;
1479     u64 mask;
1480 
1481     switch (variant) {
1482     case AARCH64_INSN_VARIANT_32BIT:
1483         esz = 32;
1484         break;
1485     case AARCH64_INSN_VARIANT_64BIT:
1486         insn |= AARCH64_INSN_SF_BIT;
1487         esz = 64;
1488         break;
1489     default:
1490         pr_err("%s: unknown variant encoding %d\n", __func__, variant);
1491         return AARCH64_BREAK_FAULT;
1492     }
1493 
1494     mask = GENMASK(esz - 1, 0);
1495 
1496     /* Can't encode full zeroes, full ones, or value wider than the mask */
1497     if (!imm || imm == mask || imm & ~mask)
1498         return AARCH64_BREAK_FAULT;
1499 
1500     /*
1501      * Inverse of Replicate(). Try to spot a repeating pattern
1502      * with a pow2 stride.
1503      */
1504     for (tmp = esz / 2; tmp >= 2; tmp /= 2) {
1505         u64 emask = BIT(tmp) - 1;
1506 
1507         if ((imm & emask) != ((imm >> tmp) & emask))
1508             break;
1509 
1510         esz = tmp;
1511         mask = emask;
1512     }
1513 
1514     /* N is only set if we're encoding a 64bit value */
1515     n = esz == 64;
1516 
1517     /* Trim imm to the element size */
1518     imm &= mask;
1519 
1520     /* That's how many ones we need to encode */
1521     ones = hweight64(imm);
1522 
1523     /*
1524      * imms is set to (ones - 1), prefixed with a string of ones
1525      * and a zero if they fit. Cap it to 6 bits.
1526      */
1527     imms  = ones - 1;
1528     imms |= 0xf << ffs(esz);
1529     imms &= BIT(6) - 1;
1530 
1531     /* Compute the rotation */
1532     if (range_of_ones(imm)) {
1533         /*
1534          * Pattern: 0..01..10..0
1535          *
1536          * Compute how many rotate we need to align it right
1537          */
1538         ror = __ffs64(imm);
1539     } else {
1540         /*
1541          * Pattern: 0..01..10..01..1
1542          *
1543          * Fill the unused top bits with ones, and check if
1544          * the result is a valid immediate (all ones with a
1545          * contiguous ranges of zeroes).
1546          */
1547         imm |= ~mask;
1548         if (!range_of_ones(~imm))
1549             return AARCH64_BREAK_FAULT;
1550 
1551         /*
1552          * Compute the rotation to get a continuous set of
1553          * ones, with the first bit set at position 0
1554          */
1555         ror = fls64(~imm);
1556     }
1557 
1558     /*
1559      * immr is the number of bits we need to rotate back to the
1560      * original set of ones. Note that this is relative to the
1561      * element size...
1562      */
1563     immr = (esz - ror) % esz;
1564 
1565     insn = aarch64_insn_encode_immediate(AARCH64_INSN_IMM_N, insn, n);
1566     insn = aarch64_insn_encode_immediate(AARCH64_INSN_IMM_R, insn, immr);
1567     return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_S, insn, imms);
1568 }
1569 
1570 u32 aarch64_insn_gen_logical_immediate(enum aarch64_insn_logic_type type,
1571                        enum aarch64_insn_variant variant,
1572                        enum aarch64_insn_register Rn,
1573                        enum aarch64_insn_register Rd,
1574                        u64 imm)
1575 {
1576     u32 insn;
1577 
1578     switch (type) {
1579     case AARCH64_INSN_LOGIC_AND:
1580         insn = aarch64_insn_get_and_imm_value();
1581         break;
1582     case AARCH64_INSN_LOGIC_ORR:
1583         insn = aarch64_insn_get_orr_imm_value();
1584         break;
1585     case AARCH64_INSN_LOGIC_EOR:
1586         insn = aarch64_insn_get_eor_imm_value();
1587         break;
1588     case AARCH64_INSN_LOGIC_AND_SETFLAGS:
1589         insn = aarch64_insn_get_ands_imm_value();
1590         break;
1591     default:
1592         pr_err("%s: unknown logical encoding %d\n", __func__, type);
1593         return AARCH64_BREAK_FAULT;
1594     }
1595 
1596     insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, Rd);
1597     insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, Rn);
1598     return aarch64_encode_immediate(imm, variant, insn);
1599 }
1600 
1601 u32 aarch64_insn_gen_extr(enum aarch64_insn_variant variant,
1602               enum aarch64_insn_register Rm,
1603               enum aarch64_insn_register Rn,
1604               enum aarch64_insn_register Rd,
1605               u8 lsb)
1606 {
1607     u32 insn;
1608 
1609     insn = aarch64_insn_get_extr_value();
1610 
1611     switch (variant) {
1612     case AARCH64_INSN_VARIANT_32BIT:
1613         if (lsb > 31)
1614             return AARCH64_BREAK_FAULT;
1615         break;
1616     case AARCH64_INSN_VARIANT_64BIT:
1617         if (lsb > 63)
1618             return AARCH64_BREAK_FAULT;
1619         insn |= AARCH64_INSN_SF_BIT;
1620         insn = aarch64_insn_encode_immediate(AARCH64_INSN_IMM_N, insn, 1);
1621         break;
1622     default:
1623         pr_err("%s: unknown variant encoding %d\n", __func__, variant);
1624         return AARCH64_BREAK_FAULT;
1625     }
1626 
1627     insn = aarch64_insn_encode_immediate(AARCH64_INSN_IMM_S, insn, lsb);
1628     insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, Rd);
1629     insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, Rn);
1630     return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, Rm);
1631 }
1632 
1633 u32 aarch64_insn_gen_dmb(enum aarch64_insn_mb_type type)
1634 {
1635     u32 opt;
1636     u32 insn;
1637 
1638     switch (type) {
1639     case AARCH64_INSN_MB_SY:
1640         opt = 0xf;
1641         break;
1642     case AARCH64_INSN_MB_ST:
1643         opt = 0xe;
1644         break;
1645     case AARCH64_INSN_MB_LD:
1646         opt = 0xd;
1647         break;
1648     case AARCH64_INSN_MB_ISH:
1649         opt = 0xb;
1650         break;
1651     case AARCH64_INSN_MB_ISHST:
1652         opt = 0xa;
1653         break;
1654     case AARCH64_INSN_MB_ISHLD:
1655         opt = 0x9;
1656         break;
1657     case AARCH64_INSN_MB_NSH:
1658         opt = 0x7;
1659         break;
1660     case AARCH64_INSN_MB_NSHST:
1661         opt = 0x6;
1662         break;
1663     case AARCH64_INSN_MB_NSHLD:
1664         opt = 0x5;
1665         break;
1666     default:
1667         pr_err("%s: unknown dmb type %d\n", __func__, type);
1668         return AARCH64_BREAK_FAULT;
1669     }
1670 
1671     insn = aarch64_insn_get_dmb_value();
1672     insn &= ~GENMASK(11, 8);
1673     insn |= (opt << 8);
1674 
1675     return insn;
1676 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!