arc/kernel/jump_label.c

0001 // SPDX-License-Identifier: GPL-2.0
0002
0003 #include <linux/kernel.h>
0004 #include <linux/jump_label.h>
0005
0006 #include "asm/cacheflush.h"
0007
0008 #define JUMPLABEL_ERR   "ARC: jump_label: ERROR: "
0009
0010 /* Halt system on fatal error to make debug easier */
0011 #define arc_jl_fatal(format...)                     \
0012 ({                                  \
0013     pr_err(JUMPLABEL_ERR format);                   \
0014     BUG();                              \
0015 })
0016
0017 static inline u32 arc_gen_nop(void)
0018 {
0019     /* 1x 32bit NOP in middle endian */
0020     return 0x7000264a;
0021 }
0022
0023 /*
0024  * Atomic update of patched instruction is only available if this
0025  * instruction doesn't cross L1 cache line boundary. You can read about
0026  * the way we achieve this in arc/include/asm/jump_label.h
0027  */
0028 static inline void instruction_align_assert(void *addr, int len)
0029 {
0030     unsigned long a = (unsigned long)addr;
0031
0032     if ((a >> L1_CACHE_SHIFT) != ((a + len - 1) >> L1_CACHE_SHIFT))
0033         arc_jl_fatal("instruction (addr %px) cross L1 cache line border",
0034                  addr);
0035 }
0036
0037 /*
0038  * ARCv2 'Branch unconditionally' instruction:
0039  * 00000ssssssssss1SSSSSSSSSSNRtttt
0040  * s S[n:0] lower bits signed immediate (number is bitfield size)
0041  * S S[m:n+1] upper bits signed immediate (number is bitfield size)
0042  * t S[24:21] upper bits signed immediate (branch unconditionally far)
0043  * N N <.d> delay slot mode
0044  * R R Reserved
0045  */
0046 static inline u32 arc_gen_branch(jump_label_t pc, jump_label_t target)
0047 {
0048     u32 instruction_l, instruction_r;
0049     u32 pcl = pc & GENMASK(31, 2);
0050     u32 u_offset = target - pcl;
0051     u32 s, S, t;
0052
0053     /*
0054      * Offset in 32-bit branch instruction must to fit into s25.
0055      * Something is terribly broken if we get such huge offset within one
0056      * function.
0057      */
0058     if ((s32)u_offset < -16777216 || (s32)u_offset > 16777214)
0059         arc_jl_fatal("gen branch with offset (%d) not fit in s25",
0060                  (s32)u_offset);
0061
0062     /*
0063      * All instructions are aligned by 2 bytes so we should never get offset
0064      * here which is not 2 bytes aligned.
0065      */
0066     if (u_offset & 0x1)
0067         arc_jl_fatal("gen branch with offset (%d) unaligned to 2 bytes",
0068                  (s32)u_offset);
0069
0070     s = (u_offset >> 1)  & GENMASK(9, 0);
0071     S = (u_offset >> 11) & GENMASK(9, 0);
0072     t = (u_offset >> 21) & GENMASK(3, 0);
0073
0074     /* 00000ssssssssss1 */
0075     instruction_l = (s << 1) | 0x1;
0076     /* SSSSSSSSSSNRtttt */
0077     instruction_r = (S << 6) | t;
0078
0079     return (instruction_r << 16) | (instruction_l & GENMASK(15, 0));
0080 }
0081
0082 void arch_jump_label_transform(struct jump_entry *entry,
0083                    enum jump_label_type type)
0084 {
0085     jump_label_t *instr_addr = (jump_label_t *)entry->code;
0086     u32 instr;
0087
0088     instruction_align_assert(instr_addr, JUMP_LABEL_NOP_SIZE);
0089
0090     if (type == JUMP_LABEL_JMP)
0091         instr = arc_gen_branch(entry->code, entry->target);
0092     else
0093         instr = arc_gen_nop();
0094
0095     WRITE_ONCE(*instr_addr, instr);
0096     flush_icache_range(entry->code, entry->code + JUMP_LABEL_NOP_SIZE);
0097 }
0098
0099 #ifdef CONFIG_ARC_DBG_JUMP_LABEL
0100 #define SELFTEST_MSG    "ARC: instruction generation self-test: "
0101
0102 struct arc_gen_branch_testdata {
0103     jump_label_t pc;
0104     jump_label_t target_address;
0105     u32 expected_instr;
0106 };
0107
0108 static __init int branch_gen_test(const struct arc_gen_branch_testdata *test)
0109 {
0110     u32 instr_got;
0111
0112     instr_got = arc_gen_branch(test->pc, test->target_address);
0113     if (instr_got == test->expected_instr)
0114         return 0;
0115
0116     pr_err(SELFTEST_MSG "FAIL:\n arc_gen_branch(0x%08x, 0x%08x) != 0x%08x, got 0x%08x\n",
0117            test->pc, test->target_address,
0118            test->expected_instr, instr_got);
0119
0120     return -EFAULT;
0121 }
0122
0123 /*
0124  * Offset field in branch instruction is not continuous. Test all
0125  * available offset field and sign combinations. Test data is generated
0126  * from real working code.
0127  */
0128 static const struct arc_gen_branch_testdata arcgenbr_test_data[] __initconst = {
0129     {0x90007548, 0x90007514, 0xffcf07cd}, /* tiny (-52) offs */
0130     {0x9000c9c0, 0x9000c782, 0xffcf05c3}, /* tiny (-574) offs */
0131     {0x9000cc1c, 0x9000c782, 0xffcf0367}, /* tiny (-1178) offs */
0132     {0x9009dce0, 0x9009d106, 0xff8f0427}, /* small (-3034) offs */
0133     {0x9000f5de, 0x90007d30, 0xfc0f0755}, /* big  (-30892) offs */
0134     {0x900a2444, 0x90035f64, 0xc9cf0321}, /* huge (-443616) offs */
0135     {0x90007514, 0x9000752c, 0x00000019}, /* tiny (+24) offs */
0136     {0x9001a578, 0x9001a77a, 0x00000203}, /* tiny (+514) offs */
0137     {0x90031ed8, 0x90032634, 0x0000075d}, /* tiny (+1884) offs */
0138     {0x9008c7f2, 0x9008d3f0, 0x00400401}, /* small (+3072) offs */
0139     {0x9000bb38, 0x9003b340, 0x17c00009}, /* big  (+194568) offs */
0140     {0x90008f44, 0x90578d80, 0xb7c2063d}  /* huge (+5701180) offs */
0141 };
0142
0143 static __init int instr_gen_test(void)
0144 {
0145     int i;
0146
0147     for (i = 0; i < ARRAY_SIZE(arcgenbr_test_data); i++)
0148         if (branch_gen_test(&arcgenbr_test_data[i]))
0149             return -EFAULT;
0150
0151     pr_info(SELFTEST_MSG "OK\n");
0152
0153     return 0;
0154 }
0155 early_initcall(instr_gen_test);
0156
0157 #endif /* CONFIG_ARC_DBG_JUMP_LABEL */