arm64/kernel/alternative.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * alternative runtime patching
0004  * inspired by the x86 version
0005  *
0006  * Copyright (C) 2014 ARM Ltd.
0007  */
0008
0009 #define pr_fmt(fmt) "alternatives: " fmt
0010
0011 #include <linux/init.h>
0012 #include <linux/cpu.h>
0013 #include <asm/cacheflush.h>
0014 #include <asm/alternative.h>
0015 #include <asm/cpufeature.h>
0016 #include <asm/insn.h>
0017 #include <asm/sections.h>
0018 #include <linux/stop_machine.h>
0019
0020 #define __ALT_PTR(a, f)     ((void *)&(a)->f + (a)->f)
0021 #define ALT_ORIG_PTR(a)     __ALT_PTR(a, orig_offset)
0022 #define ALT_REPL_PTR(a)     __ALT_PTR(a, alt_offset)
0023
0024 /* Volatile, as we may be patching the guts of READ_ONCE() */
0025 static volatile int all_alternatives_applied;
0026
0027 static DECLARE_BITMAP(applied_alternatives, ARM64_NCAPS);
0028
0029 struct alt_region {
0030     struct alt_instr *begin;
0031     struct alt_instr *end;
0032 };
0033
0034 bool alternative_is_applied(u16 cpufeature)
0035 {
0036     if (WARN_ON(cpufeature >= ARM64_NCAPS))
0037         return false;
0038
0039     return test_bit(cpufeature, applied_alternatives);
0040 }
0041
0042 /*
0043  * Check if the target PC is within an alternative block.
0044  */
0045 static __always_inline bool branch_insn_requires_update(struct alt_instr *alt, unsigned long pc)
0046 {
0047     unsigned long replptr = (unsigned long)ALT_REPL_PTR(alt);
0048     return !(pc >= replptr && pc <= (replptr + alt->alt_len));
0049 }
0050
0051 #define align_down(x, a)    ((unsigned long)(x) & ~(((unsigned long)(a)) - 1))
0052
0053 static __always_inline u32 get_alt_insn(struct alt_instr *alt, __le32 *insnptr, __le32 *altinsnptr)
0054 {
0055     u32 insn;
0056
0057     insn = le32_to_cpu(*altinsnptr);
0058
0059     if (aarch64_insn_is_branch_imm(insn)) {
0060         s32 offset = aarch64_get_branch_offset(insn);
0061         unsigned long target;
0062
0063         target = (unsigned long)altinsnptr + offset;
0064
0065         /*
0066          * If we're branching inside the alternate sequence,
0067          * do not rewrite the instruction, as it is already
0068          * correct. Otherwise, generate the new instruction.
0069          */
0070         if (branch_insn_requires_update(alt, target)) {
0071             offset = target - (unsigned long)insnptr;
0072             insn = aarch64_set_branch_offset(insn, offset);
0073         }
0074     } else if (aarch64_insn_is_adrp(insn)) {
0075         s32 orig_offset, new_offset;
0076         unsigned long target;
0077
0078         /*
0079          * If we're replacing an adrp instruction, which uses PC-relative
0080          * immediate addressing, adjust the offset to reflect the new
0081          * PC. adrp operates on 4K aligned addresses.
0082          */
0083         orig_offset  = aarch64_insn_adrp_get_offset(insn);
0084         target = align_down(altinsnptr, SZ_4K) + orig_offset;
0085         new_offset = target - align_down(insnptr, SZ_4K);
0086         insn = aarch64_insn_adrp_set_offset(insn, new_offset);
0087     } else if (aarch64_insn_uses_literal(insn)) {
0088         /*
0089          * Disallow patching unhandled instructions using PC relative
0090          * literal addresses
0091          */
0092         BUG();
0093     }
0094
0095     return insn;
0096 }
0097
0098 static noinstr void patch_alternative(struct alt_instr *alt,
0099                   __le32 *origptr, __le32 *updptr, int nr_inst)
0100 {
0101     __le32 *replptr;
0102     int i;
0103
0104     replptr = ALT_REPL_PTR(alt);
0105     for (i = 0; i < nr_inst; i++) {
0106         u32 insn;
0107
0108         insn = get_alt_insn(alt, origptr + i, replptr + i);
0109         updptr[i] = cpu_to_le32(insn);
0110     }
0111 }
0112
0113 /*
0114  * We provide our own, private D-cache cleaning function so that we don't
0115  * accidentally call into the cache.S code, which is patched by us at
0116  * runtime.
0117  */
0118 static void clean_dcache_range_nopatch(u64 start, u64 end)
0119 {
0120     u64 cur, d_size, ctr_el0;
0121
0122     ctr_el0 = read_sanitised_ftr_reg(SYS_CTR_EL0);
0123     d_size = 4 << cpuid_feature_extract_unsigned_field(ctr_el0,
0124                                CTR_EL0_DminLine_SHIFT);
0125     cur = start & ~(d_size - 1);
0126     do {
0127         /*
0128          * We must clean+invalidate to the PoC in order to avoid
0129          * Cortex-A53 errata 826319, 827319, 824069 and 819472
0130          * (this corresponds to ARM64_WORKAROUND_CLEAN_CACHE)
0131          */
0132         asm volatile("dc civac, %0" : : "r" (cur) : "memory");
0133     } while (cur += d_size, cur < end);
0134 }
0135
0136 static void __nocfi __apply_alternatives(struct alt_region *region, bool is_module,
0137                  unsigned long *feature_mask)
0138 {
0139     struct alt_instr *alt;
0140     __le32 *origptr, *updptr;
0141     alternative_cb_t alt_cb;
0142
0143     for (alt = region->begin; alt < region->end; alt++) {
0144         int nr_inst;
0145
0146         if (!test_bit(alt->cpufeature, feature_mask))
0147             continue;
0148
0149         /* Use ARM64_CB_PATCH as an unconditional patch */
0150         if (alt->cpufeature < ARM64_CB_PATCH &&
0151             !cpus_have_cap(alt->cpufeature))
0152             continue;
0153
0154         if (alt->cpufeature == ARM64_CB_PATCH)
0155             BUG_ON(alt->alt_len != 0);
0156         else
0157             BUG_ON(alt->alt_len != alt->orig_len);
0158
0159         pr_info_once("patching kernel code\n");
0160
0161         origptr = ALT_ORIG_PTR(alt);
0162         updptr = is_module ? origptr : lm_alias(origptr);
0163         nr_inst = alt->orig_len / AARCH64_INSN_SIZE;
0164
0165         if (alt->cpufeature < ARM64_CB_PATCH)
0166             alt_cb = patch_alternative;
0167         else
0168             alt_cb  = ALT_REPL_PTR(alt);
0169
0170         alt_cb(alt, origptr, updptr, nr_inst);
0171
0172         if (!is_module) {
0173             clean_dcache_range_nopatch((u64)origptr,
0174                            (u64)(origptr + nr_inst));
0175         }
0176     }
0177
0178     /*
0179      * The core module code takes care of cache maintenance in
0180      * flush_module_icache().
0181      */
0182     if (!is_module) {
0183         dsb(ish);
0184         icache_inval_all_pou();
0185         isb();
0186
0187         /* Ignore ARM64_CB bit from feature mask */
0188         bitmap_or(applied_alternatives, applied_alternatives,
0189               feature_mask, ARM64_NCAPS);
0190         bitmap_and(applied_alternatives, applied_alternatives,
0191                cpu_hwcaps, ARM64_NCAPS);
0192     }
0193 }
0194
0195 /*
0196  * We might be patching the stop_machine state machine, so implement a
0197  * really simple polling protocol here.
0198  */
0199 static int __apply_alternatives_multi_stop(void *unused)
0200 {
0201     struct alt_region region = {
0202         .begin  = (struct alt_instr *)__alt_instructions,
0203         .end    = (struct alt_instr *)__alt_instructions_end,
0204     };
0205
0206     /* We always have a CPU 0 at this point (__init) */
0207     if (smp_processor_id()) {
0208         while (!all_alternatives_applied)
0209             cpu_relax();
0210         isb();
0211     } else {
0212         DECLARE_BITMAP(remaining_capabilities, ARM64_NPATCHABLE);
0213
0214         bitmap_complement(remaining_capabilities, boot_capabilities,
0215                   ARM64_NPATCHABLE);
0216
0217         BUG_ON(all_alternatives_applied);
0218         __apply_alternatives(&region, false, remaining_capabilities);
0219         /* Barriers provided by the cache flushing */
0220         all_alternatives_applied = 1;
0221     }
0222
0223     return 0;
0224 }
0225
0226 void __init apply_alternatives_all(void)
0227 {
0228     /* better not try code patching on a live SMP system */
0229     stop_machine(__apply_alternatives_multi_stop, NULL, cpu_online_mask);
0230 }
0231
0232 /*
0233  * This is called very early in the boot process (directly after we run
0234  * a feature detect on the boot CPU). No need to worry about other CPUs
0235  * here.
0236  */
0237 void __init apply_boot_alternatives(void)
0238 {
0239     struct alt_region region = {
0240         .begin  = (struct alt_instr *)__alt_instructions,
0241         .end    = (struct alt_instr *)__alt_instructions_end,
0242     };
0243
0244     /* If called on non-boot cpu things could go wrong */
0245     WARN_ON(smp_processor_id() != 0);
0246
0247     __apply_alternatives(&region, false, &boot_capabilities[0]);
0248 }
0249
0250 #ifdef CONFIG_MODULES
0251 void apply_alternatives_module(void *start, size_t length)
0252 {
0253     struct alt_region region = {
0254         .begin  = start,
0255         .end    = start + length,
0256     };
0257     DECLARE_BITMAP(all_capabilities, ARM64_NPATCHABLE);
0258
0259     bitmap_fill(all_capabilities, ARM64_NPATCHABLE);
0260
0261     __apply_alternatives(&region, true, &all_capabilities[0]);
0262 }
0263 #endif