powerpc/lib/code-patching.c

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  *  Copyright 2008 Michael Ellerman, IBM Corporation.
0004  */
0005
0006 #include <linux/kprobes.h>
0007 #include <linux/vmalloc.h>
0008 #include <linux/init.h>
0009 #include <linux/cpuhotplug.h>
0010 #include <linux/uaccess.h>
0011 #include <linux/jump_label.h>
0012
0013 #include <asm/tlbflush.h>
0014 #include <asm/page.h>
0015 #include <asm/code-patching.h>
0016 #include <asm/inst.h>
0017
0018 static int __patch_instruction(u32 *exec_addr, ppc_inst_t instr, u32 *patch_addr)
0019 {
0020     if (!ppc_inst_prefixed(instr)) {
0021         u32 val = ppc_inst_val(instr);
0022
0023         __put_kernel_nofault(patch_addr, &val, u32, failed);
0024     } else {
0025         u64 val = ppc_inst_as_ulong(instr);
0026
0027         __put_kernel_nofault(patch_addr, &val, u64, failed);
0028     }
0029
0030     asm ("dcbst 0, %0; sync; icbi 0,%1; sync; isync" :: "r" (patch_addr),
0031                                 "r" (exec_addr));
0032
0033     return 0;
0034
0035 failed:
0036     return -EPERM;
0037 }
0038
0039 int raw_patch_instruction(u32 *addr, ppc_inst_t instr)
0040 {
0041     return __patch_instruction(addr, instr, addr);
0042 }
0043
0044 #ifdef CONFIG_STRICT_KERNEL_RWX
0045 static DEFINE_PER_CPU(struct vm_struct *, text_poke_area);
0046
0047 static int map_patch_area(void *addr, unsigned long text_poke_addr);
0048 static void unmap_patch_area(unsigned long addr);
0049
0050 static int text_area_cpu_up(unsigned int cpu)
0051 {
0052     struct vm_struct *area;
0053     unsigned long addr;
0054     int err;
0055
0056     area = get_vm_area(PAGE_SIZE, VM_ALLOC);
0057     if (!area) {
0058         WARN_ONCE(1, "Failed to create text area for cpu %d\n",
0059             cpu);
0060         return -1;
0061     }
0062
0063     // Map/unmap the area to ensure all page tables are pre-allocated
0064     addr = (unsigned long)area->addr;
0065     err = map_patch_area(empty_zero_page, addr);
0066     if (err)
0067         return err;
0068
0069     unmap_patch_area(addr);
0070
0071     this_cpu_write(text_poke_area, area);
0072
0073     return 0;
0074 }
0075
0076 static int text_area_cpu_down(unsigned int cpu)
0077 {
0078     free_vm_area(this_cpu_read(text_poke_area));
0079     return 0;
0080 }
0081
0082 static __ro_after_init DEFINE_STATIC_KEY_FALSE(poking_init_done);
0083
0084 /*
0085  * Although BUG_ON() is rude, in this case it should only happen if ENOMEM, and
0086  * we judge it as being preferable to a kernel that will crash later when
0087  * someone tries to use patch_instruction().
0088  */
0089 void __init poking_init(void)
0090 {
0091     BUG_ON(!cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
0092         "powerpc/text_poke:online", text_area_cpu_up,
0093         text_area_cpu_down));
0094     static_branch_enable(&poking_init_done);
0095 }
0096
0097 /*
0098  * This can be called for kernel text or a module.
0099  */
0100 static int map_patch_area(void *addr, unsigned long text_poke_addr)
0101 {
0102     unsigned long pfn;
0103
0104     if (IS_ENABLED(CONFIG_MODULES) && is_vmalloc_or_module_addr(addr))
0105         pfn = vmalloc_to_pfn(addr);
0106     else
0107         pfn = __pa_symbol(addr) >> PAGE_SHIFT;
0108
0109     return map_kernel_page(text_poke_addr, (pfn << PAGE_SHIFT), PAGE_KERNEL);
0110 }
0111
0112 static void unmap_patch_area(unsigned long addr)
0113 {
0114     pte_t *ptep;
0115     pmd_t *pmdp;
0116     pud_t *pudp;
0117     p4d_t *p4dp;
0118     pgd_t *pgdp;
0119
0120     pgdp = pgd_offset_k(addr);
0121     if (WARN_ON(pgd_none(*pgdp)))
0122         return;
0123
0124     p4dp = p4d_offset(pgdp, addr);
0125     if (WARN_ON(p4d_none(*p4dp)))
0126         return;
0127
0128     pudp = pud_offset(p4dp, addr);
0129     if (WARN_ON(pud_none(*pudp)))
0130         return;
0131
0132     pmdp = pmd_offset(pudp, addr);
0133     if (WARN_ON(pmd_none(*pmdp)))
0134         return;
0135
0136     ptep = pte_offset_kernel(pmdp, addr);
0137     if (WARN_ON(pte_none(*ptep)))
0138         return;
0139
0140     /*
0141      * In hash, pte_clear flushes the tlb, in radix, we have to
0142      */
0143     pte_clear(&init_mm, addr, ptep);
0144     flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
0145 }
0146
0147 static int __do_patch_instruction(u32 *addr, ppc_inst_t instr)
0148 {
0149     int err;
0150     u32 *patch_addr;
0151     unsigned long text_poke_addr;
0152
0153     text_poke_addr = (unsigned long)__this_cpu_read(text_poke_area)->addr;
0154     patch_addr = (u32 *)(text_poke_addr + offset_in_page(addr));
0155
0156     err = map_patch_area(addr, text_poke_addr);
0157     if (err)
0158         return err;
0159
0160     err = __patch_instruction(addr, instr, patch_addr);
0161
0162     unmap_patch_area(text_poke_addr);
0163
0164     return err;
0165 }
0166
0167 static int do_patch_instruction(u32 *addr, ppc_inst_t instr)
0168 {
0169     int err;
0170     unsigned long flags;
0171
0172     /*
0173      * During early early boot patch_instruction is called
0174      * when text_poke_area is not ready, but we still need
0175      * to allow patching. We just do the plain old patching
0176      */
0177     if (!static_branch_likely(&poking_init_done))
0178         return raw_patch_instruction(addr, instr);
0179
0180     local_irq_save(flags);
0181     err = __do_patch_instruction(addr, instr);
0182     local_irq_restore(flags);
0183
0184     return err;
0185 }
0186 #else /* !CONFIG_STRICT_KERNEL_RWX */
0187
0188 static int do_patch_instruction(u32 *addr, ppc_inst_t instr)
0189 {
0190     return raw_patch_instruction(addr, instr);
0191 }
0192
0193 #endif /* CONFIG_STRICT_KERNEL_RWX */
0194
0195 __ro_after_init DEFINE_STATIC_KEY_FALSE(init_mem_is_free);
0196
0197 int patch_instruction(u32 *addr, ppc_inst_t instr)
0198 {
0199     /* Make sure we aren't patching a freed init section */
0200     if (static_branch_likely(&init_mem_is_free) && init_section_contains(addr, 4))
0201         return 0;
0202
0203     return do_patch_instruction(addr, instr);
0204 }
0205 NOKPROBE_SYMBOL(patch_instruction);
0206
0207 int patch_branch(u32 *addr, unsigned long target, int flags)
0208 {
0209     ppc_inst_t instr;
0210
0211     if (create_branch(&instr, addr, target, flags))
0212         return -ERANGE;
0213
0214     return patch_instruction(addr, instr);
0215 }
0216
0217 /*
0218  * Helper to check if a given instruction is a conditional branch
0219  * Derived from the conditional checks in analyse_instr()
0220  */
0221 bool is_conditional_branch(ppc_inst_t instr)
0222 {
0223     unsigned int opcode = ppc_inst_primary_opcode(instr);
0224
0225     if (opcode == 16)       /* bc, bca, bcl, bcla */
0226         return true;
0227     if (opcode == 19) {
0228         switch ((ppc_inst_val(instr) >> 1) & 0x3ff) {
0229         case 16:        /* bclr, bclrl */
0230         case 528:       /* bcctr, bcctrl */
0231         case 560:       /* bctar, bctarl */
0232             return true;
0233         }
0234     }
0235     return false;
0236 }
0237 NOKPROBE_SYMBOL(is_conditional_branch);
0238
0239 int create_cond_branch(ppc_inst_t *instr, const u32 *addr,
0240                unsigned long target, int flags)
0241 {
0242     long offset;
0243
0244     offset = target;
0245     if (! (flags & BRANCH_ABSOLUTE))
0246         offset = offset - (unsigned long)addr;
0247
0248     /* Check we can represent the target in the instruction format */
0249     if (!is_offset_in_cond_branch_range(offset))
0250         return 1;
0251
0252     /* Mask out the flags and target, so they don't step on each other. */
0253     *instr = ppc_inst(0x40000000 | (flags & 0x3FF0003) | (offset & 0xFFFC));
0254
0255     return 0;
0256 }
0257
0258 int instr_is_relative_branch(ppc_inst_t instr)
0259 {
0260     if (ppc_inst_val(instr) & BRANCH_ABSOLUTE)
0261         return 0;
0262
0263     return instr_is_branch_iform(instr) || instr_is_branch_bform(instr);
0264 }
0265
0266 int instr_is_relative_link_branch(ppc_inst_t instr)
0267 {
0268     return instr_is_relative_branch(instr) && (ppc_inst_val(instr) & BRANCH_SET_LINK);
0269 }
0270
0271 static unsigned long branch_iform_target(const u32 *instr)
0272 {
0273     signed long imm;
0274
0275     imm = ppc_inst_val(ppc_inst_read(instr)) & 0x3FFFFFC;
0276
0277     /* If the top bit of the immediate value is set this is negative */
0278     if (imm & 0x2000000)
0279         imm -= 0x4000000;
0280
0281     if ((ppc_inst_val(ppc_inst_read(instr)) & BRANCH_ABSOLUTE) == 0)
0282         imm += (unsigned long)instr;
0283
0284     return (unsigned long)imm;
0285 }
0286
0287 static unsigned long branch_bform_target(const u32 *instr)
0288 {
0289     signed long imm;
0290
0291     imm = ppc_inst_val(ppc_inst_read(instr)) & 0xFFFC;
0292
0293     /* If the top bit of the immediate value is set this is negative */
0294     if (imm & 0x8000)
0295         imm -= 0x10000;
0296
0297     if ((ppc_inst_val(ppc_inst_read(instr)) & BRANCH_ABSOLUTE) == 0)
0298         imm += (unsigned long)instr;
0299
0300     return (unsigned long)imm;
0301 }
0302
0303 unsigned long branch_target(const u32 *instr)
0304 {
0305     if (instr_is_branch_iform(ppc_inst_read(instr)))
0306         return branch_iform_target(instr);
0307     else if (instr_is_branch_bform(ppc_inst_read(instr)))
0308         return branch_bform_target(instr);
0309
0310     return 0;
0311 }
0312
0313 int translate_branch(ppc_inst_t *instr, const u32 *dest, const u32 *src)
0314 {
0315     unsigned long target;
0316     target = branch_target(src);
0317
0318     if (instr_is_branch_iform(ppc_inst_read(src)))
0319         return create_branch(instr, dest, target,
0320                      ppc_inst_val(ppc_inst_read(src)));
0321     else if (instr_is_branch_bform(ppc_inst_read(src)))
0322         return create_cond_branch(instr, dest, target,
0323                       ppc_inst_val(ppc_inst_read(src)));
0324
0325     return 1;
0326 }