Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  *  Copyright (C) 2001 Ben. Herrenschmidt (benh@kernel.crashing.org)
0004  *
0005  *  Modifications for ppc64:
0006  *      Copyright (C) 2003 Dave Engebretsen <engebret@us.ibm.com>
0007  *
0008  *  Copyright 2008 Michael Ellerman, IBM Corporation.
0009  */
0010 
0011 #include <linux/types.h>
0012 #include <linux/jump_label.h>
0013 #include <linux/kernel.h>
0014 #include <linux/string.h>
0015 #include <linux/init.h>
0016 #include <linux/sched/mm.h>
0017 #include <linux/stop_machine.h>
0018 #include <asm/cputable.h>
0019 #include <asm/code-patching.h>
0020 #include <asm/interrupt.h>
0021 #include <asm/page.h>
0022 #include <asm/sections.h>
0023 #include <asm/setup.h>
0024 #include <asm/security_features.h>
0025 #include <asm/firmware.h>
0026 #include <asm/inst.h>
0027 
0028 struct fixup_entry {
0029     unsigned long   mask;
0030     unsigned long   value;
0031     long        start_off;
0032     long        end_off;
0033     long        alt_start_off;
0034     long        alt_end_off;
0035 };
0036 
0037 static u32 *calc_addr(struct fixup_entry *fcur, long offset)
0038 {
0039     /*
0040      * We store the offset to the code as a negative offset from
0041      * the start of the alt_entry, to support the VDSO. This
0042      * routine converts that back into an actual address.
0043      */
0044     return (u32 *)((unsigned long)fcur + offset);
0045 }
0046 
0047 static int patch_alt_instruction(u32 *src, u32 *dest, u32 *alt_start, u32 *alt_end)
0048 {
0049     int err;
0050     ppc_inst_t instr;
0051 
0052     instr = ppc_inst_read(src);
0053 
0054     if (instr_is_relative_branch(ppc_inst_read(src))) {
0055         u32 *target = (u32 *)branch_target(src);
0056 
0057         /* Branch within the section doesn't need translating */
0058         if (target < alt_start || target > alt_end) {
0059             err = translate_branch(&instr, dest, src);
0060             if (err)
0061                 return 1;
0062         }
0063     }
0064 
0065     raw_patch_instruction(dest, instr);
0066 
0067     return 0;
0068 }
0069 
0070 static int patch_feature_section(unsigned long value, struct fixup_entry *fcur)
0071 {
0072     u32 *start, *end, *alt_start, *alt_end, *src, *dest;
0073 
0074     start = calc_addr(fcur, fcur->start_off);
0075     end = calc_addr(fcur, fcur->end_off);
0076     alt_start = calc_addr(fcur, fcur->alt_start_off);
0077     alt_end = calc_addr(fcur, fcur->alt_end_off);
0078 
0079     if ((alt_end - alt_start) > (end - start))
0080         return 1;
0081 
0082     if ((value & fcur->mask) == fcur->value)
0083         return 0;
0084 
0085     src = alt_start;
0086     dest = start;
0087 
0088     for (; src < alt_end; src = ppc_inst_next(src, src),
0089                   dest = ppc_inst_next(dest, dest)) {
0090         if (patch_alt_instruction(src, dest, alt_start, alt_end))
0091             return 1;
0092     }
0093 
0094     for (; dest < end; dest++)
0095         raw_patch_instruction(dest, ppc_inst(PPC_RAW_NOP()));
0096 
0097     return 0;
0098 }
0099 
0100 void do_feature_fixups(unsigned long value, void *fixup_start, void *fixup_end)
0101 {
0102     struct fixup_entry *fcur, *fend;
0103 
0104     fcur = fixup_start;
0105     fend = fixup_end;
0106 
0107     for (; fcur < fend; fcur++) {
0108         if (patch_feature_section(value, fcur)) {
0109             WARN_ON(1);
0110             printk("Unable to patch feature section at %p - %p" \
0111                 " with %p - %p\n",
0112                 calc_addr(fcur, fcur->start_off),
0113                 calc_addr(fcur, fcur->end_off),
0114                 calc_addr(fcur, fcur->alt_start_off),
0115                 calc_addr(fcur, fcur->alt_end_off));
0116         }
0117     }
0118 }
0119 
0120 #ifdef CONFIG_PPC_BOOK3S_64
0121 static void do_stf_entry_barrier_fixups(enum stf_barrier_type types)
0122 {
0123     unsigned int instrs[3], *dest;
0124     long *start, *end;
0125     int i;
0126 
0127     start = PTRRELOC(&__start___stf_entry_barrier_fixup);
0128     end = PTRRELOC(&__stop___stf_entry_barrier_fixup);
0129 
0130     instrs[0] = PPC_RAW_NOP();
0131     instrs[1] = PPC_RAW_NOP();
0132     instrs[2] = PPC_RAW_NOP();
0133 
0134     i = 0;
0135     if (types & STF_BARRIER_FALLBACK) {
0136         instrs[i++] = PPC_RAW_MFLR(_R10);
0137         instrs[i++] = PPC_RAW_NOP(); /* branch patched below */
0138         instrs[i++] = PPC_RAW_MTLR(_R10);
0139     } else if (types & STF_BARRIER_EIEIO) {
0140         instrs[i++] = PPC_RAW_EIEIO() | 0x02000000; /* eieio + bit 6 hint */
0141     } else if (types & STF_BARRIER_SYNC_ORI) {
0142         instrs[i++] = PPC_RAW_SYNC();
0143         instrs[i++] = PPC_RAW_LD(_R10, _R13, 0);
0144         instrs[i++] = PPC_RAW_ORI(_R31, _R31, 0); /* speculation barrier */
0145     }
0146 
0147     for (i = 0; start < end; start++, i++) {
0148         dest = (void *)start + *start;
0149 
0150         pr_devel("patching dest %lx\n", (unsigned long)dest);
0151 
0152         // See comment in do_entry_flush_fixups() RE order of patching
0153         if (types & STF_BARRIER_FALLBACK) {
0154             patch_instruction(dest, ppc_inst(instrs[0]));
0155             patch_instruction(dest + 2, ppc_inst(instrs[2]));
0156             patch_branch(dest + 1,
0157                      (unsigned long)&stf_barrier_fallback, BRANCH_SET_LINK);
0158         } else {
0159             patch_instruction(dest + 1, ppc_inst(instrs[1]));
0160             patch_instruction(dest + 2, ppc_inst(instrs[2]));
0161             patch_instruction(dest, ppc_inst(instrs[0]));
0162         }
0163     }
0164 
0165     printk(KERN_DEBUG "stf-barrier: patched %d entry locations (%s barrier)\n", i,
0166         (types == STF_BARRIER_NONE)                  ? "no" :
0167         (types == STF_BARRIER_FALLBACK)              ? "fallback" :
0168         (types == STF_BARRIER_EIEIO)                 ? "eieio" :
0169         (types == (STF_BARRIER_SYNC_ORI))            ? "hwsync"
0170                                                    : "unknown");
0171 }
0172 
0173 static void do_stf_exit_barrier_fixups(enum stf_barrier_type types)
0174 {
0175     unsigned int instrs[6], *dest;
0176     long *start, *end;
0177     int i;
0178 
0179     start = PTRRELOC(&__start___stf_exit_barrier_fixup);
0180     end = PTRRELOC(&__stop___stf_exit_barrier_fixup);
0181 
0182     instrs[0] = PPC_RAW_NOP();
0183     instrs[1] = PPC_RAW_NOP();
0184     instrs[2] = PPC_RAW_NOP();
0185     instrs[3] = PPC_RAW_NOP();
0186     instrs[4] = PPC_RAW_NOP();
0187     instrs[5] = PPC_RAW_NOP();
0188 
0189     i = 0;
0190     if (types & STF_BARRIER_FALLBACK || types & STF_BARRIER_SYNC_ORI) {
0191         if (cpu_has_feature(CPU_FTR_HVMODE)) {
0192             instrs[i++] = PPC_RAW_MTSPR(SPRN_HSPRG1, _R13);
0193             instrs[i++] = PPC_RAW_MFSPR(_R13, SPRN_HSPRG0);
0194         } else {
0195             instrs[i++] = PPC_RAW_MTSPR(SPRN_SPRG2, _R13);
0196             instrs[i++] = PPC_RAW_MFSPR(_R13, SPRN_SPRG1);
0197             }
0198         instrs[i++] = PPC_RAW_SYNC();
0199         instrs[i++] = PPC_RAW_LD(_R13, _R13, 0);
0200         instrs[i++] = PPC_RAW_ORI(_R31, _R31, 0); /* speculation barrier */
0201         if (cpu_has_feature(CPU_FTR_HVMODE))
0202             instrs[i++] = PPC_RAW_MFSPR(_R13, SPRN_HSPRG1);
0203         else
0204             instrs[i++] = PPC_RAW_MFSPR(_R13, SPRN_SPRG2);
0205     } else if (types & STF_BARRIER_EIEIO) {
0206         instrs[i++] = PPC_RAW_EIEIO() | 0x02000000; /* eieio + bit 6 hint */
0207     }
0208 
0209     for (i = 0; start < end; start++, i++) {
0210         dest = (void *)start + *start;
0211 
0212         pr_devel("patching dest %lx\n", (unsigned long)dest);
0213 
0214         patch_instruction(dest, ppc_inst(instrs[0]));
0215         patch_instruction(dest + 1, ppc_inst(instrs[1]));
0216         patch_instruction(dest + 2, ppc_inst(instrs[2]));
0217         patch_instruction(dest + 3, ppc_inst(instrs[3]));
0218         patch_instruction(dest + 4, ppc_inst(instrs[4]));
0219         patch_instruction(dest + 5, ppc_inst(instrs[5]));
0220     }
0221     printk(KERN_DEBUG "stf-barrier: patched %d exit locations (%s barrier)\n", i,
0222         (types == STF_BARRIER_NONE)                  ? "no" :
0223         (types == STF_BARRIER_FALLBACK)              ? "fallback" :
0224         (types == STF_BARRIER_EIEIO)                 ? "eieio" :
0225         (types == (STF_BARRIER_SYNC_ORI))            ? "hwsync"
0226                                                    : "unknown");
0227 }
0228 
0229 static bool stf_exit_reentrant = false;
0230 static bool rfi_exit_reentrant = false;
0231 static DEFINE_MUTEX(exit_flush_lock);
0232 
0233 static int __do_stf_barrier_fixups(void *data)
0234 {
0235     enum stf_barrier_type *types = data;
0236 
0237     do_stf_entry_barrier_fixups(*types);
0238     do_stf_exit_barrier_fixups(*types);
0239 
0240     return 0;
0241 }
0242 
0243 void do_stf_barrier_fixups(enum stf_barrier_type types)
0244 {
0245     /*
0246      * The call to the fallback entry flush, and the fallback/sync-ori exit
0247      * flush can not be safely patched in/out while other CPUs are
0248      * executing them. So call __do_stf_barrier_fixups() on one CPU while
0249      * all other CPUs spin in the stop machine core with interrupts hard
0250      * disabled.
0251      *
0252      * The branch to mark interrupt exits non-reentrant is enabled first,
0253      * then stop_machine runs which will ensure all CPUs are out of the
0254      * low level interrupt exit code before patching. After the patching,
0255      * if allowed, then flip the branch to allow fast exits.
0256      */
0257 
0258     // Prevent static key update races with do_rfi_flush_fixups()
0259     mutex_lock(&exit_flush_lock);
0260     static_branch_enable(&interrupt_exit_not_reentrant);
0261 
0262     stop_machine(__do_stf_barrier_fixups, &types, NULL);
0263 
0264     if ((types & STF_BARRIER_FALLBACK) || (types & STF_BARRIER_SYNC_ORI))
0265         stf_exit_reentrant = false;
0266     else
0267         stf_exit_reentrant = true;
0268 
0269     if (stf_exit_reentrant && rfi_exit_reentrant)
0270         static_branch_disable(&interrupt_exit_not_reentrant);
0271 
0272     mutex_unlock(&exit_flush_lock);
0273 }
0274 
0275 void do_uaccess_flush_fixups(enum l1d_flush_type types)
0276 {
0277     unsigned int instrs[4], *dest;
0278     long *start, *end;
0279     int i;
0280 
0281     start = PTRRELOC(&__start___uaccess_flush_fixup);
0282     end = PTRRELOC(&__stop___uaccess_flush_fixup);
0283 
0284     instrs[0] = PPC_RAW_NOP();
0285     instrs[1] = PPC_RAW_NOP();
0286     instrs[2] = PPC_RAW_NOP();
0287     instrs[3] = PPC_RAW_BLR();
0288 
0289     i = 0;
0290     if (types == L1D_FLUSH_FALLBACK) {
0291         instrs[3] = PPC_RAW_NOP();
0292         /* fallthrough to fallback flush */
0293     }
0294 
0295     if (types & L1D_FLUSH_ORI) {
0296         instrs[i++] = PPC_RAW_ORI(_R31, _R31, 0); /* speculation barrier */
0297         instrs[i++] = PPC_RAW_ORI(_R30, _R30, 0); /* L1d flush */
0298     }
0299 
0300     if (types & L1D_FLUSH_MTTRIG)
0301         instrs[i++] = PPC_RAW_MTSPR(SPRN_TRIG2, _R0);
0302 
0303     for (i = 0; start < end; start++, i++) {
0304         dest = (void *)start + *start;
0305 
0306         pr_devel("patching dest %lx\n", (unsigned long)dest);
0307 
0308         patch_instruction(dest, ppc_inst(instrs[0]));
0309 
0310         patch_instruction(dest + 1, ppc_inst(instrs[1]));
0311         patch_instruction(dest + 2, ppc_inst(instrs[2]));
0312         patch_instruction(dest + 3, ppc_inst(instrs[3]));
0313     }
0314 
0315     printk(KERN_DEBUG "uaccess-flush: patched %d locations (%s flush)\n", i,
0316         (types == L1D_FLUSH_NONE)       ? "no" :
0317         (types == L1D_FLUSH_FALLBACK)   ? "fallback displacement" :
0318         (types &  L1D_FLUSH_ORI)        ? (types & L1D_FLUSH_MTTRIG)
0319                             ? "ori+mttrig type"
0320                             : "ori type" :
0321         (types &  L1D_FLUSH_MTTRIG)     ? "mttrig type"
0322                         : "unknown");
0323 }
0324 
0325 static int __do_entry_flush_fixups(void *data)
0326 {
0327     enum l1d_flush_type types = *(enum l1d_flush_type *)data;
0328     unsigned int instrs[3], *dest;
0329     long *start, *end;
0330     int i;
0331 
0332     instrs[0] = PPC_RAW_NOP();
0333     instrs[1] = PPC_RAW_NOP();
0334     instrs[2] = PPC_RAW_NOP();
0335 
0336     i = 0;
0337     if (types == L1D_FLUSH_FALLBACK) {
0338         instrs[i++] = PPC_RAW_MFLR(_R10);
0339         instrs[i++] = PPC_RAW_NOP(); /* branch patched below */
0340         instrs[i++] = PPC_RAW_MTLR(_R10);
0341     }
0342 
0343     if (types & L1D_FLUSH_ORI) {
0344         instrs[i++] = PPC_RAW_ORI(_R31, _R31, 0); /* speculation barrier */
0345         instrs[i++] = PPC_RAW_ORI(_R30, _R30, 0); /* L1d flush */
0346     }
0347 
0348     if (types & L1D_FLUSH_MTTRIG)
0349         instrs[i++] = PPC_RAW_MTSPR(SPRN_TRIG2, _R0);
0350 
0351     /*
0352      * If we're patching in or out the fallback flush we need to be careful about the
0353      * order in which we patch instructions. That's because it's possible we could
0354      * take a page fault after patching one instruction, so the sequence of
0355      * instructions must be safe even in a half patched state.
0356      *
0357      * To make that work, when patching in the fallback flush we patch in this order:
0358      *  - the mflr      (dest)
0359      *  - the mtlr      (dest + 2)
0360      *  - the branch    (dest + 1)
0361      *
0362      * That ensures the sequence is safe to execute at any point. In contrast if we
0363      * patch the mtlr last, it's possible we could return from the branch and not
0364      * restore LR, leading to a crash later.
0365      *
0366      * When patching out the fallback flush (either with nops or another flush type),
0367      * we patch in this order:
0368      *  - the branch    (dest + 1)
0369      *  - the mtlr      (dest + 2)
0370      *  - the mflr      (dest)
0371      *
0372      * Note we are protected by stop_machine() from other CPUs executing the code in a
0373      * semi-patched state.
0374      */
0375 
0376     start = PTRRELOC(&__start___entry_flush_fixup);
0377     end = PTRRELOC(&__stop___entry_flush_fixup);
0378     for (i = 0; start < end; start++, i++) {
0379         dest = (void *)start + *start;
0380 
0381         pr_devel("patching dest %lx\n", (unsigned long)dest);
0382 
0383         if (types == L1D_FLUSH_FALLBACK) {
0384             patch_instruction(dest, ppc_inst(instrs[0]));
0385             patch_instruction(dest + 2, ppc_inst(instrs[2]));
0386             patch_branch(dest + 1,
0387                      (unsigned long)&entry_flush_fallback, BRANCH_SET_LINK);
0388         } else {
0389             patch_instruction(dest + 1, ppc_inst(instrs[1]));
0390             patch_instruction(dest + 2, ppc_inst(instrs[2]));
0391             patch_instruction(dest, ppc_inst(instrs[0]));
0392         }
0393     }
0394 
0395     start = PTRRELOC(&__start___scv_entry_flush_fixup);
0396     end = PTRRELOC(&__stop___scv_entry_flush_fixup);
0397     for (; start < end; start++, i++) {
0398         dest = (void *)start + *start;
0399 
0400         pr_devel("patching dest %lx\n", (unsigned long)dest);
0401 
0402         if (types == L1D_FLUSH_FALLBACK) {
0403             patch_instruction(dest, ppc_inst(instrs[0]));
0404             patch_instruction(dest + 2, ppc_inst(instrs[2]));
0405             patch_branch(dest + 1,
0406                      (unsigned long)&scv_entry_flush_fallback, BRANCH_SET_LINK);
0407         } else {
0408             patch_instruction(dest + 1, ppc_inst(instrs[1]));
0409             patch_instruction(dest + 2, ppc_inst(instrs[2]));
0410             patch_instruction(dest, ppc_inst(instrs[0]));
0411         }
0412     }
0413 
0414 
0415     printk(KERN_DEBUG "entry-flush: patched %d locations (%s flush)\n", i,
0416         (types == L1D_FLUSH_NONE)       ? "no" :
0417         (types == L1D_FLUSH_FALLBACK)   ? "fallback displacement" :
0418         (types &  L1D_FLUSH_ORI)        ? (types & L1D_FLUSH_MTTRIG)
0419                             ? "ori+mttrig type"
0420                             : "ori type" :
0421         (types &  L1D_FLUSH_MTTRIG)     ? "mttrig type"
0422                         : "unknown");
0423 
0424     return 0;
0425 }
0426 
0427 void do_entry_flush_fixups(enum l1d_flush_type types)
0428 {
0429     /*
0430      * The call to the fallback flush can not be safely patched in/out while
0431      * other CPUs are executing it. So call __do_entry_flush_fixups() on one
0432      * CPU while all other CPUs spin in the stop machine core with interrupts
0433      * hard disabled.
0434      */
0435     stop_machine(__do_entry_flush_fixups, &types, NULL);
0436 }
0437 
0438 static int __do_rfi_flush_fixups(void *data)
0439 {
0440     enum l1d_flush_type types = *(enum l1d_flush_type *)data;
0441     unsigned int instrs[3], *dest;
0442     long *start, *end;
0443     int i;
0444 
0445     start = PTRRELOC(&__start___rfi_flush_fixup);
0446     end = PTRRELOC(&__stop___rfi_flush_fixup);
0447 
0448     instrs[0] = PPC_RAW_NOP();
0449     instrs[1] = PPC_RAW_NOP();
0450     instrs[2] = PPC_RAW_NOP();
0451 
0452     if (types & L1D_FLUSH_FALLBACK)
0453         /* b .+16 to fallback flush */
0454         instrs[0] = PPC_RAW_BRANCH(16);
0455 
0456     i = 0;
0457     if (types & L1D_FLUSH_ORI) {
0458         instrs[i++] = PPC_RAW_ORI(_R31, _R31, 0); /* speculation barrier */
0459         instrs[i++] = PPC_RAW_ORI(_R30, _R30, 0); /* L1d flush */
0460     }
0461 
0462     if (types & L1D_FLUSH_MTTRIG)
0463         instrs[i++] = PPC_RAW_MTSPR(SPRN_TRIG2, _R0);
0464 
0465     for (i = 0; start < end; start++, i++) {
0466         dest = (void *)start + *start;
0467 
0468         pr_devel("patching dest %lx\n", (unsigned long)dest);
0469 
0470         patch_instruction(dest, ppc_inst(instrs[0]));
0471         patch_instruction(dest + 1, ppc_inst(instrs[1]));
0472         patch_instruction(dest + 2, ppc_inst(instrs[2]));
0473     }
0474 
0475     printk(KERN_DEBUG "rfi-flush: patched %d locations (%s flush)\n", i,
0476         (types == L1D_FLUSH_NONE)       ? "no" :
0477         (types == L1D_FLUSH_FALLBACK)   ? "fallback displacement" :
0478         (types &  L1D_FLUSH_ORI)        ? (types & L1D_FLUSH_MTTRIG)
0479                             ? "ori+mttrig type"
0480                             : "ori type" :
0481         (types &  L1D_FLUSH_MTTRIG)     ? "mttrig type"
0482                         : "unknown");
0483 
0484     return 0;
0485 }
0486 
0487 void do_rfi_flush_fixups(enum l1d_flush_type types)
0488 {
0489     /*
0490      * stop_machine gets all CPUs out of the interrupt exit handler same
0491      * as do_stf_barrier_fixups. do_rfi_flush_fixups patching can run
0492      * without stop_machine, so this could be achieved with a broadcast
0493      * IPI instead, but this matches the stf sequence.
0494      */
0495 
0496     // Prevent static key update races with do_stf_barrier_fixups()
0497     mutex_lock(&exit_flush_lock);
0498     static_branch_enable(&interrupt_exit_not_reentrant);
0499 
0500     stop_machine(__do_rfi_flush_fixups, &types, NULL);
0501 
0502     if (types & L1D_FLUSH_FALLBACK)
0503         rfi_exit_reentrant = false;
0504     else
0505         rfi_exit_reentrant = true;
0506 
0507     if (stf_exit_reentrant && rfi_exit_reentrant)
0508         static_branch_disable(&interrupt_exit_not_reentrant);
0509 
0510     mutex_unlock(&exit_flush_lock);
0511 }
0512 
0513 void do_barrier_nospec_fixups_range(bool enable, void *fixup_start, void *fixup_end)
0514 {
0515     unsigned int instr, *dest;
0516     long *start, *end;
0517     int i;
0518 
0519     start = fixup_start;
0520     end = fixup_end;
0521 
0522     instr = PPC_RAW_NOP();
0523 
0524     if (enable) {
0525         pr_info("barrier-nospec: using ORI speculation barrier\n");
0526         instr = PPC_RAW_ORI(_R31, _R31, 0); /* speculation barrier */
0527     }
0528 
0529     for (i = 0; start < end; start++, i++) {
0530         dest = (void *)start + *start;
0531 
0532         pr_devel("patching dest %lx\n", (unsigned long)dest);
0533         patch_instruction(dest, ppc_inst(instr));
0534     }
0535 
0536     printk(KERN_DEBUG "barrier-nospec: patched %d locations\n", i);
0537 }
0538 
0539 #endif /* CONFIG_PPC_BOOK3S_64 */
0540 
0541 #ifdef CONFIG_PPC_BARRIER_NOSPEC
0542 void do_barrier_nospec_fixups(bool enable)
0543 {
0544     void *start, *end;
0545 
0546     start = PTRRELOC(&__start___barrier_nospec_fixup);
0547     end = PTRRELOC(&__stop___barrier_nospec_fixup);
0548 
0549     do_barrier_nospec_fixups_range(enable, start, end);
0550 }
0551 #endif /* CONFIG_PPC_BARRIER_NOSPEC */
0552 
0553 #ifdef CONFIG_PPC_FSL_BOOK3E
0554 void do_barrier_nospec_fixups_range(bool enable, void *fixup_start, void *fixup_end)
0555 {
0556     unsigned int instr[2], *dest;
0557     long *start, *end;
0558     int i;
0559 
0560     start = fixup_start;
0561     end = fixup_end;
0562 
0563     instr[0] = PPC_RAW_NOP();
0564     instr[1] = PPC_RAW_NOP();
0565 
0566     if (enable) {
0567         pr_info("barrier-nospec: using isync; sync as speculation barrier\n");
0568         instr[0] = PPC_RAW_ISYNC();
0569         instr[1] = PPC_RAW_SYNC();
0570     }
0571 
0572     for (i = 0; start < end; start++, i++) {
0573         dest = (void *)start + *start;
0574 
0575         pr_devel("patching dest %lx\n", (unsigned long)dest);
0576         patch_instruction(dest, ppc_inst(instr[0]));
0577         patch_instruction(dest + 1, ppc_inst(instr[1]));
0578     }
0579 
0580     printk(KERN_DEBUG "barrier-nospec: patched %d locations\n", i);
0581 }
0582 
0583 static void __init patch_btb_flush_section(long *curr)
0584 {
0585     unsigned int *start, *end;
0586 
0587     start = (void *)curr + *curr;
0588     end = (void *)curr + *(curr + 1);
0589     for (; start < end; start++) {
0590         pr_devel("patching dest %lx\n", (unsigned long)start);
0591         patch_instruction(start, ppc_inst(PPC_RAW_NOP()));
0592     }
0593 }
0594 
0595 void __init do_btb_flush_fixups(void)
0596 {
0597     long *start, *end;
0598 
0599     start = PTRRELOC(&__start__btb_flush_fixup);
0600     end = PTRRELOC(&__stop__btb_flush_fixup);
0601 
0602     for (; start < end; start += 2)
0603         patch_btb_flush_section(start);
0604 }
0605 #endif /* CONFIG_PPC_FSL_BOOK3E */
0606 
0607 void do_lwsync_fixups(unsigned long value, void *fixup_start, void *fixup_end)
0608 {
0609     long *start, *end;
0610     u32 *dest;
0611 
0612     if (!(value & CPU_FTR_LWSYNC))
0613         return ;
0614 
0615     start = fixup_start;
0616     end = fixup_end;
0617 
0618     for (; start < end; start++) {
0619         dest = (void *)start + *start;
0620         raw_patch_instruction(dest, ppc_inst(PPC_INST_LWSYNC));
0621     }
0622 }
0623 
0624 static void __init do_final_fixups(void)
0625 {
0626 #if defined(CONFIG_PPC64) && defined(CONFIG_RELOCATABLE)
0627     ppc_inst_t inst;
0628     u32 *src, *dest, *end;
0629 
0630     if (PHYSICAL_START == 0)
0631         return;
0632 
0633     src = (u32 *)(KERNELBASE + PHYSICAL_START);
0634     dest = (u32 *)KERNELBASE;
0635     end = (void *)src + (__end_interrupts - _stext);
0636 
0637     while (src < end) {
0638         inst = ppc_inst_read(src);
0639         raw_patch_instruction(dest, inst);
0640         src = ppc_inst_next(src, src);
0641         dest = ppc_inst_next(dest, dest);
0642     }
0643 #endif
0644 }
0645 
0646 static unsigned long __initdata saved_cpu_features;
0647 static unsigned int __initdata saved_mmu_features;
0648 #ifdef CONFIG_PPC64
0649 static unsigned long __initdata saved_firmware_features;
0650 #endif
0651 
0652 void __init apply_feature_fixups(void)
0653 {
0654     struct cpu_spec *spec = PTRRELOC(*PTRRELOC(&cur_cpu_spec));
0655 
0656     *PTRRELOC(&saved_cpu_features) = spec->cpu_features;
0657     *PTRRELOC(&saved_mmu_features) = spec->mmu_features;
0658 
0659     /*
0660      * Apply the CPU-specific and firmware specific fixups to kernel text
0661      * (nop out sections not relevant to this CPU or this firmware).
0662      */
0663     do_feature_fixups(spec->cpu_features,
0664               PTRRELOC(&__start___ftr_fixup),
0665               PTRRELOC(&__stop___ftr_fixup));
0666 
0667     do_feature_fixups(spec->mmu_features,
0668               PTRRELOC(&__start___mmu_ftr_fixup),
0669               PTRRELOC(&__stop___mmu_ftr_fixup));
0670 
0671     do_lwsync_fixups(spec->cpu_features,
0672              PTRRELOC(&__start___lwsync_fixup),
0673              PTRRELOC(&__stop___lwsync_fixup));
0674 
0675 #ifdef CONFIG_PPC64
0676     saved_firmware_features = powerpc_firmware_features;
0677     do_feature_fixups(powerpc_firmware_features,
0678               &__start___fw_ftr_fixup, &__stop___fw_ftr_fixup);
0679 #endif
0680     do_final_fixups();
0681 }
0682 
0683 void __init setup_feature_keys(void)
0684 {
0685     /*
0686      * Initialise jump label. This causes all the cpu/mmu_has_feature()
0687      * checks to take on their correct polarity based on the current set of
0688      * CPU/MMU features.
0689      */
0690     jump_label_init();
0691     cpu_feature_keys_init();
0692     mmu_feature_keys_init();
0693 }
0694 
0695 static int __init check_features(void)
0696 {
0697     WARN(saved_cpu_features != cur_cpu_spec->cpu_features,
0698          "CPU features changed after feature patching!\n");
0699     WARN(saved_mmu_features != cur_cpu_spec->mmu_features,
0700          "MMU features changed after feature patching!\n");
0701 #ifdef CONFIG_PPC64
0702     WARN(saved_firmware_features != powerpc_firmware_features,
0703          "Firmware features changed after feature patching!\n");
0704 #endif
0705 
0706     return 0;
0707 }
0708 late_initcall(check_features);
0709 
0710 #ifdef CONFIG_FTR_FIXUP_SELFTEST
0711 
0712 #define check(x)    \
0713     if (!(x)) printk("feature-fixups: test failed at line %d\n", __LINE__);
0714 
0715 /* This must be after the text it fixes up, vmlinux.lds.S enforces that atm */
0716 static struct fixup_entry fixup;
0717 
0718 static long __init calc_offset(struct fixup_entry *entry, unsigned int *p)
0719 {
0720     return (unsigned long)p - (unsigned long)entry;
0721 }
0722 
0723 static void __init test_basic_patching(void)
0724 {
0725     extern unsigned int ftr_fixup_test1[];
0726     extern unsigned int end_ftr_fixup_test1[];
0727     extern unsigned int ftr_fixup_test1_orig[];
0728     extern unsigned int ftr_fixup_test1_expected[];
0729     int size = 4 * (end_ftr_fixup_test1 - ftr_fixup_test1);
0730 
0731     fixup.value = fixup.mask = 8;
0732     fixup.start_off = calc_offset(&fixup, ftr_fixup_test1 + 1);
0733     fixup.end_off = calc_offset(&fixup, ftr_fixup_test1 + 2);
0734     fixup.alt_start_off = fixup.alt_end_off = 0;
0735 
0736     /* Sanity check */
0737     check(memcmp(ftr_fixup_test1, ftr_fixup_test1_orig, size) == 0);
0738 
0739     /* Check we don't patch if the value matches */
0740     patch_feature_section(8, &fixup);
0741     check(memcmp(ftr_fixup_test1, ftr_fixup_test1_orig, size) == 0);
0742 
0743     /* Check we do patch if the value doesn't match */
0744     patch_feature_section(0, &fixup);
0745     check(memcmp(ftr_fixup_test1, ftr_fixup_test1_expected, size) == 0);
0746 
0747     /* Check we do patch if the mask doesn't match */
0748     memcpy(ftr_fixup_test1, ftr_fixup_test1_orig, size);
0749     check(memcmp(ftr_fixup_test1, ftr_fixup_test1_orig, size) == 0);
0750     patch_feature_section(~8, &fixup);
0751     check(memcmp(ftr_fixup_test1, ftr_fixup_test1_expected, size) == 0);
0752 }
0753 
0754 static void __init test_alternative_patching(void)
0755 {
0756     extern unsigned int ftr_fixup_test2[];
0757     extern unsigned int end_ftr_fixup_test2[];
0758     extern unsigned int ftr_fixup_test2_orig[];
0759     extern unsigned int ftr_fixup_test2_alt[];
0760     extern unsigned int ftr_fixup_test2_expected[];
0761     int size = 4 * (end_ftr_fixup_test2 - ftr_fixup_test2);
0762 
0763     fixup.value = fixup.mask = 0xF;
0764     fixup.start_off = calc_offset(&fixup, ftr_fixup_test2 + 1);
0765     fixup.end_off = calc_offset(&fixup, ftr_fixup_test2 + 2);
0766     fixup.alt_start_off = calc_offset(&fixup, ftr_fixup_test2_alt);
0767     fixup.alt_end_off = calc_offset(&fixup, ftr_fixup_test2_alt + 1);
0768 
0769     /* Sanity check */
0770     check(memcmp(ftr_fixup_test2, ftr_fixup_test2_orig, size) == 0);
0771 
0772     /* Check we don't patch if the value matches */
0773     patch_feature_section(0xF, &fixup);
0774     check(memcmp(ftr_fixup_test2, ftr_fixup_test2_orig, size) == 0);
0775 
0776     /* Check we do patch if the value doesn't match */
0777     patch_feature_section(0, &fixup);
0778     check(memcmp(ftr_fixup_test2, ftr_fixup_test2_expected, size) == 0);
0779 
0780     /* Check we do patch if the mask doesn't match */
0781     memcpy(ftr_fixup_test2, ftr_fixup_test2_orig, size);
0782     check(memcmp(ftr_fixup_test2, ftr_fixup_test2_orig, size) == 0);
0783     patch_feature_section(~0xF, &fixup);
0784     check(memcmp(ftr_fixup_test2, ftr_fixup_test2_expected, size) == 0);
0785 }
0786 
0787 static void __init test_alternative_case_too_big(void)
0788 {
0789     extern unsigned int ftr_fixup_test3[];
0790     extern unsigned int end_ftr_fixup_test3[];
0791     extern unsigned int ftr_fixup_test3_orig[];
0792     extern unsigned int ftr_fixup_test3_alt[];
0793     int size = 4 * (end_ftr_fixup_test3 - ftr_fixup_test3);
0794 
0795     fixup.value = fixup.mask = 0xC;
0796     fixup.start_off = calc_offset(&fixup, ftr_fixup_test3 + 1);
0797     fixup.end_off = calc_offset(&fixup, ftr_fixup_test3 + 2);
0798     fixup.alt_start_off = calc_offset(&fixup, ftr_fixup_test3_alt);
0799     fixup.alt_end_off = calc_offset(&fixup, ftr_fixup_test3_alt + 2);
0800 
0801     /* Sanity check */
0802     check(memcmp(ftr_fixup_test3, ftr_fixup_test3_orig, size) == 0);
0803 
0804     /* Expect nothing to be patched, and the error returned to us */
0805     check(patch_feature_section(0xF, &fixup) == 1);
0806     check(memcmp(ftr_fixup_test3, ftr_fixup_test3_orig, size) == 0);
0807     check(patch_feature_section(0, &fixup) == 1);
0808     check(memcmp(ftr_fixup_test3, ftr_fixup_test3_orig, size) == 0);
0809     check(patch_feature_section(~0xF, &fixup) == 1);
0810     check(memcmp(ftr_fixup_test3, ftr_fixup_test3_orig, size) == 0);
0811 }
0812 
0813 static void __init test_alternative_case_too_small(void)
0814 {
0815     extern unsigned int ftr_fixup_test4[];
0816     extern unsigned int end_ftr_fixup_test4[];
0817     extern unsigned int ftr_fixup_test4_orig[];
0818     extern unsigned int ftr_fixup_test4_alt[];
0819     extern unsigned int ftr_fixup_test4_expected[];
0820     int size = 4 * (end_ftr_fixup_test4 - ftr_fixup_test4);
0821     unsigned long flag;
0822 
0823     /* Check a high-bit flag */
0824     flag = 1UL << ((sizeof(unsigned long) - 1) * 8);
0825     fixup.value = fixup.mask = flag;
0826     fixup.start_off = calc_offset(&fixup, ftr_fixup_test4 + 1);
0827     fixup.end_off = calc_offset(&fixup, ftr_fixup_test4 + 5);
0828     fixup.alt_start_off = calc_offset(&fixup, ftr_fixup_test4_alt);
0829     fixup.alt_end_off = calc_offset(&fixup, ftr_fixup_test4_alt + 2);
0830 
0831     /* Sanity check */
0832     check(memcmp(ftr_fixup_test4, ftr_fixup_test4_orig, size) == 0);
0833 
0834     /* Check we don't patch if the value matches */
0835     patch_feature_section(flag, &fixup);
0836     check(memcmp(ftr_fixup_test4, ftr_fixup_test4_orig, size) == 0);
0837 
0838     /* Check we do patch if the value doesn't match */
0839     patch_feature_section(0, &fixup);
0840     check(memcmp(ftr_fixup_test4, ftr_fixup_test4_expected, size) == 0);
0841 
0842     /* Check we do patch if the mask doesn't match */
0843     memcpy(ftr_fixup_test4, ftr_fixup_test4_orig, size);
0844     check(memcmp(ftr_fixup_test4, ftr_fixup_test4_orig, size) == 0);
0845     patch_feature_section(~flag, &fixup);
0846     check(memcmp(ftr_fixup_test4, ftr_fixup_test4_expected, size) == 0);
0847 }
0848 
0849 static void test_alternative_case_with_branch(void)
0850 {
0851     extern unsigned int ftr_fixup_test5[];
0852     extern unsigned int end_ftr_fixup_test5[];
0853     extern unsigned int ftr_fixup_test5_expected[];
0854     int size = 4 * (end_ftr_fixup_test5 - ftr_fixup_test5);
0855 
0856     check(memcmp(ftr_fixup_test5, ftr_fixup_test5_expected, size) == 0);
0857 }
0858 
0859 static void __init test_alternative_case_with_external_branch(void)
0860 {
0861     extern unsigned int ftr_fixup_test6[];
0862     extern unsigned int end_ftr_fixup_test6[];
0863     extern unsigned int ftr_fixup_test6_expected[];
0864     int size = 4 * (end_ftr_fixup_test6 - ftr_fixup_test6);
0865 
0866     check(memcmp(ftr_fixup_test6, ftr_fixup_test6_expected, size) == 0);
0867 }
0868 
0869 static void __init test_alternative_case_with_branch_to_end(void)
0870 {
0871     extern unsigned int ftr_fixup_test7[];
0872     extern unsigned int end_ftr_fixup_test7[];
0873     extern unsigned int ftr_fixup_test7_expected[];
0874     int size = 4 * (end_ftr_fixup_test7 - ftr_fixup_test7);
0875 
0876     check(memcmp(ftr_fixup_test7, ftr_fixup_test7_expected, size) == 0);
0877 }
0878 
0879 static void __init test_cpu_macros(void)
0880 {
0881     extern u8 ftr_fixup_test_FTR_macros[];
0882     extern u8 ftr_fixup_test_FTR_macros_expected[];
0883     unsigned long size = ftr_fixup_test_FTR_macros_expected -
0884                  ftr_fixup_test_FTR_macros;
0885 
0886     /* The fixups have already been done for us during boot */
0887     check(memcmp(ftr_fixup_test_FTR_macros,
0888              ftr_fixup_test_FTR_macros_expected, size) == 0);
0889 }
0890 
0891 static void __init test_fw_macros(void)
0892 {
0893 #ifdef CONFIG_PPC64
0894     extern u8 ftr_fixup_test_FW_FTR_macros[];
0895     extern u8 ftr_fixup_test_FW_FTR_macros_expected[];
0896     unsigned long size = ftr_fixup_test_FW_FTR_macros_expected -
0897                  ftr_fixup_test_FW_FTR_macros;
0898 
0899     /* The fixups have already been done for us during boot */
0900     check(memcmp(ftr_fixup_test_FW_FTR_macros,
0901              ftr_fixup_test_FW_FTR_macros_expected, size) == 0);
0902 #endif
0903 }
0904 
0905 static void __init test_lwsync_macros(void)
0906 {
0907     extern u8 lwsync_fixup_test[];
0908     extern u8 end_lwsync_fixup_test[];
0909     extern u8 lwsync_fixup_test_expected_LWSYNC[];
0910     extern u8 lwsync_fixup_test_expected_SYNC[];
0911     unsigned long size = end_lwsync_fixup_test -
0912                  lwsync_fixup_test;
0913 
0914     /* The fixups have already been done for us during boot */
0915     if (cur_cpu_spec->cpu_features & CPU_FTR_LWSYNC) {
0916         check(memcmp(lwsync_fixup_test,
0917                  lwsync_fixup_test_expected_LWSYNC, size) == 0);
0918     } else {
0919         check(memcmp(lwsync_fixup_test,
0920                  lwsync_fixup_test_expected_SYNC, size) == 0);
0921     }
0922 }
0923 
0924 #ifdef CONFIG_PPC64
0925 static void __init test_prefix_patching(void)
0926 {
0927     extern unsigned int ftr_fixup_prefix1[];
0928     extern unsigned int end_ftr_fixup_prefix1[];
0929     extern unsigned int ftr_fixup_prefix1_orig[];
0930     extern unsigned int ftr_fixup_prefix1_expected[];
0931     int size = sizeof(unsigned int) * (end_ftr_fixup_prefix1 - ftr_fixup_prefix1);
0932 
0933     fixup.value = fixup.mask = 8;
0934     fixup.start_off = calc_offset(&fixup, ftr_fixup_prefix1 + 1);
0935     fixup.end_off = calc_offset(&fixup, ftr_fixup_prefix1 + 3);
0936     fixup.alt_start_off = fixup.alt_end_off = 0;
0937 
0938     /* Sanity check */
0939     check(memcmp(ftr_fixup_prefix1, ftr_fixup_prefix1_orig, size) == 0);
0940 
0941     patch_feature_section(0, &fixup);
0942     check(memcmp(ftr_fixup_prefix1, ftr_fixup_prefix1_expected, size) == 0);
0943     check(memcmp(ftr_fixup_prefix1, ftr_fixup_prefix1_orig, size) != 0);
0944 }
0945 
0946 static void __init test_prefix_alt_patching(void)
0947 {
0948     extern unsigned int ftr_fixup_prefix2[];
0949     extern unsigned int end_ftr_fixup_prefix2[];
0950     extern unsigned int ftr_fixup_prefix2_orig[];
0951     extern unsigned int ftr_fixup_prefix2_expected[];
0952     extern unsigned int ftr_fixup_prefix2_alt[];
0953     int size = sizeof(unsigned int) * (end_ftr_fixup_prefix2 - ftr_fixup_prefix2);
0954 
0955     fixup.value = fixup.mask = 8;
0956     fixup.start_off = calc_offset(&fixup, ftr_fixup_prefix2 + 1);
0957     fixup.end_off = calc_offset(&fixup, ftr_fixup_prefix2 + 3);
0958     fixup.alt_start_off = calc_offset(&fixup, ftr_fixup_prefix2_alt);
0959     fixup.alt_end_off = calc_offset(&fixup, ftr_fixup_prefix2_alt + 2);
0960     /* Sanity check */
0961     check(memcmp(ftr_fixup_prefix2, ftr_fixup_prefix2_orig, size) == 0);
0962 
0963     patch_feature_section(0, &fixup);
0964     check(memcmp(ftr_fixup_prefix2, ftr_fixup_prefix2_expected, size) == 0);
0965     check(memcmp(ftr_fixup_prefix2, ftr_fixup_prefix2_orig, size) != 0);
0966 }
0967 
0968 static void __init test_prefix_word_alt_patching(void)
0969 {
0970     extern unsigned int ftr_fixup_prefix3[];
0971     extern unsigned int end_ftr_fixup_prefix3[];
0972     extern unsigned int ftr_fixup_prefix3_orig[];
0973     extern unsigned int ftr_fixup_prefix3_expected[];
0974     extern unsigned int ftr_fixup_prefix3_alt[];
0975     int size = sizeof(unsigned int) * (end_ftr_fixup_prefix3 - ftr_fixup_prefix3);
0976 
0977     fixup.value = fixup.mask = 8;
0978     fixup.start_off = calc_offset(&fixup, ftr_fixup_prefix3 + 1);
0979     fixup.end_off = calc_offset(&fixup, ftr_fixup_prefix3 + 4);
0980     fixup.alt_start_off = calc_offset(&fixup, ftr_fixup_prefix3_alt);
0981     fixup.alt_end_off = calc_offset(&fixup, ftr_fixup_prefix3_alt + 3);
0982     /* Sanity check */
0983     check(memcmp(ftr_fixup_prefix3, ftr_fixup_prefix3_orig, size) == 0);
0984 
0985     patch_feature_section(0, &fixup);
0986     check(memcmp(ftr_fixup_prefix3, ftr_fixup_prefix3_expected, size) == 0);
0987     patch_feature_section(0, &fixup);
0988     check(memcmp(ftr_fixup_prefix3, ftr_fixup_prefix3_orig, size) != 0);
0989 }
0990 #else
0991 static inline void test_prefix_patching(void) {}
0992 static inline void test_prefix_alt_patching(void) {}
0993 static inline void test_prefix_word_alt_patching(void) {}
0994 #endif /* CONFIG_PPC64 */
0995 
0996 static int __init test_feature_fixups(void)
0997 {
0998     printk(KERN_DEBUG "Running feature fixup self-tests ...\n");
0999 
1000     test_basic_patching();
1001     test_alternative_patching();
1002     test_alternative_case_too_big();
1003     test_alternative_case_too_small();
1004     test_alternative_case_with_branch();
1005     test_alternative_case_with_external_branch();
1006     test_alternative_case_with_branch_to_end();
1007     test_cpu_macros();
1008     test_fw_macros();
1009     test_lwsync_macros();
1010     test_prefix_patching();
1011     test_prefix_alt_patching();
1012     test_prefix_word_alt_patching();
1013 
1014     return 0;
1015 }
1016 late_initcall(test_feature_fixups);
1017 
1018 #endif /* CONFIG_FTR_FIXUP_SELFTEST */
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!