powerpc/kernel/misc_64.S

0001 /* SPDX-License-Identifier: GPL-2.0-or-later */
0002 /*
0003  * This file contains miscellaneous low-level functions.
0004  *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
0005  *
0006  * Largely rewritten by Cort Dougan (cort@cs.nmt.edu)
0007  * and Paul Mackerras.
0008  * Adapted for iSeries by Mike Corrigan (mikejc@us.ibm.com)
0009  * PPC64 updates by Dave Engebretsen (engebret@us.ibm.com)
0010  */
0011
0012 #include <linux/sys.h>
0013 #include <asm/unistd.h>
0014 #include <asm/errno.h>
0015 #include <asm/processor.h>
0016 #include <asm/page.h>
0017 #include <asm/cache.h>
0018 #include <asm/ppc_asm.h>
0019 #include <asm/asm-offsets.h>
0020 #include <asm/cputable.h>
0021 #include <asm/thread_info.h>
0022 #include <asm/kexec.h>
0023 #include <asm/ptrace.h>
0024 #include <asm/mmu.h>
0025 #include <asm/export.h>
0026 #include <asm/feature-fixups.h>
0027
0028     .text
0029
0030 _GLOBAL(__bswapdi2)
0031 EXPORT_SYMBOL(__bswapdi2)
0032     srdi    r8,r3,32
0033     rlwinm  r7,r3,8,0xffffffff
0034     rlwimi  r7,r3,24,0,7
0035     rlwinm  r9,r8,8,0xffffffff
0036     rlwimi  r7,r3,24,16,23
0037     rlwimi  r9,r8,24,0,7
0038     rlwimi  r9,r8,24,16,23
0039     sldi    r7,r7,32
0040     or  r3,r7,r9
0041     blr
0042
0043
0044 #ifdef CONFIG_PPC_EARLY_DEBUG_BOOTX
0045 _GLOBAL(rmci_on)
0046     sync
0047     isync
0048     li  r3,0x100
0049     rldicl  r3,r3,32,0
0050     mfspr   r5,SPRN_HID4
0051     or  r5,r5,r3
0052     sync
0053     mtspr   SPRN_HID4,r5
0054     isync
0055     slbia
0056     isync
0057     sync
0058     blr
0059
0060 _GLOBAL(rmci_off)
0061     sync
0062     isync
0063     li  r3,0x100
0064     rldicl  r3,r3,32,0
0065     mfspr   r5,SPRN_HID4
0066     andc    r5,r5,r3
0067     sync
0068     mtspr   SPRN_HID4,r5
0069     isync
0070     slbia
0071     isync
0072     sync
0073     blr
0074 #endif /* CONFIG_PPC_EARLY_DEBUG_BOOTX */
0075
0076 #if defined(CONFIG_PPC_PMAC) || defined(CONFIG_PPC_MAPLE)
0077
0078 /*
0079  * Do an IO access in real mode
0080  */
0081 _GLOBAL(real_readb)
0082     mfmsr   r7
0083     ori r0,r7,MSR_DR
0084     xori    r0,r0,MSR_DR
0085     sync
0086     mtmsrd  r0
0087     sync
0088     isync
0089     mfspr   r6,SPRN_HID4
0090     rldicl  r5,r6,32,0
0091     ori r5,r5,0x100
0092     rldicl  r5,r5,32,0
0093     sync
0094     mtspr   SPRN_HID4,r5
0095     isync
0096     slbia
0097     isync
0098     lbz r3,0(r3)
0099     sync
0100     mtspr   SPRN_HID4,r6
0101     isync
0102     slbia
0103     isync
0104     mtmsrd  r7
0105     sync
0106     isync
0107     blr
0108
0109     /*
0110  * Do an IO access in real mode
0111  */
0112 _GLOBAL(real_writeb)
0113     mfmsr   r7
0114     ori r0,r7,MSR_DR
0115     xori    r0,r0,MSR_DR
0116     sync
0117     mtmsrd  r0
0118     sync
0119     isync
0120     mfspr   r6,SPRN_HID4
0121     rldicl  r5,r6,32,0
0122     ori r5,r5,0x100
0123     rldicl  r5,r5,32,0
0124     sync
0125     mtspr   SPRN_HID4,r5
0126     isync
0127     slbia
0128     isync
0129     stb r3,0(r4)
0130     sync
0131     mtspr   SPRN_HID4,r6
0132     isync
0133     slbia
0134     isync
0135     mtmsrd  r7
0136     sync
0137     isync
0138     blr
0139 #endif /* defined(CONFIG_PPC_PMAC) || defined(CONFIG_PPC_MAPLE) */
0140
0141 #ifdef CONFIG_PPC_PASEMI
0142
0143 _GLOBAL(real_205_readb)
0144     mfmsr   r7
0145     ori r0,r7,MSR_DR
0146     xori    r0,r0,MSR_DR
0147     sync
0148     mtmsrd  r0
0149     sync
0150     isync
0151     LBZCIX(R3,R0,R3)
0152     isync
0153     mtmsrd  r7
0154     sync
0155     isync
0156     blr
0157
0158 _GLOBAL(real_205_writeb)
0159     mfmsr   r7
0160     ori r0,r7,MSR_DR
0161     xori    r0,r0,MSR_DR
0162     sync
0163     mtmsrd  r0
0164     sync
0165     isync
0166     STBCIX(R3,R0,R4)
0167     isync
0168     mtmsrd  r7
0169     sync
0170     isync
0171     blr
0172
0173 #endif /* CONFIG_PPC_PASEMI */
0174
0175
0176 #if defined(CONFIG_CPU_FREQ_PMAC64) || defined(CONFIG_CPU_FREQ_MAPLE)
0177 /*
0178  * SCOM access functions for 970 (FX only for now)
0179  *
0180  * unsigned long scom970_read(unsigned int address);
0181  * void scom970_write(unsigned int address, unsigned long value);
0182  *
0183  * The address passed in is the 24 bits register address. This code
0184  * is 970 specific and will not check the status bits, so you should
0185  * know what you are doing.
0186  */
0187 _GLOBAL(scom970_read)
0188     /* interrupts off */
0189     mfmsr   r4
0190     ori r0,r4,MSR_EE
0191     xori    r0,r0,MSR_EE
0192     mtmsrd  r0,1
0193
0194     /* rotate 24 bits SCOM address 8 bits left and mask out it's low 8 bits
0195      * (including parity). On current CPUs they must be 0'd,
0196      * and finally or in RW bit
0197      */
0198     rlwinm  r3,r3,8,0,15
0199     ori r3,r3,0x8000
0200
0201     /* do the actual scom read */
0202     sync
0203     mtspr   SPRN_SCOMC,r3
0204     isync
0205     mfspr   r3,SPRN_SCOMD
0206     isync
0207     mfspr   r0,SPRN_SCOMC
0208     isync
0209
0210     /* XXX: fixup result on some buggy 970's (ouch ! we lost a bit, bah
0211      * that's the best we can do). Not implemented yet as we don't use
0212      * the scom on any of the bogus CPUs yet, but may have to be done
0213      * ultimately
0214      */
0215
0216     /* restore interrupts */
0217     mtmsrd  r4,1
0218     blr
0219
0220
0221 _GLOBAL(scom970_write)
0222     /* interrupts off */
0223     mfmsr   r5
0224     ori r0,r5,MSR_EE
0225     xori    r0,r0,MSR_EE
0226     mtmsrd  r0,1
0227
0228     /* rotate 24 bits SCOM address 8 bits left and mask out it's low 8 bits
0229      * (including parity). On current CPUs they must be 0'd.
0230      */
0231
0232     rlwinm  r3,r3,8,0,15
0233
0234     sync
0235     mtspr   SPRN_SCOMD,r4      /* write data */
0236     isync
0237     mtspr   SPRN_SCOMC,r3      /* write command */
0238     isync
0239     mfspr   3,SPRN_SCOMC
0240     isync
0241
0242     /* restore interrupts */
0243     mtmsrd  r5,1
0244     blr
0245 #endif /* CONFIG_CPU_FREQ_PMAC64 || CONFIG_CPU_FREQ_MAPLE */
0246
0247 /* kexec_wait(phys_cpu)
0248  *
0249  * wait for the flag to change, indicating this kernel is going away but
0250  * the slave code for the next one is at addresses 0 to 100.
0251  *
0252  * This is used by all slaves, even those that did not find a matching
0253  * paca in the secondary startup code.
0254  *
0255  * Physical (hardware) cpu id should be in r3.
0256  */
0257 _GLOBAL(kexec_wait)
0258     bcl 20,31,$+4
0259 1:  mflr    r5
0260     addi    r5,r5,kexec_flag-1b
0261
0262 99: HMT_LOW
0263 #ifdef CONFIG_KEXEC_CORE    /* use no memory without kexec */
0264     lwz r4,0(r5)
0265     cmpwi   0,r4,0
0266     beq 99b
0267 #ifdef CONFIG_PPC_BOOK3S_64
0268     li  r10,0x60
0269     mfmsr   r11
0270     clrrdi  r11,r11,1   /* Clear MSR_LE */
0271     mtsrr0  r10
0272     mtsrr1  r11
0273     rfid
0274 #else
0275     /* Create TLB entry in book3e_secondary_core_init */
0276     li  r4,0
0277     ba  0x60
0278 #endif
0279 #endif
0280
0281 /* this can be in text because we won't change it until we are
0282  * running in real anyways
0283  */
0284 kexec_flag:
0285     .long   0
0286
0287
0288 #ifdef CONFIG_KEXEC_CORE
0289 #ifdef CONFIG_PPC_BOOK3E
0290 /*
0291  * BOOK3E has no real MMU mode, so we have to setup the initial TLB
0292  * for a core to identity map v:0 to p:0.  This current implementation
0293  * assumes that 1G is enough for kexec.
0294  */
0295 kexec_create_tlb:
0296     /*
0297      * Invalidate all non-IPROT TLB entries to avoid any TLB conflict.
0298      * IPROT TLB entries should be >= PAGE_OFFSET and thus not conflict.
0299      */
0300     PPC_TLBILX_ALL(0,R0)
0301     sync
0302     isync
0303
0304     mfspr   r10,SPRN_TLB1CFG
0305     andi.   r10,r10,TLBnCFG_N_ENTRY /* Extract # entries */
0306     subi    r10,r10,1   /* Last entry: no conflict with kernel text */
0307     lis r9,MAS0_TLBSEL(1)@h
0308     rlwimi  r9,r10,16,4,15      /* Setup MAS0 = TLBSEL | ESEL(r9) */
0309
0310 /* Set up a temp identity mapping v:0 to p:0 and return to it. */
0311     mtspr   SPRN_MAS0,r9
0312
0313     lis r9,(MAS1_VALID|MAS1_IPROT)@h
0314     ori r9,r9,(MAS1_TSIZE(BOOK3E_PAGESZ_1GB))@l
0315     mtspr   SPRN_MAS1,r9
0316
0317     LOAD_REG_IMMEDIATE(r9, 0x0 | MAS2_M_IF_NEEDED)
0318     mtspr   SPRN_MAS2,r9
0319
0320     LOAD_REG_IMMEDIATE(r9, 0x0 | MAS3_SR | MAS3_SW | MAS3_SX)
0321     mtspr   SPRN_MAS3,r9
0322     li  r9,0
0323     mtspr   SPRN_MAS7,r9
0324
0325     tlbwe
0326     isync
0327     blr
0328 #endif
0329
0330 /* kexec_smp_wait(void)
0331  *
0332  * call with interrupts off
0333  * note: this is a terminal routine, it does not save lr
0334  *
0335  * get phys id from paca
0336  * switch to real mode
0337  * mark the paca as no longer used
0338  * join other cpus in kexec_wait(phys_id)
0339  */
0340 _GLOBAL(kexec_smp_wait)
0341     lhz r3,PACAHWCPUID(r13)
0342     bl  real_mode
0343
0344     li  r4,KEXEC_STATE_REAL_MODE
0345     stb r4,PACAKEXECSTATE(r13)
0346
0347     b   kexec_wait
0348
0349 /*
0350  * switch to real mode (turn mmu off)
0351  * we use the early kernel trick that the hardware ignores bits
0352  * 0 and 1 (big endian) of the effective address in real mode
0353  *
0354  * don't overwrite r3 here, it is live for kexec_wait above.
0355  */
0356 real_mode:  /* assume normal blr return */
0357 #ifdef CONFIG_PPC_BOOK3E
0358     /* Create an identity mapping. */
0359     b   kexec_create_tlb
0360 #else
0361 1:  li  r9,MSR_RI
0362     li  r10,MSR_DR|MSR_IR
0363     mflr    r11     /* return address to SRR0 */
0364     mfmsr   r12
0365     andc    r9,r12,r9
0366     andc    r10,r12,r10
0367
0368     mtmsrd  r9,1
0369     mtspr   SPRN_SRR1,r10
0370     mtspr   SPRN_SRR0,r11
0371     rfid
0372 #endif
0373
0374 /*
0375  * kexec_sequence(newstack, start, image, control, clear_all(),
0376               copy_with_mmu_off)
0377  *
0378  * does the grungy work with stack switching and real mode switches
0379  * also does simple calls to other code
0380  */
0381
0382 _GLOBAL(kexec_sequence)
0383     mflr    r0
0384     std r0,16(r1)
0385
0386     /* switch stacks to newstack -- &kexec_stack.stack */
0387     stdu    r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r3)
0388     mr  r1,r3
0389
0390     li  r0,0
0391     std r0,16(r1)
0392
0393     /* save regs for local vars on new stack.
0394      * yes, we won't go back, but ...
0395      */
0396     std r31,-8(r1)
0397     std r30,-16(r1)
0398     std r29,-24(r1)
0399     std r28,-32(r1)
0400     std r27,-40(r1)
0401     std r26,-48(r1)
0402     std r25,-56(r1)
0403
0404     stdu    r1,-STACK_FRAME_OVERHEAD-64(r1)
0405
0406     /* save args into preserved regs */
0407     mr  r31,r3          /* newstack (both) */
0408     mr  r30,r4          /* start (real) */
0409     mr  r29,r5          /* image (virt) */
0410     mr  r28,r6          /* control, unused */
0411     mr  r27,r7          /* clear_all() fn desc */
0412     mr  r26,r8          /* copy_with_mmu_off */
0413     lhz r25,PACAHWCPUID(r13)    /* get our phys cpu from paca */
0414
0415     /* disable interrupts, we are overwriting kernel data next */
0416 #ifdef CONFIG_PPC_BOOK3E
0417     wrteei  0
0418 #else
0419     mfmsr   r3
0420     rlwinm  r3,r3,0,17,15
0421     mtmsrd  r3,1
0422 #endif
0423
0424     /* We need to turn the MMU off unless we are in hash mode
0425      * under a hypervisor
0426      */
0427     cmpdi   r26,0
0428     beq 1f
0429     bl  real_mode
0430 1:
0431     /* copy dest pages, flush whole dest image */
0432     mr  r3,r29
0433     bl  kexec_copy_flush    /* (image) */
0434
0435     /* turn off mmu now if not done earlier */
0436     cmpdi   r26,0
0437     bne 1f
0438     bl  real_mode
0439
0440     /* copy  0x100 bytes starting at start to 0 */
0441 1:  li  r3,0
0442     mr  r4,r30      /* start, aka phys mem offset */
0443     li  r5,0x100
0444     li  r6,0
0445     bl  copy_and_flush  /* (dest, src, copy limit, start offset) */
0446 1:  /* assume normal blr return */
0447
0448     /* release other cpus to the new kernel secondary start at 0x60 */
0449     mflr    r5
0450     li  r6,1
0451     stw r6,kexec_flag-1b(5)
0452
0453     cmpdi   r27,0
0454     beq 1f
0455
0456     /* clear out hardware hash page table and tlb */
0457 #ifdef CONFIG_PPC64_ELF_ABI_V1
0458     ld  r12,0(r27)      /* deref function descriptor */
0459 #else
0460     mr  r12,r27
0461 #endif
0462     mtctr   r12
0463     bctrl               /* mmu_hash_ops.hpte_clear_all(void); */
0464
0465 /*
0466  *   kexec image calling is:
0467  *      the first 0x100 bytes of the entry point are copied to 0
0468  *
0469  *      all slaves branch to slave = 0x60 (absolute)
0470  *              slave(phys_cpu_id);
0471  *
0472  *      master goes to start = entry point
0473  *              start(phys_cpu_id, start, 0);
0474  *
0475  *
0476  *   a wrapper is needed to call existing kernels, here is an approximate
0477  *   description of one method:
0478  *
0479  * v2: (2.6.10)
0480  *   start will be near the boot_block (maybe 0x100 bytes before it?)
0481  *   it will have a 0x60, which will b to boot_block, where it will wait
0482  *   and 0 will store phys into struct boot-block and load r3 from there,
0483  *   copy kernel 0-0x100 and tell slaves to back down to 0x60 again
0484  *
0485  * v1: (2.6.9)
0486  *    boot block will have all cpus scanning device tree to see if they
0487  *    are the boot cpu ?????
0488  *    other device tree differences (prop sizes, va vs pa, etc)...
0489  */
0490 1:  mr  r3,r25  # my phys cpu
0491     mr  r4,r30  # start, aka phys mem offset
0492     mtlr    4
0493     li  r5,0
0494     blr /* image->start(physid, image->start, 0); */
0495 #endif /* CONFIG_KEXEC_CORE */