powerpc/kexec/relocate_32.S

0001 /* SPDX-License-Identifier: GPL-2.0-or-later */
0002 /*
0003  * This file contains kexec low-level functions.
0004  *
0005  * Copyright (C) 2002-2003 Eric Biederman  <ebiederm@xmission.com>
0006  * GameCube/ppc32 port Copyright (C) 2004 Albert Herranz
0007  * PPC44x port. Copyright (C) 2011,  IBM Corporation
0008  *      Author: Suzuki Poulose <suzuki@in.ibm.com>
0009  */
0010
0011 #include <asm/reg.h>
0012 #include <asm/page.h>
0013 #include <asm/mmu.h>
0014 #include <asm/ppc_asm.h>
0015 #include <asm/kexec.h>
0016
0017     .text
0018
0019     /*
0020      * Must be relocatable PIC code callable as a C function.
0021      */
0022     .globl relocate_new_kernel
0023 relocate_new_kernel:
0024     /* r3 = page_list   */
0025     /* r4 = reboot_code_buffer */
0026     /* r5 = start_address      */
0027
0028 #ifdef CONFIG_FSL_BOOKE
0029
0030     mr  r29, r3
0031     mr  r30, r4
0032     mr  r31, r5
0033
0034 #define ENTRY_MAPPING_KEXEC_SETUP
0035 #include <kernel/fsl_booke_entry_mapping.S>
0036 #undef ENTRY_MAPPING_KEXEC_SETUP
0037
0038     mr      r3, r29
0039     mr      r4, r30
0040     mr      r5, r31
0041
0042     li  r0, 0
0043 #elif defined(CONFIG_44x)
0044
0045     /* Save our parameters */
0046     mr  r29, r3
0047     mr  r30, r4
0048     mr  r31, r5
0049
0050 #ifdef CONFIG_PPC_47x
0051     /* Check for 47x cores */
0052     mfspr   r3,SPRN_PVR
0053     srwi    r3,r3,16
0054     cmplwi  cr0,r3,PVR_476FPE@h
0055     beq setup_map_47x
0056     cmplwi  cr0,r3,PVR_476@h
0057     beq setup_map_47x
0058     cmplwi  cr0,r3,PVR_476_ISS@h
0059     beq setup_map_47x
0060 #endif /* CONFIG_PPC_47x */
0061
0062 /*
0063  * Code for setting up 1:1 mapping for PPC440x for KEXEC
0064  *
0065  * We cannot switch off the MMU on PPC44x.
0066  * So we:
0067  * 1) Invalidate all the mappings except the one we are running from.
0068  * 2) Create a tmp mapping for our code in the other address space(TS) and
0069  *    jump to it. Invalidate the entry we started in.
0070  * 3) Create a 1:1 mapping for 0-2GiB in chunks of 256M in original TS.
0071  * 4) Jump to the 1:1 mapping in original TS.
0072  * 5) Invalidate the tmp mapping.
0073  *
0074  * - Based on the kexec support code for FSL BookE
0075  *
0076  */
0077
0078     /*
0079      * Load the PID with kernel PID (0).
0080      * Also load our MSR_IS and TID to MMUCR for TLB search.
0081      */
0082     li  r3, 0
0083     mtspr   SPRN_PID, r3
0084     mfmsr   r4
0085     andi.   r4,r4,MSR_IS@l
0086     beq wmmucr
0087     oris    r3,r3,PPC44x_MMUCR_STS@h
0088 wmmucr:
0089     mtspr   SPRN_MMUCR,r3
0090     sync
0091
0092     /*
0093      * Invalidate all the TLB entries except the current entry
0094      * where we are running from
0095      */
0096     bcl 20,31,$+4           /* Find our address */
0097 0:  mflr    r5              /* Make it accessible */
0098     tlbsx   r23,0,r5            /* Find entry we are in */
0099     li  r4,0                /* Start at TLB entry 0 */
0100     li  r3,0                /* Set PAGEID inval value */
0101 1:  cmpw    r23,r4              /* Is this our entry? */
0102     beq skip                /* If so, skip the inval */
0103     tlbwe   r3,r4,PPC44x_TLB_PAGEID     /* If not, inval the entry */
0104 skip:
0105     addi    r4,r4,1             /* Increment */
0106     cmpwi   r4,64               /* Are we done? */
0107     bne 1b              /* If not, repeat */
0108     isync
0109
0110     /* Create a temp mapping and jump to it */
0111     andi.   r6, r23, 1      /* Find the index to use */
0112     addi    r24, r6, 1      /* r24 will contain 1 or 2 */
0113
0114     mfmsr   r9          /* get the MSR */
0115     rlwinm  r5, r9, 27, 31, 31  /* Extract the MSR[IS] */
0116     xori    r7, r5, 1       /* Use the other address space */
0117
0118     /* Read the current mapping entries */
0119     tlbre   r3, r23, PPC44x_TLB_PAGEID
0120     tlbre   r4, r23, PPC44x_TLB_XLAT
0121     tlbre   r5, r23, PPC44x_TLB_ATTRIB
0122
0123     /* Save our current XLAT entry */
0124     mr  r25, r4
0125
0126     /* Extract the TLB PageSize */
0127     li  r10, 1          /* r10 will hold PageSize */
0128     rlwinm  r11, r3, 0, 24, 27  /* bits 24-27 */
0129
0130     /* XXX: As of now we use 256M, 4K pages */
0131     cmpwi   r11, PPC44x_TLB_256M
0132     bne tlb_4k
0133     rotlwi  r10, r10, 28        /* r10 = 256M */
0134     b   write_out
0135 tlb_4k:
0136     cmpwi   r11, PPC44x_TLB_4K
0137     bne default
0138     rotlwi  r10, r10, 12        /* r10 = 4K */
0139     b   write_out
0140 default:
0141     rotlwi  r10, r10, 10        /* r10 = 1K */
0142
0143 write_out:
0144     /*
0145      * Write out the tmp 1:1 mapping for this code in other address space
0146      * Fixup  EPN = RPN , TS=other address space
0147      */
0148     insrwi  r3, r7, 1, 23       /* Bit 23 is TS for PAGEID field */
0149
0150     /* Write out the tmp mapping entries */
0151     tlbwe   r3, r24, PPC44x_TLB_PAGEID
0152     tlbwe   r4, r24, PPC44x_TLB_XLAT
0153     tlbwe   r5, r24, PPC44x_TLB_ATTRIB
0154
0155     subi    r11, r10, 1     /* PageOffset Mask = PageSize - 1 */
0156     not r10, r11        /* Mask for PageNum */
0157
0158     /* Switch to other address space in MSR */
0159     insrwi  r9, r7, 1, 26       /* Set MSR[IS] = r7 */
0160
0161     bcl 20,31,$+4
0162 1:  mflr    r8
0163     addi    r8, r8, (2f-1b)     /* Find the target offset */
0164
0165     /* Jump to the tmp mapping */
0166     mtspr   SPRN_SRR0, r8
0167     mtspr   SPRN_SRR1, r9
0168     rfi
0169
0170 2:
0171     /* Invalidate the entry we were executing from */
0172     li  r3, 0
0173     tlbwe   r3, r23, PPC44x_TLB_PAGEID
0174
0175     /* attribute fields. rwx for SUPERVISOR mode */
0176     li  r5, 0
0177     ori r5, r5, (PPC44x_TLB_SW | PPC44x_TLB_SR | PPC44x_TLB_SX | PPC44x_TLB_G)
0178
0179     /* Create 1:1 mapping in 256M pages */
0180     xori    r7, r7, 1           /* Revert back to Original TS */
0181
0182     li  r8, 0               /* PageNumber */
0183     li  r6, 3               /* TLB Index, start at 3  */
0184
0185 next_tlb:
0186     rotlwi  r3, r8, 28          /* Create EPN (bits 0-3) */
0187     mr  r4, r3              /* RPN = EPN  */
0188     ori r3, r3, (PPC44x_TLB_VALID | PPC44x_TLB_256M) /* SIZE = 256M, Valid */
0189     insrwi  r3, r7, 1, 23           /* Set TS from r7 */
0190
0191     tlbwe   r3, r6, PPC44x_TLB_PAGEID   /* PageID field : EPN, V, SIZE */
0192     tlbwe   r4, r6, PPC44x_TLB_XLAT     /* Address translation : RPN   */
0193     tlbwe   r5, r6, PPC44x_TLB_ATTRIB   /* Attributes */
0194
0195     addi    r8, r8, 1           /* Increment PN */
0196     addi    r6, r6, 1           /* Increment TLB Index */
0197     cmpwi   r8, 8               /* Are we done ? */
0198     bne next_tlb
0199     isync
0200
0201     /* Jump to the new mapping 1:1 */
0202     li  r9,0
0203     insrwi  r9, r7, 1, 26           /* Set MSR[IS] = r7 */
0204
0205     bcl 20,31,$+4
0206 1:  mflr    r8
0207     and r8, r8, r11         /* Get our offset within page */
0208     addi    r8, r8, (2f-1b)
0209
0210     and r5, r25, r10            /* Get our target PageNum */
0211     or  r8, r8, r5          /* Target jump address */
0212
0213     mtspr   SPRN_SRR0, r8
0214     mtspr   SPRN_SRR1, r9
0215     rfi
0216 2:
0217     /* Invalidate the tmp entry we used */
0218     li  r3, 0
0219     tlbwe   r3, r24, PPC44x_TLB_PAGEID
0220     sync
0221     b   ppc44x_map_done
0222
0223 #ifdef CONFIG_PPC_47x
0224
0225     /* 1:1 mapping for 47x */
0226
0227 setup_map_47x:
0228
0229     /*
0230      * Load the kernel pid (0) to PID and also to MMUCR[TID].
0231      * Also set the MSR IS->MMUCR STS
0232      */
0233     li  r3, 0
0234     mtspr   SPRN_PID, r3            /* Set PID */
0235     mfmsr   r4              /* Get MSR */
0236     andi.   r4, r4, MSR_IS@l        /* TS=1? */
0237     beq 1f              /* If not, leave STS=0 */
0238     oris    r3, r3, PPC47x_MMUCR_STS@h  /* Set STS=1 */
0239 1:  mtspr   SPRN_MMUCR, r3          /* Put MMUCR */
0240     sync
0241
0242     /* Find the entry we are running from */
0243     bcl 20,31,$+4
0244 2:  mflr    r23
0245     tlbsx   r23, 0, r23
0246     tlbre   r24, r23, 0         /* TLB Word 0 */
0247     tlbre   r25, r23, 1         /* TLB Word 1 */
0248     tlbre   r26, r23, 2         /* TLB Word 2 */
0249
0250
0251     /*
0252      * Invalidates all the tlb entries by writing to 256 RPNs(r4)
0253      * of 4k page size in all  4 ways (0-3 in r3).
0254      * This would invalidate the entire UTLB including the one we are
0255      * running from. However the shadow TLB entries would help us
0256      * to continue the execution, until we flush them (rfi/isync).
0257      */
0258     addis   r3, 0, 0x8000           /* specify the way */
0259     addi    r4, 0, 0            /* TLB Word0 = (EPN=0, VALID = 0) */
0260     addi    r5, 0, 0
0261     b   clear_utlb_entry
0262
0263     /* Align the loop to speed things up. from head_44x.S */
0264     .align  6
0265
0266 clear_utlb_entry:
0267
0268     tlbwe   r4, r3, 0
0269     tlbwe   r5, r3, 1
0270     tlbwe   r5, r3, 2
0271     addis   r3, r3, 0x2000          /* Increment the way */
0272     cmpwi   r3, 0
0273     bne clear_utlb_entry
0274     addis   r3, 0, 0x8000
0275     addis   r4, r4, 0x100           /* Increment the EPN */
0276     cmpwi   r4, 0
0277     bne clear_utlb_entry
0278
0279     /* Create the entries in the other address space */
0280     mfmsr   r5
0281     rlwinm  r7, r5, 27, 31, 31      /* Get the TS (Bit 26) from MSR */
0282     xori    r7, r7, 1           /* r7 = !TS */
0283
0284     insrwi  r24, r7, 1, 21          /* Change the TS in the saved TLB word 0 */
0285
0286     /*
0287      * write out the TLB entries for the tmp mapping
0288      * Use way '0' so that we could easily invalidate it later.
0289      */
0290     lis r3, 0x8000          /* Way '0' */
0291
0292     tlbwe   r24, r3, 0
0293     tlbwe   r25, r3, 1
0294     tlbwe   r26, r3, 2
0295
0296     /* Update the msr to the new TS */
0297     insrwi  r5, r7, 1, 26
0298
0299     bcl 20,31,$+4
0300 1:  mflr    r6
0301     addi    r6, r6, (2f-1b)
0302
0303     mtspr   SPRN_SRR0, r6
0304     mtspr   SPRN_SRR1, r5
0305     rfi
0306
0307     /*
0308      * Now we are in the tmp address space.
0309      * Create a 1:1 mapping for 0-2GiB in the original TS.
0310      */
0311 2:
0312     li  r3, 0
0313     li  r4, 0               /* TLB Word 0 */
0314     li  r5, 0               /* TLB Word 1 */
0315     li  r6, 0
0316     ori r6, r6, PPC47x_TLB2_S_RWX   /* TLB word 2 */
0317
0318     li  r8, 0               /* PageIndex */
0319
0320     xori    r7, r7, 1           /* revert back to original TS */
0321
0322 write_utlb:
0323     rotlwi  r5, r8, 28          /* RPN = PageIndex * 256M */
0324                         /* ERPN = 0 as we don't use memory above 2G */
0325
0326     mr  r4, r5              /* EPN = RPN */
0327     ori r4, r4, (PPC47x_TLB0_VALID | PPC47x_TLB0_256M)
0328     insrwi  r4, r7, 1, 21           /* Insert the TS to Word 0 */
0329
0330     tlbwe   r4, r3, 0           /* Write out the entries */
0331     tlbwe   r5, r3, 1
0332     tlbwe   r6, r3, 2
0333     addi    r8, r8, 1
0334     cmpwi   r8, 8               /* Have we completed ? */
0335     bne write_utlb
0336
0337     /* make sure we complete the TLB write up */
0338     isync
0339
0340     /*
0341      * Prepare to jump to the 1:1 mapping.
0342      * 1) Extract page size of the tmp mapping
0343      *    DSIZ = TLB_Word0[22:27]
0344      * 2) Calculate the physical address of the address
0345      *    to jump to.
0346      */
0347     rlwinm  r10, r24, 0, 22, 27
0348
0349     cmpwi   r10, PPC47x_TLB0_4K
0350     bne 0f
0351     li  r10, 0x1000         /* r10 = 4k */
0352     bl  1f
0353
0354 0:
0355     /* Defaults to 256M */
0356     lis r10, 0x1000
0357
0358     bcl 20,31,$+4
0359 1:  mflr    r4
0360     addi    r4, r4, (2f-1b)         /* virtual address  of 2f */
0361
0362     subi    r11, r10, 1         /* offsetmask = Pagesize - 1 */
0363     not r10, r11            /* Pagemask = ~(offsetmask) */
0364
0365     and r5, r25, r10            /* Physical page */
0366     and r6, r4, r11         /* offset within the current page */
0367
0368     or  r5, r5, r6          /* Physical address for 2f */
0369
0370     /* Switch the TS in MSR to the original one */
0371     mfmsr   r8
0372     insrwi  r8, r7, 1, 26
0373
0374     mtspr   SPRN_SRR1, r8
0375     mtspr   SPRN_SRR0, r5
0376     rfi
0377
0378 2:
0379     /* Invalidate the tmp mapping */
0380     lis r3, 0x8000          /* Way '0' */
0381
0382     clrrwi  r24, r24, 12            /* Clear the valid bit */
0383     tlbwe   r24, r3, 0
0384     tlbwe   r25, r3, 1
0385     tlbwe   r26, r3, 2
0386
0387     /* Make sure we complete the TLB write and flush the shadow TLB */
0388     isync
0389
0390 #endif
0391
0392 ppc44x_map_done:
0393
0394
0395     /* Restore the parameters */
0396     mr  r3, r29
0397     mr  r4, r30
0398     mr  r5, r31
0399
0400     li  r0, 0
0401 #else
0402     li  r0, 0
0403
0404     /*
0405      * Set Machine Status Register to a known status,
0406      * switch the MMU off and jump to 1: in a single step.
0407      */
0408
0409     mr  r8, r0
0410     ori     r8, r8, MSR_RI|MSR_ME
0411     mtspr   SPRN_SRR1, r8
0412     addi    r8, r4, 1f - relocate_new_kernel
0413     mtspr   SPRN_SRR0, r8
0414     sync
0415     rfi
0416
0417 1:
0418 #endif
0419     /* from this point address translation is turned off */
0420     /* and interrupts are disabled */
0421
0422     /* set a new stack at the bottom of our page... */
0423     /* (not really needed now) */
0424     addi    r1, r4, KEXEC_CONTROL_PAGE_SIZE - 8 /* for LR Save+Back Chain */
0425     stw r0, 0(r1)
0426
0427     /* Do the copies */
0428     li  r6, 0 /* checksum */
0429     mr  r0, r3
0430     b   1f
0431
0432 0:  /* top, read another word for the indirection page */
0433     lwzu    r0, 4(r3)
0434
0435 1:
0436     /* is it a destination page? (r8) */
0437     rlwinm. r7, r0, 0, 31, 31 /* IND_DESTINATION (1<<0) */
0438     beq 2f
0439
0440     rlwinm  r8, r0, 0, 0, 19 /* clear kexec flags, page align */
0441     b   0b
0442
0443 2:  /* is it an indirection page? (r3) */
0444     rlwinm. r7, r0, 0, 30, 30 /* IND_INDIRECTION (1<<1) */
0445     beq 2f
0446
0447     rlwinm  r3, r0, 0, 0, 19 /* clear kexec flags, page align */
0448     subi    r3, r3, 4
0449     b   0b
0450
0451 2:  /* are we done? */
0452     rlwinm. r7, r0, 0, 29, 29 /* IND_DONE (1<<2) */
0453     beq 2f
0454     b   3f
0455
0456 2:  /* is it a source page? (r9) */
0457     rlwinm. r7, r0, 0, 28, 28 /* IND_SOURCE (1<<3) */
0458     beq 0b
0459
0460     rlwinm  r9, r0, 0, 0, 19 /* clear kexec flags, page align */
0461
0462     li  r7, PAGE_SIZE / 4
0463     mtctr   r7
0464     subi    r9, r9, 4
0465     subi    r8, r8, 4
0466 9:
0467     lwzu    r0, 4(r9)  /* do the copy */
0468     xor r6, r6, r0
0469     stwu    r0, 4(r8)
0470     dcbst   0, r8
0471     sync
0472     icbi    0, r8
0473     bdnz    9b
0474
0475     addi    r9, r9, 4
0476     addi    r8, r8, 4
0477     b   0b
0478
0479 3:
0480
0481     /* To be certain of avoiding problems with self-modifying code
0482      * execute a serializing instruction here.
0483      */
0484     isync
0485     sync
0486
0487     mfspr   r3, SPRN_PIR /* current core we are running on */
0488     mr  r4, r5 /* load physical address of chunk called */
0489
0490     /* jump to the entry point, usually the setup routine */
0491     mtlr    r5
0492     blrl
0493
0494 1:  b   1b
0495
0496 relocate_new_kernel_end:
0497
0498     .globl relocate_new_kernel_size
0499 relocate_new_kernel_size:
0500     .long relocate_new_kernel_end - relocate_new_kernel