powerpc/kernel/head_40x.S

0001 /* SPDX-License-Identifier: GPL-2.0-or-later */
0002 /*
0003  *    Copyright (c) 1995-1996 Gary Thomas <gdt@linuxppc.org>
0004  *      Initial PowerPC version.
0005  *    Copyright (c) 1996 Cort Dougan <cort@cs.nmt.edu>
0006  *      Rewritten for PReP
0007  *    Copyright (c) 1996 Paul Mackerras <paulus@cs.anu.edu.au>
0008  *      Low-level exception handers, MMU support, and rewrite.
0009  *    Copyright (c) 1997 Dan Malek <dmalek@jlc.net>
0010  *      PowerPC 8xx modifications.
0011  *    Copyright (c) 1998-1999 TiVo, Inc.
0012  *      PowerPC 403GCX modifications.
0013  *    Copyright (c) 1999 Grant Erickson <grant@lcse.umn.edu>
0014  *      PowerPC 403GCX/405GP modifications.
0015  *    Copyright 2000 MontaVista Software Inc.
0016  *  PPC405 modifications
0017  *      PowerPC 403GCX/405GP modifications.
0018  *  Author: MontaVista Software, Inc.
0019  *          frank_rowand@mvista.com or source@mvista.com
0020  *      debbie_chu@mvista.com
0021  *
0022  *    Module name: head_4xx.S
0023  *
0024  *    Description:
0025  *      Kernel execution entry point code.
0026  */
0027
0028 #include <linux/init.h>
0029 #include <linux/pgtable.h>
0030 #include <linux/sizes.h>
0031 #include <asm/processor.h>
0032 #include <asm/page.h>
0033 #include <asm/mmu.h>
0034 #include <asm/cputable.h>
0035 #include <asm/thread_info.h>
0036 #include <asm/ppc_asm.h>
0037 #include <asm/asm-offsets.h>
0038 #include <asm/ptrace.h>
0039 #include <asm/export.h>
0040
0041 #include "head_32.h"
0042
0043 /* As with the other PowerPC ports, it is expected that when code
0044  * execution begins here, the following registers contain valid, yet
0045  * optional, information:
0046  *
0047  *   r3 - Board info structure pointer (DRAM, frequency, MAC address, etc.)
0048  *   r4 - Starting address of the init RAM disk
0049  *   r5 - Ending address of the init RAM disk
0050  *   r6 - Start of kernel command line string (e.g. "mem=96m")
0051  *   r7 - End of kernel command line string
0052  *
0053  * This is all going to change RSN when we add bi_recs.......  -- Dan
0054  */
0055     __HEAD
0056 _GLOBAL(_stext);
0057 _GLOBAL(_start);
0058
0059     mr  r31,r3          /* save device tree ptr */
0060
0061     /* We have to turn on the MMU right away so we get cache modes
0062      * set correctly.
0063      */
0064     bl  initial_mmu
0065
0066 /* We now have the lower 16 Meg mapped into TLB entries, and the caches
0067  * ready to work.
0068  */
0069 turn_on_mmu:
0070     lis r0,MSR_KERNEL@h
0071     ori r0,r0,MSR_KERNEL@l
0072     mtspr   SPRN_SRR1,r0
0073     lis r0,start_here@h
0074     ori r0,r0,start_here@l
0075     mtspr   SPRN_SRR0,r0
0076     rfi             /* enables MMU */
0077     b   .           /* prevent prefetch past rfi */
0078
0079 /*
0080  * This area is used for temporarily saving registers during the
0081  * critical exception prolog.
0082  */
0083     . = 0xc0
0084 crit_save:
0085 _GLOBAL(crit_r10)
0086     .space  4
0087 _GLOBAL(crit_r11)
0088     .space  4
0089 _GLOBAL(crit_srr0)
0090     .space  4
0091 _GLOBAL(crit_srr1)
0092     .space  4
0093 _GLOBAL(crit_r1)
0094     .space  4
0095 _GLOBAL(crit_dear)
0096     .space  4
0097 _GLOBAL(crit_esr)
0098     .space  4
0099
0100 /*
0101  * Exception prolog for critical exceptions.  This is a little different
0102  * from the normal exception prolog above since a critical exception
0103  * can potentially occur at any point during normal exception processing.
0104  * Thus we cannot use the same SPRG registers as the normal prolog above.
0105  * Instead we use a couple of words of memory at low physical addresses.
0106  * This is OK since we don't support SMP on these processors.
0107  */
0108 .macro CRITICAL_EXCEPTION_PROLOG trapno name
0109     stw r10,crit_r10@l(0)   /* save two registers to work with */
0110     stw r11,crit_r11@l(0)
0111     mfspr   r10,SPRN_SRR0
0112     mfspr   r11,SPRN_SRR1
0113     stw r10,crit_srr0@l(0)
0114     stw r11,crit_srr1@l(0)
0115     mfspr   r10,SPRN_DEAR
0116     mfspr   r11,SPRN_ESR
0117     stw r10,crit_dear@l(0)
0118     stw r11,crit_esr@l(0)
0119     mfcr    r10         /* save CR in r10 for now      */
0120     mfspr   r11,SPRN_SRR3       /* check whether user or kernel    */
0121     andi.   r11,r11,MSR_PR
0122     lis r11,(critirq_ctx-PAGE_OFFSET)@ha
0123     lwz r11,(critirq_ctx-PAGE_OFFSET)@l(r11)
0124     beq 1f
0125     /* COMING FROM USER MODE */
0126     mfspr   r11,SPRN_SPRG_THREAD    /* if from user, start at top of   */
0127     lwz r11,TASK_STACK-THREAD(r11) /* this thread's kernel stack */
0128 1:  stw r1,crit_r1@l(0)
0129     addi    r1,r11,THREAD_SIZE-INT_FRAME_SIZE /* Alloc an excpt frm  */
0130     LOAD_REG_IMMEDIATE(r11, MSR_KERNEL & ~(MSR_ME|MSR_DE|MSR_CE)) /* re-enable MMU */
0131     mtspr   SPRN_SRR1, r11
0132     lis r11, 1f@h
0133     ori r11, r11, 1f@l
0134     mtspr   SPRN_SRR0, r11
0135     rfi
0136
0137     .text
0138 1:
0139 \name\()_virt:
0140     lwz r11,crit_r1@l(0)
0141     stw r11,GPR1(r1)
0142     stw r11,0(r1)
0143     mr  r11,r1
0144     stw r10,_CCR(r11)       /* save various registers      */
0145     stw r12,GPR12(r11)
0146     stw r9,GPR9(r11)
0147     mflr    r10
0148     stw r10,_LINK(r11)
0149     lis r9,PAGE_OFFSET@ha
0150     lwz r10,crit_r10@l(r9)
0151     lwz r12,crit_r11@l(r9)
0152     stw r10,GPR10(r11)
0153     stw r12,GPR11(r11)
0154     lwz r12,crit_dear@l(r9)
0155     lwz r9,crit_esr@l(r9)
0156     stw r12,_DEAR(r11)      /* since they may have had stuff   */
0157     stw r9,_ESR(r11)        /* exception was taken         */
0158     mfspr   r12,SPRN_SRR2
0159     mfspr   r9,SPRN_SRR3
0160     rlwinm  r9,r9,0,14,12       /* clear MSR_WE (necessary?)       */
0161     COMMON_EXCEPTION_PROLOG_END \trapno + 2
0162 _ASM_NOKPROBE_SYMBOL(\name\()_virt)
0163 .endm
0164
0165     /*
0166      * State at this point:
0167      * r9 saved in stack frame, now saved SRR3 & ~MSR_WE
0168      * r10 saved in crit_r10 and in stack frame, trashed
0169      * r11 saved in crit_r11 and in stack frame,
0170      *  now phys stack/exception frame pointer
0171      * r12 saved in stack frame, now saved SRR2
0172      * CR saved in stack frame, CR0.EQ = !SRR3.PR
0173      * LR, DEAR, ESR in stack frame
0174      * r1 saved in stack frame, now virt stack/excframe pointer
0175      * r0, r3-r8 saved in stack frame
0176      */
0177
0178 /*
0179  * Exception vectors.
0180  */
0181 #define CRITICAL_EXCEPTION(n, label, hdlr)          \
0182     START_EXCEPTION(n, label);              \
0183     CRITICAL_EXCEPTION_PROLOG n label;              \
0184     prepare_transfer_to_handler;                \
0185     bl  hdlr;                       \
0186     b   ret_from_crit_exc
0187
0188 /*
0189  * 0x0100 - Critical Interrupt Exception
0190  */
0191     CRITICAL_EXCEPTION(0x0100, CriticalInterrupt, unknown_exception)
0192
0193 /*
0194  * 0x0200 - Machine Check Exception
0195  */
0196     CRITICAL_EXCEPTION(0x0200, MachineCheck, machine_check_exception)
0197
0198 /*
0199  * 0x0300 - Data Storage Exception
0200  * This happens for just a few reasons.  U0 set (but we don't do that),
0201  * or zone protection fault (user violation, write to protected page).
0202  * The other Data TLB exceptions bail out to this point
0203  * if they can't resolve the lightweight TLB fault.
0204  */
0205     START_EXCEPTION(0x0300, DataStorage)
0206     EXCEPTION_PROLOG 0x300 DataStorage handle_dar_dsisr=1
0207     prepare_transfer_to_handler
0208     bl  do_page_fault
0209     b   interrupt_return
0210
0211 /*
0212  * 0x0400 - Instruction Storage Exception
0213  * This is caused by a fetch from non-execute or guarded pages.
0214  */
0215     START_EXCEPTION(0x0400, InstructionAccess)
0216     EXCEPTION_PROLOG 0x400 InstructionAccess
0217     li  r5,0
0218     stw r5, _ESR(r11)       /* Zero ESR */
0219     stw r12, _DEAR(r11)     /* SRR0 as DEAR */
0220     prepare_transfer_to_handler
0221     bl  do_page_fault
0222     b   interrupt_return
0223
0224 /* 0x0500 - External Interrupt Exception */
0225     EXCEPTION(0x0500, HardwareInterrupt, do_IRQ)
0226
0227 /* 0x0600 - Alignment Exception */
0228     START_EXCEPTION(0x0600, Alignment)
0229     EXCEPTION_PROLOG 0x600 Alignment handle_dar_dsisr=1
0230     prepare_transfer_to_handler
0231     bl  alignment_exception
0232     REST_NVGPRS(r1)
0233     b   interrupt_return
0234
0235 /* 0x0700 - Program Exception */
0236     START_EXCEPTION(0x0700, ProgramCheck)
0237     EXCEPTION_PROLOG 0x700 ProgramCheck handle_dar_dsisr=1
0238     prepare_transfer_to_handler
0239     bl  program_check_exception
0240     REST_NVGPRS(r1)
0241     b   interrupt_return
0242
0243     EXCEPTION(0x0800, Trap_08, unknown_exception)
0244     EXCEPTION(0x0900, Trap_09, unknown_exception)
0245     EXCEPTION(0x0A00, Trap_0A, unknown_exception)
0246     EXCEPTION(0x0B00, Trap_0B, unknown_exception)
0247
0248 /* 0x0C00 - System Call Exception */
0249     START_EXCEPTION(0x0C00, SystemCall)
0250     SYSCALL_ENTRY   0xc00
0251 /*  Trap_0D is commented out to get more space for system call exception */
0252
0253 /*  EXCEPTION(0x0D00, Trap_0D, unknown_exception) */
0254     EXCEPTION(0x0E00, Trap_0E, unknown_exception)
0255     EXCEPTION(0x0F00, Trap_0F, unknown_exception)
0256
0257 /* 0x1000 - Programmable Interval Timer (PIT) Exception */
0258     START_EXCEPTION(0x1000, DecrementerTrap)
0259     b Decrementer
0260
0261 /* 0x1010 - Fixed Interval Timer (FIT) Exception */
0262     START_EXCEPTION(0x1010, FITExceptionTrap)
0263     b FITException
0264
0265 /* 0x1020 - Watchdog Timer (WDT) Exception */
0266     START_EXCEPTION(0x1020, WDTExceptionTrap)
0267     b WDTException
0268
0269 /* 0x1100 - Data TLB Miss Exception
0270  * As the name implies, translation is not in the MMU, so search the
0271  * page tables and fix it.  The only purpose of this function is to
0272  * load TLB entries from the page table if they exist.
0273  */
0274     START_EXCEPTION(0x1100, DTLBMiss)
0275     mtspr   SPRN_SPRG_SCRATCH5, r10 /* Save some working registers */
0276     mtspr   SPRN_SPRG_SCRATCH6, r11
0277     mtspr   SPRN_SPRG_SCRATCH3, r12
0278     mtspr   SPRN_SPRG_SCRATCH4, r9
0279     mfcr    r12
0280     mfspr   r9, SPRN_PID
0281     rlwimi  r12, r9, 0, 0xff
0282     mfspr   r10, SPRN_DEAR      /* Get faulting address */
0283
0284     /* If we are faulting a kernel address, we have to use the
0285      * kernel page tables.
0286      */
0287     lis r11, PAGE_OFFSET@h
0288     cmplw   r10, r11
0289     blt+    3f
0290     lis r11, swapper_pg_dir@h
0291     ori r11, r11, swapper_pg_dir@l
0292     li  r9, 0
0293     mtspr   SPRN_PID, r9        /* TLB will have 0 TID */
0294     b   4f
0295
0296     /* Get the PGD for the current thread.
0297      */
0298 3:
0299     mfspr   r11,SPRN_SPRG_THREAD
0300     lwz r11,PGDIR(r11)
0301 #ifdef CONFIG_PPC_KUAP
0302     rlwinm. r9, r9, 0, 0xff
0303     beq 5f          /* Kuap fault */
0304 #endif
0305 4:
0306     tophys(r11, r11)
0307     rlwimi  r11, r10, 12, 20, 29    /* Create L1 (pgdir/pmd) address */
0308     lwz r11, 0(r11)     /* Get L1 entry */
0309     andi.   r9, r11, _PMD_PRESENT   /* Check if it points to a PTE page */
0310     beq 2f          /* Bail if no table */
0311
0312     rlwimi  r11, r10, 22, 20, 29    /* Compute PTE address */
0313     lwz r11, 0(r11)     /* Get Linux PTE */
0314     li  r9, _PAGE_PRESENT | _PAGE_ACCESSED
0315     andc.   r9, r9, r11     /* Check permission */
0316     bne 5f
0317
0318     rlwinm  r9, r11, 1, _PAGE_RW    /* dirty => rw */
0319     and r9, r9, r11     /* hwwrite = dirty & rw */
0320     rlwimi  r11, r9, 0, _PAGE_RW    /* replace rw by hwwrite */
0321
0322     /* Create TLB tag.  This is the faulting address plus a static
0323      * set of bits.  These are size, valid, E, U0.
0324     */
0325     li  r9, 0x00c0
0326     rlwimi  r10, r9, 0, 20, 31
0327
0328     b   finish_tlb_load
0329
0330 2:  /* Check for possible large-page pmd entry */
0331     rlwinm. r9, r11, 2, 22, 24
0332     beq 5f
0333
0334     /* Create TLB tag.  This is the faulting address, plus a static
0335      * set of bits (valid, E, U0) plus the size from the PMD.
0336      */
0337     ori r9, r9, 0x40
0338     rlwimi  r10, r9, 0, 20, 31
0339
0340     b   finish_tlb_load
0341
0342 5:
0343     /* The bailout.  Restore registers to pre-exception conditions
0344      * and call the heavyweights to help us out.
0345      */
0346     mtspr   SPRN_PID, r12
0347     mtcrf   0x80, r12
0348     mfspr   r9, SPRN_SPRG_SCRATCH4
0349     mfspr   r12, SPRN_SPRG_SCRATCH3
0350     mfspr   r11, SPRN_SPRG_SCRATCH6
0351     mfspr   r10, SPRN_SPRG_SCRATCH5
0352     b   DataStorage
0353
0354 /* 0x1200 - Instruction TLB Miss Exception
0355  * Nearly the same as above, except we get our information from different
0356  * registers and bailout to a different point.
0357  */
0358     START_EXCEPTION(0x1200, ITLBMiss)
0359     mtspr   SPRN_SPRG_SCRATCH5, r10  /* Save some working registers */
0360     mtspr   SPRN_SPRG_SCRATCH6, r11
0361     mtspr   SPRN_SPRG_SCRATCH3, r12
0362     mtspr   SPRN_SPRG_SCRATCH4, r9
0363     mfcr    r12
0364     mfspr   r9, SPRN_PID
0365     rlwimi  r12, r9, 0, 0xff
0366     mfspr   r10, SPRN_SRR0      /* Get faulting address */
0367
0368     /* If we are faulting a kernel address, we have to use the
0369      * kernel page tables.
0370      */
0371     lis r11, PAGE_OFFSET@h
0372     cmplw   r10, r11
0373     blt+    3f
0374     lis r11, swapper_pg_dir@h
0375     ori r11, r11, swapper_pg_dir@l
0376     li  r9, 0
0377     mtspr   SPRN_PID, r9        /* TLB will have 0 TID */
0378     b   4f
0379
0380     /* Get the PGD for the current thread.
0381      */
0382 3:
0383     mfspr   r11,SPRN_SPRG_THREAD
0384     lwz r11,PGDIR(r11)
0385 #ifdef CONFIG_PPC_KUAP
0386     rlwinm. r9, r9, 0, 0xff
0387     beq 5f          /* Kuap fault */
0388 #endif
0389 4:
0390     tophys(r11, r11)
0391     rlwimi  r11, r10, 12, 20, 29    /* Create L1 (pgdir/pmd) address */
0392     lwz r11, 0(r11)     /* Get L1 entry */
0393     andi.   r9, r11, _PMD_PRESENT   /* Check if it points to a PTE page */
0394     beq 2f          /* Bail if no table */
0395
0396     rlwimi  r11, r10, 22, 20, 29    /* Compute PTE address */
0397     lwz r11, 0(r11)     /* Get Linux PTE */
0398     li  r9, _PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_EXEC
0399     andc.   r9, r9, r11     /* Check permission */
0400     bne 5f
0401
0402     rlwinm  r9, r11, 1, _PAGE_RW    /* dirty => rw */
0403     and r9, r9, r11     /* hwwrite = dirty & rw */
0404     rlwimi  r11, r9, 0, _PAGE_RW    /* replace rw by hwwrite */
0405
0406     /* Create TLB tag.  This is the faulting address plus a static
0407      * set of bits.  These are size, valid, E, U0.
0408     */
0409     li  r9, 0x00c0
0410     rlwimi  r10, r9, 0, 20, 31
0411
0412     b   finish_tlb_load
0413
0414 2:  /* Check for possible large-page pmd entry */
0415     rlwinm. r9, r11, 2, 22, 24
0416     beq 5f
0417
0418     /* Create TLB tag.  This is the faulting address, plus a static
0419      * set of bits (valid, E, U0) plus the size from the PMD.
0420      */
0421     ori r9, r9, 0x40
0422     rlwimi  r10, r9, 0, 20, 31
0423
0424     b   finish_tlb_load
0425
0426 5:
0427     /* The bailout.  Restore registers to pre-exception conditions
0428      * and call the heavyweights to help us out.
0429      */
0430     mtspr   SPRN_PID, r12
0431     mtcrf   0x80, r12
0432     mfspr   r9, SPRN_SPRG_SCRATCH4
0433     mfspr   r12, SPRN_SPRG_SCRATCH3
0434     mfspr   r11, SPRN_SPRG_SCRATCH6
0435     mfspr   r10, SPRN_SPRG_SCRATCH5
0436     b   InstructionAccess
0437
0438     EXCEPTION(0x1300, Trap_13, unknown_exception)
0439     EXCEPTION(0x1400, Trap_14, unknown_exception)
0440     EXCEPTION(0x1500, Trap_15, unknown_exception)
0441     EXCEPTION(0x1600, Trap_16, unknown_exception)
0442     EXCEPTION(0x1700, Trap_17, unknown_exception)
0443     EXCEPTION(0x1800, Trap_18, unknown_exception)
0444     EXCEPTION(0x1900, Trap_19, unknown_exception)
0445     EXCEPTION(0x1A00, Trap_1A, unknown_exception)
0446     EXCEPTION(0x1B00, Trap_1B, unknown_exception)
0447     EXCEPTION(0x1C00, Trap_1C, unknown_exception)
0448     EXCEPTION(0x1D00, Trap_1D, unknown_exception)
0449     EXCEPTION(0x1E00, Trap_1E, unknown_exception)
0450     EXCEPTION(0x1F00, Trap_1F, unknown_exception)
0451
0452 /* Check for a single step debug exception while in an exception
0453  * handler before state has been saved.  This is to catch the case
0454  * where an instruction that we are trying to single step causes
0455  * an exception (eg ITLB/DTLB miss) and thus the first instruction of
0456  * the exception handler generates a single step debug exception.
0457  *
0458  * If we get a debug trap on the first instruction of an exception handler,
0459  * we reset the MSR_DE in the _exception handler's_ MSR (the debug trap is
0460  * a critical exception, so we are using SPRN_CSRR1 to manipulate the MSR).
0461  * The exception handler was handling a non-critical interrupt, so it will
0462  * save (and later restore) the MSR via SPRN_SRR1, which will still have
0463  * the MSR_DE bit set.
0464  */
0465     /* 0x2000 - Debug Exception */
0466     START_EXCEPTION(0x2000, DebugTrap)
0467     CRITICAL_EXCEPTION_PROLOG 0x2000 DebugTrap
0468
0469     /*
0470      * If this is a single step or branch-taken exception in an
0471      * exception entry sequence, it was probably meant to apply to
0472      * the code where the exception occurred (since exception entry
0473      * doesn't turn off DE automatically).  We simulate the effect
0474      * of turning off DE on entry to an exception handler by turning
0475      * off DE in the SRR3 value and clearing the debug status.
0476      */
0477     mfspr   r10,SPRN_DBSR       /* check single-step/branch taken */
0478     andis.  r10,r10,DBSR_IC@h
0479     beq+    2f
0480
0481     andi.   r10,r9,MSR_IR|MSR_PR    /* check supervisor + MMU off */
0482     beq 1f          /* branch and fix it up */
0483
0484     mfspr   r10,SPRN_SRR2       /* Faulting instruction address */
0485     cmplwi  r10,0x2100
0486     bgt+    2f          /* address above exception vectors */
0487
0488     /* here it looks like we got an inappropriate debug exception. */
0489 1:  rlwinm  r9,r9,0,~MSR_DE     /* clear DE in the SRR3 value */
0490     lis r10,DBSR_IC@h       /* clear the IC event */
0491     mtspr   SPRN_DBSR,r10
0492     /* restore state and get out */
0493     lwz r10,_CCR(r11)
0494     lwz r0,GPR0(r11)
0495     lwz r1,GPR1(r11)
0496     mtcrf   0x80,r10
0497     mtspr   SPRN_SRR2,r12
0498     mtspr   SPRN_SRR3,r9
0499     lwz r9,GPR9(r11)
0500     lwz r12,GPR12(r11)
0501     lwz r10,crit_r10@l(0)
0502     lwz r11,crit_r11@l(0)
0503     rfci
0504     b   .
0505
0506     /* continue normal handling for a critical exception... */
0507 2:  mfspr   r4,SPRN_DBSR
0508     stw r4,_ESR(r11)        /* DebugException takes DBSR in _ESR */
0509     prepare_transfer_to_handler
0510     bl  DebugException
0511     b   ret_from_crit_exc
0512
0513     /* Programmable Interval Timer (PIT) Exception. (from 0x1000) */
0514     __HEAD
0515 Decrementer:
0516     EXCEPTION_PROLOG 0x1000 Decrementer
0517     lis r0,TSR_PIS@h
0518     mtspr   SPRN_TSR,r0     /* Clear the PIT exception */
0519     prepare_transfer_to_handler
0520     bl  timer_interrupt
0521     b   interrupt_return
0522
0523     /* Fixed Interval Timer (FIT) Exception. (from 0x1010) */
0524     __HEAD
0525 FITException:
0526     EXCEPTION_PROLOG 0x1010 FITException
0527     prepare_transfer_to_handler
0528     bl  unknown_exception
0529     b   interrupt_return
0530
0531     /* Watchdog Timer (WDT) Exception. (from 0x1020) */
0532     __HEAD
0533 WDTException:
0534     CRITICAL_EXCEPTION_PROLOG 0x1020 WDTException
0535     prepare_transfer_to_handler
0536     bl  WatchdogException
0537     b   ret_from_crit_exc
0538
0539 /* Other PowerPC processors, namely those derived from the 6xx-series
0540  * have vectors from 0x2100 through 0x2F00 defined, but marked as reserved.
0541  * However, for the 4xx-series processors these are neither defined nor
0542  * reserved.
0543  */
0544
0545     __HEAD
0546     /* Damn, I came up one instruction too many to fit into the
0547      * exception space :-).  Both the instruction and data TLB
0548      * miss get to this point to load the TLB.
0549      *  r10 - TLB_TAG value
0550      *  r11 - Linux PTE
0551      *  r9 - available to use
0552      *  PID - loaded with proper value when we get here
0553      *  Upon exit, we reload everything and RFI.
0554      * Actually, it will fit now, but oh well.....a common place
0555      * to load the TLB.
0556      */
0557 tlb_4xx_index:
0558     .long   0
0559 finish_tlb_load:
0560     /*
0561      * Clear out the software-only bits in the PTE to generate the
0562      * TLB_DATA value.  These are the bottom 2 bits of the RPM, the
0563      * top 3 bits of the zone field, and M.
0564      */
0565     li  r9, 0x0ce2
0566     andc    r11, r11, r9
0567
0568     /* load the next available TLB index. */
0569     lwz r9, tlb_4xx_index@l(0)
0570     addi    r9, r9, 1
0571     andi.   r9, r9, PPC40X_TLB_SIZE - 1
0572     stw r9, tlb_4xx_index@l(0)
0573
0574     tlbwe   r11, r9, TLB_DATA       /* Load TLB LO */
0575     tlbwe   r10, r9, TLB_TAG        /* Load TLB HI */
0576
0577     /* Done...restore registers and get out of here.
0578     */
0579     mtspr   SPRN_PID, r12
0580     mtcrf   0x80, r12
0581     mfspr   r9, SPRN_SPRG_SCRATCH4
0582     mfspr   r12, SPRN_SPRG_SCRATCH3
0583     mfspr   r11, SPRN_SPRG_SCRATCH6
0584     mfspr   r10, SPRN_SPRG_SCRATCH5
0585     rfi         /* Should sync shadow TLBs */
0586     b   .       /* prevent prefetch past rfi */
0587
0588 /* This is where the main kernel code starts.
0589  */
0590 start_here:
0591
0592     /* ptr to current */
0593     lis r2,init_task@h
0594     ori r2,r2,init_task@l
0595
0596     /* ptr to phys current thread */
0597     tophys(r4,r2)
0598     addi    r4,r4,THREAD    /* init task's THREAD */
0599     mtspr   SPRN_SPRG_THREAD,r4
0600
0601     /* stack */
0602     lis r1,init_thread_union@ha
0603     addi    r1,r1,init_thread_union@l
0604     li  r0,0
0605     stwu    r0,THREAD_SIZE-STACK_FRAME_OVERHEAD(r1)
0606
0607     bl  early_init  /* We have to do this with MMU on */
0608
0609 /*
0610  * Decide what sort of machine this is and initialize the MMU.
0611  */
0612 #ifdef CONFIG_KASAN
0613     bl  kasan_early_init
0614 #endif
0615     li  r3,0
0616     mr  r4,r31
0617     bl  machine_init
0618     bl  MMU_init
0619
0620 /* Go back to running unmapped so we can load up new values
0621  * and change to using our exception vectors.
0622  * On the 4xx, all we have to do is invalidate the TLB to clear
0623  * the old 16M byte TLB mappings.
0624  */
0625     lis r4,2f@h
0626     ori r4,r4,2f@l
0627     tophys(r4,r4)
0628     lis r3,(MSR_KERNEL & ~(MSR_IR|MSR_DR))@h
0629     ori r3,r3,(MSR_KERNEL & ~(MSR_IR|MSR_DR))@l
0630     mtspr   SPRN_SRR0,r4
0631     mtspr   SPRN_SRR1,r3
0632     rfi
0633     b   .       /* prevent prefetch past rfi */
0634
0635 /* Load up the kernel context */
0636 2:
0637     sync            /* Flush to memory before changing TLB */
0638     tlbia
0639     isync           /* Flush shadow TLBs */
0640
0641     /* set up the PTE pointers for the Abatron bdiGDB.
0642     */
0643     lis r6, swapper_pg_dir@h
0644     ori r6, r6, swapper_pg_dir@l
0645     lis r5, abatron_pteptrs@h
0646     ori r5, r5, abatron_pteptrs@l
0647     stw r5, 0xf0(0) /* Must match your Abatron config file */
0648     tophys(r5,r5)
0649     stw r6, 0(r5)
0650
0651 /* Now turn on the MMU for real! */
0652     lis r4,MSR_KERNEL@h
0653     ori r4,r4,MSR_KERNEL@l
0654     lis r3,start_kernel@h
0655     ori r3,r3,start_kernel@l
0656     mtspr   SPRN_SRR0,r3
0657     mtspr   SPRN_SRR1,r4
0658     rfi         /* enable MMU and jump to start_kernel */
0659     b   .       /* prevent prefetch past rfi */
0660
0661 /* Set up the initial MMU state so we can do the first level of
0662  * kernel initialization.  This maps the first 32 MBytes of memory 1:1
0663  * virtual to physical and more importantly sets the cache mode.
0664  */
0665 initial_mmu:
0666     tlbia           /* Invalidate all TLB entries */
0667     isync
0668
0669     /* We should still be executing code at physical address 0x0000xxxx
0670      * at this point. However, start_here is at virtual address
0671      * 0xC000xxxx. So, set up a TLB mapping to cover this once
0672      * translation is enabled.
0673      */
0674
0675     lis r3,KERNELBASE@h     /* Load the kernel virtual address */
0676     ori r3,r3,KERNELBASE@l
0677     tophys(r4,r3)           /* Load the kernel physical address */
0678
0679     iccci   r0,r3           /* Invalidate the i-cache before use */
0680
0681     /* Load the kernel PID.
0682     */
0683     li  r0,0
0684     mtspr   SPRN_PID,r0
0685     sync
0686
0687     /* Configure and load one entry into TLB slots 63 */
0688     clrrwi  r4,r4,10        /* Mask off the real page number */
0689     ori r4,r4,(TLB_WR | TLB_EX) /* Set the write and execute bits */
0690
0691     clrrwi  r3,r3,10        /* Mask off the effective page number */
0692     ori r3,r3,(TLB_VALID | TLB_PAGESZ(PAGESZ_16M))
0693
0694         li      r0,63                    /* TLB slot 63 */
0695
0696     tlbwe   r4,r0,TLB_DATA      /* Load the data portion of the entry */
0697     tlbwe   r3,r0,TLB_TAG       /* Load the tag portion of the entry */
0698
0699     li  r0,62           /* TLB slot 62 */
0700     addis   r4,r4,SZ_16M@h
0701     addis   r3,r3,SZ_16M@h
0702     tlbwe   r4,r0,TLB_DATA      /* Load the data portion of the entry */
0703     tlbwe   r3,r0,TLB_TAG       /* Load the tag portion of the entry */
0704
0705     isync
0706
0707     /* Establish the exception vector base
0708     */
0709     lis r4,KERNELBASE@h     /* EVPR only uses the high 16-bits */
0710     tophys(r0,r4)           /* Use the physical address */
0711     mtspr   SPRN_EVPR,r0
0712
0713     blr
0714
0715 _GLOBAL(abort)
0716         mfspr   r13,SPRN_DBCR0
0717         oris    r13,r13,DBCR0_RST_SYSTEM@h
0718         mtspr   SPRN_DBCR0,r13