Back to home page

OSCL-LXR
 
 

     

0001 /* SPDX-License-Identifier: GPL-2.0-or-later */
0002 /*
0003  *  Low level TLB miss handlers for Book3E
0004  *
0005  *  Copyright (C) 2008-2009
0006  *      Ben. Herrenschmidt (benh@kernel.crashing.org), IBM Corp.
0007  */
0008 
0009 #include <linux/pgtable.h>
0010 #include <asm/processor.h>
0011 #include <asm/reg.h>
0012 #include <asm/page.h>
0013 #include <asm/mmu.h>
0014 #include <asm/ppc_asm.h>
0015 #include <asm/asm-offsets.h>
0016 #include <asm/cputable.h>
0017 #include <asm/exception-64e.h>
0018 #include <asm/ppc-opcode.h>
0019 #include <asm/kvm_asm.h>
0020 #include <asm/kvm_booke_hv_asm.h>
0021 #include <asm/feature-fixups.h>
0022 
0023 #define VPTE_PMD_SHIFT  (PTE_INDEX_SIZE)
0024 #define VPTE_PUD_SHIFT  (VPTE_PMD_SHIFT + PMD_INDEX_SIZE)
0025 #define VPTE_PGD_SHIFT  (VPTE_PUD_SHIFT + PUD_INDEX_SIZE)
0026 #define VPTE_INDEX_SIZE (VPTE_PGD_SHIFT + PGD_INDEX_SIZE)
0027 
0028 /**********************************************************************
0029  *                                                                    *
0030  * TLB miss handling for Book3E with a bolted linear mapping          *
0031  * No virtual page table, no nested TLB misses                        *
0032  *                                                                    *
0033  **********************************************************************/
0034 
0035 /*
0036  * Note that, unlike non-bolted handlers, TLB_EXFRAME is not
0037  * modified by the TLB miss handlers themselves, since the TLB miss
0038  * handler code will not itself cause a recursive TLB miss.
0039  *
0040  * TLB_EXFRAME will be modified when crit/mc/debug exceptions are
0041  * entered/exited.
0042  */
0043 .macro tlb_prolog_bolted intnum addr
0044     mtspr   SPRN_SPRG_GEN_SCRATCH,r12
0045     mfspr   r12,SPRN_SPRG_TLB_EXFRAME
0046     std r13,EX_TLB_R13(r12)
0047     std r10,EX_TLB_R10(r12)
0048     mfspr   r13,SPRN_SPRG_PACA
0049 
0050     mfcr    r10
0051     std r11,EX_TLB_R11(r12)
0052 #ifdef CONFIG_KVM_BOOKE_HV
0053 BEGIN_FTR_SECTION
0054     mfspr   r11, SPRN_SRR1
0055 END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV)
0056 #endif
0057     DO_KVM  \intnum, SPRN_SRR1
0058     std r16,EX_TLB_R16(r12)
0059     mfspr   r16,\addr       /* get faulting address */
0060     std r14,EX_TLB_R14(r12)
0061     ld  r14,PACAPGD(r13)
0062     std r15,EX_TLB_R15(r12)
0063     std r10,EX_TLB_CR(r12)
0064 #ifdef CONFIG_PPC_FSL_BOOK3E
0065 START_BTB_FLUSH_SECTION
0066     mfspr r11, SPRN_SRR1
0067     andi. r10,r11,MSR_PR
0068     beq 1f
0069     BTB_FLUSH(r10)
0070 1:
0071 END_BTB_FLUSH_SECTION
0072     std r7,EX_TLB_R7(r12)
0073 #endif
0074 .endm
0075 
0076 .macro tlb_epilog_bolted
0077     ld  r14,EX_TLB_CR(r12)
0078 #ifdef CONFIG_PPC_FSL_BOOK3E
0079     ld  r7,EX_TLB_R7(r12)
0080 #endif
0081     ld  r10,EX_TLB_R10(r12)
0082     ld  r11,EX_TLB_R11(r12)
0083     ld  r13,EX_TLB_R13(r12)
0084     mtcr    r14
0085     ld  r14,EX_TLB_R14(r12)
0086     ld  r15,EX_TLB_R15(r12)
0087     ld  r16,EX_TLB_R16(r12)
0088     mfspr   r12,SPRN_SPRG_GEN_SCRATCH
0089 .endm
0090 
0091 /* Data TLB miss */
0092     START_EXCEPTION(data_tlb_miss_bolted)
0093     tlb_prolog_bolted BOOKE_INTERRUPT_DTLB_MISS SPRN_DEAR
0094 
0095     /* We need _PAGE_PRESENT and  _PAGE_ACCESSED set */
0096 
0097     /* We do the user/kernel test for the PID here along with the RW test
0098      */
0099     /* We pre-test some combination of permissions to avoid double
0100      * faults:
0101      *
0102      * We move the ESR:ST bit into the position of _PAGE_BAP_SW in the PTE
0103      * ESR_ST   is 0x00800000
0104      * _PAGE_BAP_SW is 0x00000010
0105      * So the shift is >> 19. This tests for supervisor writeability.
0106      * If the page happens to be supervisor writeable and not user
0107      * writeable, we will take a new fault later, but that should be
0108      * a rare enough case.
0109      *
0110      * We also move ESR_ST in _PAGE_DIRTY position
0111      * _PAGE_DIRTY is 0x00001000 so the shift is >> 11
0112      *
0113      * MAS1 is preset for all we need except for TID that needs to
0114      * be cleared for kernel translations
0115      */
0116 
0117     mfspr   r11,SPRN_ESR
0118 
0119     srdi    r15,r16,60      /* get region */
0120     rldicl. r10,r16,64-PGTABLE_EADDR_SIZE,PGTABLE_EADDR_SIZE+4
0121     bne-    dtlb_miss_fault_bolted  /* Bail if fault addr is invalid */
0122 
0123     rlwinm  r10,r11,32-19,27,27
0124     rlwimi  r10,r11,32-16,19,19
0125     cmpwi   r15,0           /* user vs kernel check */
0126     ori r10,r10,_PAGE_PRESENT
0127     oris    r11,r10,_PAGE_ACCESSED@h
0128 
0129     bne tlb_miss_kernel_bolted
0130 
0131 tlb_miss_user_bolted:
0132 #ifdef CONFIG_PPC_KUAP
0133     mfspr   r10,SPRN_MAS1
0134     rlwinm. r10,r10,0,0x3fff0000
0135     beq-    tlb_miss_fault_bolted /* KUAP fault */
0136 #endif
0137 
0138 tlb_miss_common_bolted:
0139 /*
0140  * This is the guts of the TLB miss handler for bolted-linear.
0141  * We are entered with:
0142  *
0143  * r16 = faulting address
0144  * r15 = crap (free to use)
0145  * r14 = page table base
0146  * r13 = PACA
0147  * r11 = PTE permission mask
0148  * r10 = crap (free to use)
0149  */
0150     rldicl  r15,r16,64-PGDIR_SHIFT+3,64-PGD_INDEX_SIZE-3
0151     cmpldi  cr0,r14,0
0152     clrrdi  r15,r15,3
0153     beq tlb_miss_fault_bolted   /* No PGDIR, bail */
0154 
0155     ldx r14,r14,r15     /* grab pgd entry */
0156 
0157     rldicl  r15,r16,64-PUD_SHIFT+3,64-PUD_INDEX_SIZE-3
0158     clrrdi  r15,r15,3
0159     cmpdi   cr0,r14,0
0160     bge tlb_miss_fault_bolted   /* Bad pgd entry or hugepage; bail */
0161     ldx r14,r14,r15     /* grab pud entry */
0162 
0163     rldicl  r15,r16,64-PMD_SHIFT+3,64-PMD_INDEX_SIZE-3
0164     clrrdi  r15,r15,3
0165     cmpdi   cr0,r14,0
0166     bge tlb_miss_fault_bolted
0167     ldx r14,r14,r15     /* Grab pmd entry */
0168 
0169     rldicl  r15,r16,64-PAGE_SHIFT+3,64-PTE_INDEX_SIZE-3
0170     clrrdi  r15,r15,3
0171     cmpdi   cr0,r14,0
0172     bge tlb_miss_fault_bolted
0173     ldx r14,r14,r15     /* Grab PTE, normal (!huge) page */
0174 
0175     /* Check if required permissions are met */
0176     andc.   r15,r11,r14
0177     rldicr  r15,r14,64-(PTE_RPN_SHIFT-PAGE_SHIFT),63-PAGE_SHIFT
0178     bne-    tlb_miss_fault_bolted
0179 
0180     /* Now we build the MAS:
0181      *
0182      * MAS 0   :    Fully setup with defaults in MAS4 and TLBnCFG
0183      * MAS 1   :    Almost fully setup
0184      *               - PID already updated by caller if necessary
0185      *               - TSIZE need change if !base page size, not
0186      *                 yet implemented for now
0187      * MAS 2   :    Defaults not useful, need to be redone
0188      * MAS 3+7 :    Needs to be done
0189      */
0190     clrrdi  r11,r16,12      /* Clear low crap in EA */
0191     clrldi  r15,r15,12      /* Clear crap at the top */
0192     rlwimi  r11,r14,32-19,27,31 /* Insert WIMGE */
0193     rlwimi  r15,r14,32-8,22,25  /* Move in U bits */
0194     mtspr   SPRN_MAS2,r11
0195     andi.   r11,r14,_PAGE_DIRTY
0196     rlwimi  r15,r14,32-2,26,31  /* Move in BAP bits */
0197 
0198     /* Mask out SW and UW if !DIRTY (XXX optimize this !) */
0199     bne 1f
0200     li  r11,MAS3_SW|MAS3_UW
0201     andc    r15,r15,r11
0202 1:
0203     mtspr   SPRN_MAS7_MAS3,r15
0204     tlbwe
0205 
0206 tlb_miss_done_bolted:
0207     tlb_epilog_bolted
0208     rfi
0209 
0210 itlb_miss_kernel_bolted:
0211     li  r11,_PAGE_PRESENT|_PAGE_BAP_SX  /* Base perm */
0212     oris    r11,r11,_PAGE_ACCESSED@h
0213 tlb_miss_kernel_bolted:
0214     mfspr   r10,SPRN_MAS1
0215     ld  r14,PACA_KERNELPGD(r13)
0216     srdi    r15,r16,44      /* get kernel region */
0217     andi.   r15,r15,1       /* Check for vmalloc region */
0218     rlwinm  r10,r10,0,16,1      /* Clear TID */
0219     mtspr   SPRN_MAS1,r10
0220     bne+    tlb_miss_common_bolted
0221 
0222 tlb_miss_fault_bolted:
0223     /* We need to check if it was an instruction miss */
0224     andi.   r10,r11,_PAGE_BAP_UX|_PAGE_BAP_SX
0225     bne itlb_miss_fault_bolted
0226 dtlb_miss_fault_bolted:
0227     tlb_epilog_bolted
0228     b   exc_data_storage_book3e
0229 itlb_miss_fault_bolted:
0230     tlb_epilog_bolted
0231     b   exc_instruction_storage_book3e
0232 
0233 /* Instruction TLB miss */
0234     START_EXCEPTION(instruction_tlb_miss_bolted)
0235     tlb_prolog_bolted BOOKE_INTERRUPT_ITLB_MISS SPRN_SRR0
0236 
0237     rldicl. r10,r16,64-PGTABLE_EADDR_SIZE,PGTABLE_EADDR_SIZE+4
0238     srdi    r15,r16,60      /* get region */
0239     bne-    itlb_miss_fault_bolted
0240 
0241     li  r11,_PAGE_PRESENT|_PAGE_BAP_UX  /* Base perm */
0242 
0243     /* We do the user/kernel test for the PID here along with the RW test
0244      */
0245 
0246     cmpldi  cr0,r15,0           /* Check for user region */
0247     oris    r11,r11,_PAGE_ACCESSED@h
0248     beq tlb_miss_user_bolted
0249     b   itlb_miss_kernel_bolted
0250 
0251 #ifdef CONFIG_PPC_FSL_BOOK3E
0252 /*
0253  * TLB miss handling for e6500 and derivatives, using hardware tablewalk.
0254  *
0255  * Linear mapping is bolted: no virtual page table or nested TLB misses
0256  * Indirect entries in TLB1, hardware loads resulting direct entries
0257  *    into TLB0
0258  * No HES or NV hint on TLB1, so we need to do software round-robin
0259  * No tlbsrx. so we need a spinlock, and we have to deal
0260  *    with MAS-damage caused by tlbsx
0261  * 4K pages only
0262  */
0263 
0264     START_EXCEPTION(instruction_tlb_miss_e6500)
0265     tlb_prolog_bolted BOOKE_INTERRUPT_ITLB_MISS SPRN_SRR0
0266 
0267     ld  r11,PACA_TCD_PTR(r13)
0268     srdi.   r15,r16,60      /* get region */
0269     ori r16,r16,1
0270 
0271     bne tlb_miss_kernel_e6500   /* user/kernel test */
0272 
0273     b   tlb_miss_common_e6500
0274 
0275     START_EXCEPTION(data_tlb_miss_e6500)
0276     tlb_prolog_bolted BOOKE_INTERRUPT_DTLB_MISS SPRN_DEAR
0277 
0278     ld  r11,PACA_TCD_PTR(r13)
0279     srdi.   r15,r16,60      /* get region */
0280     rldicr  r16,r16,0,62
0281 
0282     bne tlb_miss_kernel_e6500   /* user vs kernel check */
0283 
0284 /*
0285  * This is the guts of the TLB miss handler for e6500 and derivatives.
0286  * We are entered with:
0287  *
0288  * r16 = page of faulting address (low bit 0 if data, 1 if instruction)
0289  * r15 = crap (free to use)
0290  * r14 = page table base
0291  * r13 = PACA
0292  * r11 = tlb_per_core ptr
0293  * r10 = crap (free to use)
0294  * r7  = esel_next
0295  */
0296 tlb_miss_common_e6500:
0297     crmove  cr2*4+2,cr0*4+2     /* cr2.eq != 0 if kernel address */
0298 
0299 BEGIN_FTR_SECTION       /* CPU_FTR_SMT */
0300     /*
0301      * Search if we already have an indirect entry for that virtual
0302      * address, and if we do, bail out.
0303      *
0304      * MAS6:IND should be already set based on MAS4
0305      */
0306     lhz r10,PACAPACAINDEX(r13)
0307     addi    r10,r10,1
0308     crclr   cr1*4+eq    /* set cr1.eq = 0 for non-recursive */
0309 1:  lbarx   r15,0,r11
0310     cmpdi   r15,0
0311     bne 2f
0312     stbcx.  r10,0,r11
0313     bne 1b
0314 3:
0315     .subsection 1
0316 2:  cmpd    cr1,r15,r10 /* recursive lock due to mcheck/crit/etc? */
0317     beq cr1,3b      /* unlock will happen if cr1.eq = 0 */
0318 10: lbz r15,0(r11)
0319     cmpdi   r15,0
0320     bne 10b
0321     b   1b
0322     .previous
0323 END_FTR_SECTION_IFSET(CPU_FTR_SMT)
0324 
0325     lbz r7,TCD_ESEL_NEXT(r11)
0326 
0327 BEGIN_FTR_SECTION       /* CPU_FTR_SMT */
0328     /*
0329      * Erratum A-008139 says that we can't use tlbwe to change
0330      * an indirect entry in any way (including replacing or
0331      * invalidating) if the other thread could be in the process
0332      * of a lookup.  The workaround is to invalidate the entry
0333      * with tlbilx before overwriting.
0334      */
0335 
0336     rlwinm  r10,r7,16,0xff0000
0337     oris    r10,r10,MAS0_TLBSEL(1)@h
0338     mtspr   SPRN_MAS0,r10
0339     isync
0340     tlbre
0341     mfspr   r15,SPRN_MAS1
0342     andis.  r15,r15,MAS1_VALID@h
0343     beq 5f
0344 
0345 BEGIN_FTR_SECTION_NESTED(532)
0346     mfspr   r10,SPRN_MAS8
0347     rlwinm  r10,r10,0,0x80000fff  /* tgs,tlpid -> sgs,slpid */
0348     mtspr   SPRN_MAS5,r10
0349 END_FTR_SECTION_NESTED(CPU_FTR_EMB_HV,CPU_FTR_EMB_HV,532)
0350 
0351     mfspr   r10,SPRN_MAS1
0352     rlwinm  r15,r10,0,0x3fff0000  /* tid -> spid */
0353     rlwimi  r15,r10,20,0x00000003 /* ind,ts -> sind,sas */
0354     mfspr   r10,SPRN_MAS6
0355     mtspr   SPRN_MAS6,r15
0356 
0357     mfspr   r15,SPRN_MAS2
0358     isync
0359     tlbilxva 0,r15
0360     isync
0361 
0362     mtspr   SPRN_MAS6,r10
0363 
0364 5:
0365 BEGIN_FTR_SECTION_NESTED(532)
0366     li  r10,0
0367     mtspr   SPRN_MAS8,r10
0368     mtspr   SPRN_MAS5,r10
0369 END_FTR_SECTION_NESTED(CPU_FTR_EMB_HV,CPU_FTR_EMB_HV,532)
0370 
0371     tlbsx   0,r16
0372     mfspr   r10,SPRN_MAS1
0373     andis.  r15,r10,MAS1_VALID@h
0374     bne tlb_miss_done_e6500
0375 FTR_SECTION_ELSE
0376     mfspr   r10,SPRN_MAS1
0377 ALT_FTR_SECTION_END_IFSET(CPU_FTR_SMT)
0378 
0379     oris    r10,r10,MAS1_VALID@h
0380     beq cr2,4f
0381     rlwinm  r10,r10,0,16,1      /* Clear TID */
0382 4:  mtspr   SPRN_MAS1,r10
0383 
0384     /* Now, we need to walk the page tables. First check if we are in
0385      * range.
0386      */
0387     rldicl. r10,r16,64-PGTABLE_EADDR_SIZE,PGTABLE_EADDR_SIZE+4
0388     bne-    tlb_miss_fault_e6500
0389 
0390     rldicl  r15,r16,64-PGDIR_SHIFT+3,64-PGD_INDEX_SIZE-3
0391     cmpldi  cr0,r14,0
0392     clrrdi  r15,r15,3
0393     beq-    tlb_miss_fault_e6500 /* No PGDIR, bail */
0394     ldx r14,r14,r15     /* grab pgd entry */
0395 
0396     rldicl  r15,r16,64-PUD_SHIFT+3,64-PUD_INDEX_SIZE-3
0397     clrrdi  r15,r15,3
0398     cmpdi   cr0,r14,0
0399     bge tlb_miss_huge_e6500 /* Bad pgd entry or hugepage; bail */
0400     ldx r14,r14,r15     /* grab pud entry */
0401 
0402     rldicl  r15,r16,64-PMD_SHIFT+3,64-PMD_INDEX_SIZE-3
0403     clrrdi  r15,r15,3
0404     cmpdi   cr0,r14,0
0405     bge tlb_miss_huge_e6500
0406     ldx r14,r14,r15     /* Grab pmd entry */
0407 
0408     mfspr   r10,SPRN_MAS0
0409     cmpdi   cr0,r14,0
0410     bge tlb_miss_huge_e6500
0411 
0412     /* Now we build the MAS for a 2M indirect page:
0413      *
0414      * MAS 0   :    ESEL needs to be filled by software round-robin
0415      * MAS 1   :    Fully set up
0416      *               - PID already updated by caller if necessary
0417      *               - TSIZE for now is base ind page size always
0418      *               - TID already cleared if necessary
0419      * MAS 2   :    Default not 2M-aligned, need to be redone
0420      * MAS 3+7 :    Needs to be done
0421      */
0422 
0423     ori r14,r14,(BOOK3E_PAGESZ_4K << MAS3_SPSIZE_SHIFT)
0424     mtspr   SPRN_MAS7_MAS3,r14
0425 
0426     clrrdi  r15,r16,21      /* make EA 2M-aligned */
0427     mtspr   SPRN_MAS2,r15
0428 
0429 tlb_miss_huge_done_e6500:
0430     lbz r16,TCD_ESEL_MAX(r11)
0431     lbz r14,TCD_ESEL_FIRST(r11)
0432     rlwimi  r10,r7,16,0x00ff0000    /* insert esel_next into MAS0 */
0433     addi    r7,r7,1         /* increment esel_next */
0434     mtspr   SPRN_MAS0,r10
0435     cmpw    r7,r16
0436     iseleq  r7,r14,r7       /* if next == last use first */
0437     stb r7,TCD_ESEL_NEXT(r11)
0438 
0439     tlbwe
0440 
0441 tlb_miss_done_e6500:
0442     .macro  tlb_unlock_e6500
0443 BEGIN_FTR_SECTION
0444     beq cr1,1f      /* no unlock if lock was recursively grabbed */
0445     li  r15,0
0446     isync
0447     stb r15,0(r11)
0448 1:
0449 END_FTR_SECTION_IFSET(CPU_FTR_SMT)
0450     .endm
0451 
0452     tlb_unlock_e6500
0453     tlb_epilog_bolted
0454     rfi
0455 
0456 tlb_miss_huge_e6500:
0457     beq tlb_miss_fault_e6500
0458     li  r10,1
0459     andi.   r15,r14,HUGEPD_SHIFT_MASK@l /* r15 = psize */
0460     rldimi  r14,r10,63,0        /* Set PD_HUGE */
0461     xor r14,r14,r15     /* Clear size bits */
0462     ldx r14,0,r14
0463 
0464     /*
0465      * Now we build the MAS for a huge page.
0466      *
0467      * MAS 0   :    ESEL needs to be filled by software round-robin
0468      *       - can be handled by indirect code
0469      * MAS 1   :    Need to clear IND and set TSIZE
0470      * MAS 2,3+7:   Needs to be redone similar to non-tablewalk handler
0471      */
0472 
0473     subi    r15,r15,10      /* Convert psize to tsize */
0474     mfspr   r10,SPRN_MAS1
0475     rlwinm  r10,r10,0,~MAS1_IND
0476     rlwimi  r10,r15,MAS1_TSIZE_SHIFT,MAS1_TSIZE_MASK
0477     mtspr   SPRN_MAS1,r10
0478 
0479     li  r10,-0x400
0480     sld r15,r10,r15     /* Generate mask based on size */
0481     and r10,r16,r15
0482     rldicr  r15,r14,64-(PTE_RPN_SHIFT-PAGE_SHIFT),63-PAGE_SHIFT
0483     rlwimi  r10,r14,32-19,27,31 /* Insert WIMGE */
0484     clrldi  r15,r15,PAGE_SHIFT  /* Clear crap at the top */
0485     rlwimi  r15,r14,32-8,22,25  /* Move in U bits */
0486     mtspr   SPRN_MAS2,r10
0487     andi.   r10,r14,_PAGE_DIRTY
0488     rlwimi  r15,r14,32-2,26,31  /* Move in BAP bits */
0489 
0490     /* Mask out SW and UW if !DIRTY (XXX optimize this !) */
0491     bne 1f
0492     li  r10,MAS3_SW|MAS3_UW
0493     andc    r15,r15,r10
0494 1:
0495     mtspr   SPRN_MAS7_MAS3,r15
0496 
0497     mfspr   r10,SPRN_MAS0
0498     b   tlb_miss_huge_done_e6500
0499 
0500 tlb_miss_kernel_e6500:
0501     ld  r14,PACA_KERNELPGD(r13)
0502     srdi    r15,r16,44      /* get kernel region */
0503     xoris   r15,r15,0xc     /* Check for vmalloc region */
0504     cmplwi  cr1,r15,1
0505     beq+    cr1,tlb_miss_common_e6500
0506 
0507 tlb_miss_fault_e6500:
0508     tlb_unlock_e6500
0509     /* We need to check if it was an instruction miss */
0510     andi.   r16,r16,1
0511     bne itlb_miss_fault_e6500
0512 dtlb_miss_fault_e6500:
0513     tlb_epilog_bolted
0514     b   exc_data_storage_book3e
0515 itlb_miss_fault_e6500:
0516     tlb_epilog_bolted
0517     b   exc_instruction_storage_book3e
0518 #endif /* CONFIG_PPC_FSL_BOOK3E */
0519 
0520 /**********************************************************************
0521  *                                                                    *
0522  * TLB miss handling for Book3E with TLB reservation and HES support  *
0523  *                                                                    *
0524  **********************************************************************/
0525 
0526 
0527 /* Data TLB miss */
0528     START_EXCEPTION(data_tlb_miss)
0529     TLB_MISS_PROLOG
0530 
0531     /* Now we handle the fault proper. We only save DEAR in normal
0532      * fault case since that's the only interesting values here.
0533      * We could probably also optimize by not saving SRR0/1 in the
0534      * linear mapping case but I'll leave that for later
0535      */
0536     mfspr   r14,SPRN_ESR
0537     mfspr   r16,SPRN_DEAR       /* get faulting address */
0538     srdi    r15,r16,44      /* get region */
0539     xoris   r15,r15,0xc
0540     cmpldi  cr0,r15,0       /* linear mapping ? */
0541     beq tlb_load_linear     /* yes -> go to linear map load */
0542     cmpldi  cr1,r15,1       /* vmalloc mapping ? */
0543 
0544     /* The page tables are mapped virtually linear. At this point, though,
0545      * we don't know whether we are trying to fault in a first level
0546      * virtual address or a virtual page table address. We can get that
0547      * from bit 0x1 of the region ID which we have set for a page table
0548      */
0549     andis.  r10,r15,0x1
0550     bne-    virt_page_table_tlb_miss
0551 
0552     std r14,EX_TLB_ESR(r12);    /* save ESR */
0553     std r16,EX_TLB_DEAR(r12);   /* save DEAR */
0554 
0555      /* We need _PAGE_PRESENT and  _PAGE_ACCESSED set */
0556     li  r11,_PAGE_PRESENT
0557     oris    r11,r11,_PAGE_ACCESSED@h
0558 
0559     /* We do the user/kernel test for the PID here along with the RW test
0560      */
0561     srdi.   r15,r16,60      /* Check for user region */
0562 
0563     /* We pre-test some combination of permissions to avoid double
0564      * faults:
0565      *
0566      * We move the ESR:ST bit into the position of _PAGE_BAP_SW in the PTE
0567      * ESR_ST   is 0x00800000
0568      * _PAGE_BAP_SW is 0x00000010
0569      * So the shift is >> 19. This tests for supervisor writeability.
0570      * If the page happens to be supervisor writeable and not user
0571      * writeable, we will take a new fault later, but that should be
0572      * a rare enough case.
0573      *
0574      * We also move ESR_ST in _PAGE_DIRTY position
0575      * _PAGE_DIRTY is 0x00001000 so the shift is >> 11
0576      *
0577      * MAS1 is preset for all we need except for TID that needs to
0578      * be cleared for kernel translations
0579      */
0580     rlwimi  r11,r14,32-19,27,27
0581     rlwimi  r11,r14,32-16,19,19
0582     beq normal_tlb_miss_user
0583     /* XXX replace the RMW cycles with immediate loads + writes */
0584 1:  mfspr   r10,SPRN_MAS1
0585     rlwinm  r10,r10,0,16,1      /* Clear TID */
0586     mtspr   SPRN_MAS1,r10
0587     beq+    cr1,normal_tlb_miss
0588 
0589     /* We got a crappy address, just fault with whatever DEAR and ESR
0590      * are here
0591      */
0592     TLB_MISS_EPILOG_ERROR
0593     b   exc_data_storage_book3e
0594 
0595 /* Instruction TLB miss */
0596     START_EXCEPTION(instruction_tlb_miss)
0597     TLB_MISS_PROLOG
0598 
0599     /* If we take a recursive fault, the second level handler may need
0600      * to know whether we are handling a data or instruction fault in
0601      * order to get to the right store fault handler. We provide that
0602      * info by writing a crazy value in ESR in our exception frame
0603      */
0604     li  r14,-1  /* store to exception frame is done later */
0605 
0606     /* Now we handle the fault proper. We only save DEAR in the non
0607      * linear mapping case since we know the linear mapping case will
0608      * not re-enter. We could indeed optimize and also not save SRR0/1
0609      * in the linear mapping case but I'll leave that for later
0610      *
0611      * Faulting address is SRR0 which is already in r16
0612      */
0613     srdi    r15,r16,44      /* get region */
0614     xoris   r15,r15,0xc
0615     cmpldi  cr0,r15,0       /* linear mapping ? */
0616     beq tlb_load_linear     /* yes -> go to linear map load */
0617     cmpldi  cr1,r15,1       /* vmalloc mapping ? */
0618 
0619     /* We do the user/kernel test for the PID here along with the RW test
0620      */
0621     li  r11,_PAGE_PRESENT|_PAGE_BAP_UX  /* Base perm */
0622     oris    r11,r11,_PAGE_ACCESSED@h
0623 
0624     srdi.   r15,r16,60          /* Check for user region */
0625     std r14,EX_TLB_ESR(r12)     /* write crazy -1 to frame */
0626     beq normal_tlb_miss_user
0627 
0628     li  r11,_PAGE_PRESENT|_PAGE_BAP_SX  /* Base perm */
0629     oris    r11,r11,_PAGE_ACCESSED@h
0630     /* XXX replace the RMW cycles with immediate loads + writes */
0631     mfspr   r10,SPRN_MAS1
0632     rlwinm  r10,r10,0,16,1          /* Clear TID */
0633     mtspr   SPRN_MAS1,r10
0634     beq+    cr1,normal_tlb_miss
0635 
0636     /* We got a crappy address, just fault */
0637     TLB_MISS_EPILOG_ERROR
0638     b   exc_instruction_storage_book3e
0639 
0640 /*
0641  * This is the guts of the first-level TLB miss handler for direct
0642  * misses. We are entered with:
0643  *
0644  * r16 = faulting address
0645  * r15 = region ID
0646  * r14 = crap (free to use)
0647  * r13 = PACA
0648  * r12 = TLB exception frame in PACA
0649  * r11 = PTE permission mask
0650  * r10 = crap (free to use)
0651  */
0652 normal_tlb_miss_user:
0653 #ifdef CONFIG_PPC_KUAP
0654     mfspr   r14,SPRN_MAS1
0655     rlwinm. r14,r14,0,0x3fff0000
0656     beq-    normal_tlb_miss_access_fault /* KUAP fault */
0657 #endif
0658 normal_tlb_miss:
0659     /* So we first construct the page table address. We do that by
0660      * shifting the bottom of the address (not the region ID) by
0661      * PAGE_SHIFT-3, clearing the bottom 3 bits (get a PTE ptr) and
0662      * or'ing the fourth high bit.
0663      *
0664      * NOTE: For 64K pages, we do things slightly differently in
0665      * order to handle the weird page table format used by linux
0666      */
0667     srdi    r15,r16,44
0668     oris    r10,r15,0x1
0669     rldicl  r14,r16,64-(PAGE_SHIFT-3),PAGE_SHIFT-3+4
0670     sldi    r15,r10,44
0671     clrrdi  r14,r14,19
0672     or  r10,r15,r14
0673 
0674     ld  r14,0(r10)
0675 
0676 finish_normal_tlb_miss:
0677     /* Check if required permissions are met */
0678     andc.   r15,r11,r14
0679     bne-    normal_tlb_miss_access_fault
0680 
0681     /* Now we build the MAS:
0682      *
0683      * MAS 0   :    Fully setup with defaults in MAS4 and TLBnCFG
0684      * MAS 1   :    Almost fully setup
0685      *               - PID already updated by caller if necessary
0686      *               - TSIZE need change if !base page size, not
0687      *                 yet implemented for now
0688      * MAS 2   :    Defaults not useful, need to be redone
0689      * MAS 3+7 :    Needs to be done
0690      *
0691      * TODO: mix up code below for better scheduling
0692      */
0693     clrrdi  r10,r16,12      /* Clear low crap in EA */
0694     rlwimi  r10,r14,32-19,27,31 /* Insert WIMGE */
0695     mtspr   SPRN_MAS2,r10
0696 
0697     /* Check page size, if not standard, update MAS1 */
0698     rldicl  r10,r14,64-8,64-8
0699     cmpldi  cr0,r10,BOOK3E_PAGESZ_4K
0700     beq-    1f
0701     mfspr   r11,SPRN_MAS1
0702     rlwimi  r11,r14,31,21,24
0703     rlwinm  r11,r11,0,21,19
0704     mtspr   SPRN_MAS1,r11
0705 1:
0706     /* Move RPN in position */
0707     rldicr  r11,r14,64-(PTE_RPN_SHIFT-PAGE_SHIFT),63-PAGE_SHIFT
0708     clrldi  r15,r11,12      /* Clear crap at the top */
0709     rlwimi  r15,r14,32-8,22,25  /* Move in U bits */
0710     rlwimi  r15,r14,32-2,26,31  /* Move in BAP bits */
0711 
0712     /* Mask out SW and UW if !DIRTY (XXX optimize this !) */
0713     andi.   r11,r14,_PAGE_DIRTY
0714     bne 1f
0715     li  r11,MAS3_SW|MAS3_UW
0716     andc    r15,r15,r11
0717 1:
0718     srdi    r16,r15,32
0719     mtspr   SPRN_MAS3,r15
0720     mtspr   SPRN_MAS7,r16
0721 
0722     tlbwe
0723 
0724 normal_tlb_miss_done:
0725     /* We don't bother with restoring DEAR or ESR since we know we are
0726      * level 0 and just going back to userland. They are only needed
0727      * if you are going to take an access fault
0728      */
0729     TLB_MISS_EPILOG_SUCCESS
0730     rfi
0731 
0732 normal_tlb_miss_access_fault:
0733     /* We need to check if it was an instruction miss */
0734     andi.   r10,r11,_PAGE_BAP_UX
0735     bne 1f
0736     ld  r14,EX_TLB_DEAR(r12)
0737     ld  r15,EX_TLB_ESR(r12)
0738     mtspr   SPRN_DEAR,r14
0739     mtspr   SPRN_ESR,r15
0740     TLB_MISS_EPILOG_ERROR
0741     b   exc_data_storage_book3e
0742 1:  TLB_MISS_EPILOG_ERROR
0743     b   exc_instruction_storage_book3e
0744 
0745 
0746 /*
0747  * This is the guts of the second-level TLB miss handler for direct
0748  * misses. We are entered with:
0749  *
0750  * r16 = virtual page table faulting address
0751  * r15 = region (top 4 bits of address)
0752  * r14 = crap (free to use)
0753  * r13 = PACA
0754  * r12 = TLB exception frame in PACA
0755  * r11 = crap (free to use)
0756  * r10 = crap (free to use)
0757  *
0758  * Note that this should only ever be called as a second level handler
0759  * with the current scheme when using SW load.
0760  * That means we can always get the original fault DEAR at
0761  * EX_TLB_DEAR-EX_TLB_SIZE(r12)
0762  *
0763  * It can be re-entered by the linear mapping miss handler. However, to
0764  * avoid too much complication, it will restart the whole fault at level
0765  * 0 so we don't care too much about clobbers
0766  *
0767  * XXX That code was written back when we couldn't clobber r14. We can now,
0768  * so we could probably optimize things a bit
0769  */
0770 virt_page_table_tlb_miss:
0771     /* Are we hitting a kernel page table ? */
0772     srdi    r15,r16,60
0773     andi.   r10,r15,0x8
0774 
0775     /* The cool thing now is that r10 contains 0 for user and 8 for kernel,
0776      * and we happen to have the swapper_pg_dir at offset 8 from the user
0777      * pgdir in the PACA :-).
0778      */
0779     add r11,r10,r13
0780 
0781     /* If kernel, we need to clear MAS1 TID */
0782     beq 1f
0783     /* XXX replace the RMW cycles with immediate loads + writes */
0784     mfspr   r10,SPRN_MAS1
0785     rlwinm  r10,r10,0,16,1          /* Clear TID */
0786     mtspr   SPRN_MAS1,r10
0787 #ifdef CONFIG_PPC_KUAP
0788     b   2f
0789 1:
0790     mfspr   r10,SPRN_MAS1
0791     rlwinm. r10,r10,0,0x3fff0000
0792     beq-    virt_page_table_tlb_miss_fault /* KUAP fault */
0793 2:
0794 #else
0795 1:
0796 #endif
0797 
0798     /* Now, we need to walk the page tables. First check if we are in
0799      * range.
0800      */
0801     rldicl  r10,r16,64-(VPTE_INDEX_SIZE+3),VPTE_INDEX_SIZE+3+4
0802     cmpldi  r10,0x80
0803     bne-    virt_page_table_tlb_miss_fault
0804 
0805     /* Get the PGD pointer */
0806     ld  r15,PACAPGD(r11)
0807     cmpldi  cr0,r15,0
0808     beq-    virt_page_table_tlb_miss_fault
0809 
0810     /* Get to PGD entry */
0811     rldicl  r11,r16,64-VPTE_PGD_SHIFT,64-PGD_INDEX_SIZE-3
0812     clrrdi  r10,r11,3
0813     ldx r15,r10,r15
0814     cmpdi   cr0,r15,0
0815     bge virt_page_table_tlb_miss_fault
0816 
0817     /* Get to PUD entry */
0818     rldicl  r11,r16,64-VPTE_PUD_SHIFT,64-PUD_INDEX_SIZE-3
0819     clrrdi  r10,r11,3
0820     ldx r15,r10,r15
0821     cmpdi   cr0,r15,0
0822     bge virt_page_table_tlb_miss_fault
0823 
0824     /* Get to PMD entry */
0825     rldicl  r11,r16,64-VPTE_PMD_SHIFT,64-PMD_INDEX_SIZE-3
0826     clrrdi  r10,r11,3
0827     ldx r15,r10,r15
0828     cmpdi   cr0,r15,0
0829     bge virt_page_table_tlb_miss_fault
0830 
0831     /* Ok, we're all right, we can now create a kernel translation for
0832      * a 4K or 64K page from r16 -> r15.
0833      */
0834     /* Now we build the MAS:
0835      *
0836      * MAS 0   :    Fully setup with defaults in MAS4 and TLBnCFG
0837      * MAS 1   :    Almost fully setup
0838      *               - PID already updated by caller if necessary
0839      *               - TSIZE for now is base page size always
0840      * MAS 2   :    Use defaults
0841      * MAS 3+7 :    Needs to be done
0842      *
0843      * So we only do MAS 2 and 3 for now...
0844      */
0845     clrldi  r11,r15,4       /* remove region ID from RPN */
0846     ori r10,r11,1       /* Or-in SR */
0847 
0848     srdi    r16,r10,32
0849     mtspr   SPRN_MAS3,r10
0850     mtspr   SPRN_MAS7,r16
0851 
0852     tlbwe
0853 
0854     /* Return to caller, normal case */
0855     TLB_MISS_EPILOG_SUCCESS
0856     rfi
0857 
0858 virt_page_table_tlb_miss_fault:
0859     /* If we fault here, things are a little bit tricky. We need to call
0860      * either data or instruction store fault, and we need to retrieve
0861      * the original fault address and ESR (for data).
0862      *
0863      * The thing is, we know that in normal circumstances, this is
0864      * always called as a second level tlb miss for SW load or as a first
0865      * level TLB miss for HW load, so we should be able to peek at the
0866      * relevant information in the first exception frame in the PACA.
0867      *
0868      * However, we do need to double check that, because we may just hit
0869      * a stray kernel pointer or a userland attack trying to hit those
0870      * areas. If that is the case, we do a data fault. (We can't get here
0871      * from an instruction tlb miss anyway).
0872      *
0873      * Note also that when going to a fault, we must unwind the previous
0874      * level as well. Since we are doing that, we don't need to clear or
0875      * restore the TLB reservation neither.
0876      */
0877     subf    r10,r13,r12
0878     cmpldi  cr0,r10,PACA_EXTLB+EX_TLB_SIZE
0879     bne-    virt_page_table_tlb_miss_whacko_fault
0880 
0881     /* We dig the original DEAR and ESR from slot 0 */
0882     ld  r15,EX_TLB_DEAR+PACA_EXTLB(r13)
0883     ld  r16,EX_TLB_ESR+PACA_EXTLB(r13)
0884 
0885     /* We check for the "special" ESR value for instruction faults */
0886     cmpdi   cr0,r16,-1
0887     beq 1f
0888     mtspr   SPRN_DEAR,r15
0889     mtspr   SPRN_ESR,r16
0890     TLB_MISS_EPILOG_ERROR
0891     b   exc_data_storage_book3e
0892 1:  TLB_MISS_EPILOG_ERROR
0893     b   exc_instruction_storage_book3e
0894 
0895 virt_page_table_tlb_miss_whacko_fault:
0896     /* The linear fault will restart everything so ESR and DEAR will
0897      * not have been clobbered, let's just fault with what we have
0898      */
0899     TLB_MISS_EPILOG_ERROR
0900     b   exc_data_storage_book3e
0901 
0902 
0903 /**************************************************************
0904  *                                                            *
0905  * TLB miss handling for Book3E with hw page table support    *
0906  *                                                            *
0907  **************************************************************/
0908 
0909 
0910 /* Data TLB miss */
0911     START_EXCEPTION(data_tlb_miss_htw)
0912     TLB_MISS_PROLOG
0913 
0914     /* Now we handle the fault proper. We only save DEAR in normal
0915      * fault case since that's the only interesting values here.
0916      * We could probably also optimize by not saving SRR0/1 in the
0917      * linear mapping case but I'll leave that for later
0918      */
0919     mfspr   r14,SPRN_ESR
0920     mfspr   r16,SPRN_DEAR       /* get faulting address */
0921     srdi    r11,r16,44      /* get region */
0922     xoris   r11,r11,0xc
0923     cmpldi  cr0,r11,0       /* linear mapping ? */
0924     beq tlb_load_linear     /* yes -> go to linear map load */
0925     cmpldi  cr1,r11,1       /* vmalloc mapping ? */
0926 
0927     /* We do the user/kernel test for the PID here along with the RW test
0928      */
0929     srdi.   r11,r16,60      /* Check for user region */
0930     ld  r15,PACAPGD(r13)    /* Load user pgdir */
0931     beq htw_tlb_miss
0932 
0933     /* XXX replace the RMW cycles with immediate loads + writes */
0934 1:  mfspr   r10,SPRN_MAS1
0935     rlwinm  r10,r10,0,16,1      /* Clear TID */
0936     mtspr   SPRN_MAS1,r10
0937     ld  r15,PACA_KERNELPGD(r13) /* Load kernel pgdir */
0938     beq+    cr1,htw_tlb_miss
0939 
0940     /* We got a crappy address, just fault with whatever DEAR and ESR
0941      * are here
0942      */
0943     TLB_MISS_EPILOG_ERROR
0944     b   exc_data_storage_book3e
0945 
0946 /* Instruction TLB miss */
0947     START_EXCEPTION(instruction_tlb_miss_htw)
0948     TLB_MISS_PROLOG
0949 
0950     /* If we take a recursive fault, the second level handler may need
0951      * to know whether we are handling a data or instruction fault in
0952      * order to get to the right store fault handler. We provide that
0953      * info by keeping a crazy value for ESR in r14
0954      */
0955     li  r14,-1  /* store to exception frame is done later */
0956 
0957     /* Now we handle the fault proper. We only save DEAR in the non
0958      * linear mapping case since we know the linear mapping case will
0959      * not re-enter. We could indeed optimize and also not save SRR0/1
0960      * in the linear mapping case but I'll leave that for later
0961      *
0962      * Faulting address is SRR0 which is already in r16
0963      */
0964     srdi    r11,r16,44      /* get region */
0965     xoris   r11,r11,0xc
0966     cmpldi  cr0,r11,0       /* linear mapping ? */
0967     beq tlb_load_linear     /* yes -> go to linear map load */
0968     cmpldi  cr1,r11,1       /* vmalloc mapping ? */
0969 
0970     /* We do the user/kernel test for the PID here along with the RW test
0971      */
0972     srdi.   r11,r16,60      /* Check for user region */
0973     ld  r15,PACAPGD(r13)        /* Load user pgdir */
0974     beq htw_tlb_miss
0975 
0976     /* XXX replace the RMW cycles with immediate loads + writes */
0977 1:  mfspr   r10,SPRN_MAS1
0978     rlwinm  r10,r10,0,16,1          /* Clear TID */
0979     mtspr   SPRN_MAS1,r10
0980     ld  r15,PACA_KERNELPGD(r13)     /* Load kernel pgdir */
0981     beq+    htw_tlb_miss
0982 
0983     /* We got a crappy address, just fault */
0984     TLB_MISS_EPILOG_ERROR
0985     b   exc_instruction_storage_book3e
0986 
0987 
0988 /*
0989  * This is the guts of the second-level TLB miss handler for direct
0990  * misses. We are entered with:
0991  *
0992  * r16 = virtual page table faulting address
0993  * r15 = PGD pointer
0994  * r14 = ESR
0995  * r13 = PACA
0996  * r12 = TLB exception frame in PACA
0997  * r11 = crap (free to use)
0998  * r10 = crap (free to use)
0999  *
1000  * It can be re-entered by the linear mapping miss handler. However, to
1001  * avoid too much complication, it will save/restore things for us
1002  */
1003 htw_tlb_miss:
1004 #ifdef CONFIG_PPC_KUAP
1005     mfspr   r10,SPRN_MAS1
1006     rlwinm. r10,r10,0,0x3fff0000
1007     beq-    htw_tlb_miss_fault /* KUAP fault */
1008 #endif
1009     /* Search if we already have a TLB entry for that virtual address, and
1010      * if we do, bail out.
1011      *
1012      * MAS1:IND should be already set based on MAS4
1013      */
1014     PPC_TLBSRX_DOT(0,R16)
1015     beq htw_tlb_miss_done
1016 
1017     /* Now, we need to walk the page tables. First check if we are in
1018      * range.
1019      */
1020     rldicl. r10,r16,64-PGTABLE_EADDR_SIZE,PGTABLE_EADDR_SIZE+4
1021     bne-    htw_tlb_miss_fault
1022 
1023     /* Get the PGD pointer */
1024     cmpldi  cr0,r15,0
1025     beq-    htw_tlb_miss_fault
1026 
1027     /* Get to PGD entry */
1028     rldicl  r11,r16,64-(PGDIR_SHIFT-3),64-PGD_INDEX_SIZE-3
1029     clrrdi  r10,r11,3
1030     ldx r15,r10,r15
1031     cmpdi   cr0,r15,0
1032     bge htw_tlb_miss_fault
1033 
1034     /* Get to PUD entry */
1035     rldicl  r11,r16,64-(PUD_SHIFT-3),64-PUD_INDEX_SIZE-3
1036     clrrdi  r10,r11,3
1037     ldx r15,r10,r15
1038     cmpdi   cr0,r15,0
1039     bge htw_tlb_miss_fault
1040 
1041     /* Get to PMD entry */
1042     rldicl  r11,r16,64-(PMD_SHIFT-3),64-PMD_INDEX_SIZE-3
1043     clrrdi  r10,r11,3
1044     ldx r15,r10,r15
1045     cmpdi   cr0,r15,0
1046     bge htw_tlb_miss_fault
1047 
1048     /* Ok, we're all right, we can now create an indirect entry for
1049      * a 1M or 256M page.
1050      *
1051      * The last trick is now that because we use "half" pages for
1052      * the HTW (1M IND is 2K and 256M IND is 32K) we need to account
1053      * for an added LSB bit to the RPN. For 64K pages, there is no
1054      * problem as we already use 32K arrays (half PTE pages), but for
1055      * 4K page we need to extract a bit from the virtual address and
1056      * insert it into the "PA52" bit of the RPN.
1057      */
1058     rlwimi  r15,r16,32-9,20,20
1059     /* Now we build the MAS:
1060      *
1061      * MAS 0   :    Fully setup with defaults in MAS4 and TLBnCFG
1062      * MAS 1   :    Almost fully setup
1063      *               - PID already updated by caller if necessary
1064      *               - TSIZE for now is base ind page size always
1065      * MAS 2   :    Use defaults
1066      * MAS 3+7 :    Needs to be done
1067      */
1068     ori r10,r15,(BOOK3E_PAGESZ_4K << MAS3_SPSIZE_SHIFT)
1069 
1070     srdi    r16,r10,32
1071     mtspr   SPRN_MAS3,r10
1072     mtspr   SPRN_MAS7,r16
1073 
1074     tlbwe
1075 
1076 htw_tlb_miss_done:
1077     /* We don't bother with restoring DEAR or ESR since we know we are
1078      * level 0 and just going back to userland. They are only needed
1079      * if you are going to take an access fault
1080      */
1081     TLB_MISS_EPILOG_SUCCESS
1082     rfi
1083 
1084 htw_tlb_miss_fault:
1085     /* We need to check if it was an instruction miss. We know this
1086      * though because r14 would contain -1
1087      */
1088     cmpdi   cr0,r14,-1
1089     beq 1f
1090     mtspr   SPRN_DEAR,r16
1091     mtspr   SPRN_ESR,r14
1092     TLB_MISS_EPILOG_ERROR
1093     b   exc_data_storage_book3e
1094 1:  TLB_MISS_EPILOG_ERROR
1095     b   exc_instruction_storage_book3e
1096 
1097 /*
1098  * This is the guts of "any" level TLB miss handler for kernel linear
1099  * mapping misses. We are entered with:
1100  *
1101  *
1102  * r16 = faulting address
1103  * r15 = crap (free to use)
1104  * r14 = ESR (data) or -1 (instruction)
1105  * r13 = PACA
1106  * r12 = TLB exception frame in PACA
1107  * r11 = crap (free to use)
1108  * r10 = crap (free to use)
1109  *
1110  * In addition we know that we will not re-enter, so in theory, we could
1111  * use a simpler epilog not restoring SRR0/1 etc.. but we'll do that later.
1112  *
1113  * We also need to be careful about MAS registers here & TLB reservation,
1114  * as we know we'll have clobbered them if we interrupt the main TLB miss
1115  * handlers in which case we probably want to do a full restart at level
1116  * 0 rather than saving / restoring the MAS.
1117  *
1118  * Note: If we care about performance of that core, we can easily shuffle
1119  *       a few things around
1120  */
1121 tlb_load_linear:
1122     /* For now, we assume the linear mapping is contiguous and stops at
1123      * linear_map_top. We also assume the size is a multiple of 1G, thus
1124      * we only use 1G pages for now. That might have to be changed in a
1125      * final implementation, especially when dealing with hypervisors
1126      */
1127     ld  r11,PACATOC(r13)
1128     ld  r11,linear_map_top@got(r11)
1129     ld  r10,0(r11)
1130     tovirt(10,10)
1131     cmpld   cr0,r16,r10
1132     bge tlb_load_linear_fault
1133 
1134     /* MAS1 need whole new setup. */
1135     li  r15,(BOOK3E_PAGESZ_1GB<<MAS1_TSIZE_SHIFT)
1136     oris    r15,r15,MAS1_VALID@h    /* MAS1 needs V and TSIZE */
1137     mtspr   SPRN_MAS1,r15
1138 
1139     /* Already somebody there ? */
1140     PPC_TLBSRX_DOT(0,R16)
1141     beq tlb_load_linear_done
1142 
1143     /* Now we build the remaining MAS. MAS0 and 2 should be fine
1144      * with their defaults, which leaves us with MAS 3 and 7. The
1145      * mapping is linear, so we just take the address, clear the
1146      * region bits, and or in the permission bits which are currently
1147      * hard wired
1148      */
1149     clrrdi  r10,r16,30      /* 1G page index */
1150     clrldi  r10,r10,4       /* clear region bits */
1151     ori r10,r10,MAS3_SR|MAS3_SW|MAS3_SX
1152 
1153     srdi    r16,r10,32
1154     mtspr   SPRN_MAS3,r10
1155     mtspr   SPRN_MAS7,r16
1156 
1157     tlbwe
1158 
1159 tlb_load_linear_done:
1160     /* We use the "error" epilog for success as we do want to
1161      * restore to the initial faulting context, whatever it was.
1162      * We do that because we can't resume a fault within a TLB
1163      * miss handler, due to MAS and TLB reservation being clobbered.
1164      */
1165     TLB_MISS_EPILOG_ERROR
1166     rfi
1167 
1168 tlb_load_linear_fault:
1169     /* We keep the DEAR and ESR around, this shouldn't have happened */
1170     cmpdi   cr0,r14,-1
1171     beq 1f
1172     TLB_MISS_EPILOG_ERROR_SPECIAL
1173     b   exc_data_storage_book3e
1174 1:  TLB_MISS_EPILOG_ERROR_SPECIAL
1175     b   exc_instruction_storage_book3e
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!