sparc/kernel/trampoline_64.S

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 /*
0003  * trampoline.S: Jump start slave processors on sparc64.
0004  *
0005  * Copyright (C) 1997 David S. Miller (davem@caip.rutgers.edu)
0006  */
0007
0008
0009 #include <linux/pgtable.h>
0010 #include <asm/head.h>
0011 #include <asm/asi.h>
0012 #include <asm/lsu.h>
0013 #include <asm/dcr.h>
0014 #include <asm/dcu.h>
0015 #include <asm/pstate.h>
0016 #include <asm/page.h>
0017 #include <asm/spitfire.h>
0018 #include <asm/processor.h>
0019 #include <asm/thread_info.h>
0020 #include <asm/mmu.h>
0021 #include <asm/hypervisor.h>
0022 #include <asm/cpudata.h>
0023
0024     .data
0025     .align  8
0026 call_method:
0027     .asciz  "call-method"
0028     .align  8
0029 itlb_load:
0030     .asciz  "SUNW,itlb-load"
0031     .align  8
0032 dtlb_load:
0033     .asciz  "SUNW,dtlb-load"
0034
0035 #define TRAMP_STACK_SIZE    1024
0036     .align  16
0037 tramp_stack:
0038     .skip   TRAMP_STACK_SIZE
0039
0040     .align      8
0041     .globl      sparc64_cpu_startup, sparc64_cpu_startup_end
0042 sparc64_cpu_startup:
0043     BRANCH_IF_SUN4V(g1, niagara_startup)
0044     BRANCH_IF_CHEETAH_BASE(g1, g5, cheetah_startup)
0045     BRANCH_IF_CHEETAH_PLUS_OR_FOLLOWON(g1, g5, cheetah_plus_startup)
0046
0047     ba,pt   %xcc, spitfire_startup
0048      nop
0049
0050 cheetah_plus_startup:
0051     /* Preserve OBP chosen DCU and DCR register settings.  */
0052     ba,pt   %xcc, cheetah_generic_startup
0053      nop
0054
0055 cheetah_startup:
0056     mov DCR_BPE | DCR_RPE | DCR_SI | DCR_IFPOE | DCR_MS, %g1
0057     wr  %g1, %asr18
0058
0059     sethi   %uhi(DCU_ME|DCU_RE|DCU_HPE|DCU_SPE|DCU_SL|DCU_WE), %g5
0060     or  %g5, %ulo(DCU_ME|DCU_RE|DCU_HPE|DCU_SPE|DCU_SL|DCU_WE), %g5
0061     sllx    %g5, 32, %g5
0062     or  %g5, DCU_DM | DCU_IM | DCU_DC | DCU_IC, %g5
0063     stxa    %g5, [%g0] ASI_DCU_CONTROL_REG
0064     membar  #Sync
0065     /* fallthru */
0066
0067 cheetah_generic_startup:
0068     mov TSB_EXTENSION_P, %g3
0069     stxa    %g0, [%g3] ASI_DMMU
0070     stxa    %g0, [%g3] ASI_IMMU
0071     membar  #Sync
0072
0073     mov TSB_EXTENSION_S, %g3
0074     stxa    %g0, [%g3] ASI_DMMU
0075     membar  #Sync
0076
0077     mov TSB_EXTENSION_N, %g3
0078     stxa    %g0, [%g3] ASI_DMMU
0079     stxa    %g0, [%g3] ASI_IMMU
0080     membar  #Sync
0081     /* fallthru */
0082
0083 niagara_startup:
0084     /* Disable STICK_INT interrupts. */
0085     sethi       %hi(0x80000000), %g5
0086     sllx        %g5, 32, %g5
0087     wr      %g5, %asr25
0088
0089     ba,pt       %xcc, startup_continue
0090      nop
0091
0092 spitfire_startup:
0093     mov     (LSU_CONTROL_IC | LSU_CONTROL_DC | LSU_CONTROL_IM | LSU_CONTROL_DM), %g1
0094     stxa        %g1, [%g0] ASI_LSU_CONTROL
0095     membar      #Sync
0096
0097 startup_continue:
0098     mov     %o0, %l0
0099     BRANCH_IF_SUN4V(g1, niagara_lock_tlb)
0100
0101     sethi       %hi(0x80000000), %g2
0102     sllx        %g2, 32, %g2
0103     wr      %g2, 0, %tick_cmpr
0104
0105     /* Call OBP by hand to lock KERNBASE into i/d tlbs.
0106      * We lock 'num_kernel_image_mappings' consequetive entries.
0107      */
0108     sethi       %hi(prom_entry_lock), %g2
0109 1:  ldstub      [%g2 + %lo(prom_entry_lock)], %g1
0110     brnz,pn     %g1, 1b
0111      nop
0112
0113     /* Get onto temporary stack which will be in the locked
0114      * kernel image.
0115      */
0116     sethi       %hi(tramp_stack), %g1
0117     or      %g1, %lo(tramp_stack), %g1
0118     add     %g1, TRAMP_STACK_SIZE, %g1
0119     sub     %g1, STACKFRAME_SZ + STACK_BIAS + 256, %sp
0120     flushw
0121
0122     /* Setup the loop variables:
0123      * %l3: VADDR base
0124      * %l4: TTE base
0125      * %l5: Loop iterator, iterates from 0 to 'num_kernel_image_mappings'
0126      * %l6: Number of TTE entries to map
0127      * %l7: Highest TTE entry number, we count down
0128      */
0129     sethi       %hi(KERNBASE), %l3
0130     sethi       %hi(kern_locked_tte_data), %l4
0131     ldx     [%l4 + %lo(kern_locked_tte_data)], %l4
0132     clr     %l5
0133     sethi       %hi(num_kernel_image_mappings), %l6
0134     lduw        [%l6 + %lo(num_kernel_image_mappings)], %l6
0135
0136     mov     15, %l7
0137     BRANCH_IF_ANY_CHEETAH(g1,g5,2f)
0138
0139     mov     63, %l7
0140 2:
0141
0142 3:
0143     /* Lock into I-MMU */
0144     sethi       %hi(call_method), %g2
0145     or      %g2, %lo(call_method), %g2
0146     stx     %g2, [%sp + 2047 + 128 + 0x00]
0147     mov     5, %g2
0148     stx     %g2, [%sp + 2047 + 128 + 0x08]
0149     mov     1, %g2
0150     stx     %g2, [%sp + 2047 + 128 + 0x10]
0151     sethi       %hi(itlb_load), %g2
0152     or      %g2, %lo(itlb_load), %g2
0153     stx     %g2, [%sp + 2047 + 128 + 0x18]
0154     sethi       %hi(prom_mmu_ihandle_cache), %g2
0155     lduw        [%g2 + %lo(prom_mmu_ihandle_cache)], %g2
0156     stx     %g2, [%sp + 2047 + 128 + 0x20]
0157
0158     /* Each TTE maps 4MB, convert index to offset.  */
0159     sllx        %l5, 22, %g1
0160
0161     add     %l3, %g1, %g2
0162     stx     %g2, [%sp + 2047 + 128 + 0x28]  ! VADDR
0163     add     %l4, %g1, %g2
0164     stx     %g2, [%sp + 2047 + 128 + 0x30]  ! TTE
0165
0166     /* TTE index is highest minus loop index.  */
0167     sub     %l7, %l5, %g2
0168     stx     %g2, [%sp + 2047 + 128 + 0x38]
0169
0170     sethi       %hi(p1275buf), %g2
0171     or      %g2, %lo(p1275buf), %g2
0172     ldx     [%g2 + 0x08], %o1
0173     call        %o1
0174      add        %sp, (2047 + 128), %o0
0175
0176     /* Lock into D-MMU */
0177     sethi       %hi(call_method), %g2
0178     or      %g2, %lo(call_method), %g2
0179     stx     %g2, [%sp + 2047 + 128 + 0x00]
0180     mov     5, %g2
0181     stx     %g2, [%sp + 2047 + 128 + 0x08]
0182     mov     1, %g2
0183     stx     %g2, [%sp + 2047 + 128 + 0x10]
0184     sethi       %hi(dtlb_load), %g2
0185     or      %g2, %lo(dtlb_load), %g2
0186     stx     %g2, [%sp + 2047 + 128 + 0x18]
0187     sethi       %hi(prom_mmu_ihandle_cache), %g2
0188     lduw        [%g2 + %lo(prom_mmu_ihandle_cache)], %g2
0189     stx     %g2, [%sp + 2047 + 128 + 0x20]
0190
0191     /* Each TTE maps 4MB, convert index to offset.  */
0192     sllx        %l5, 22, %g1
0193
0194     add     %l3, %g1, %g2
0195     stx     %g2, [%sp + 2047 + 128 + 0x28]  ! VADDR
0196     add     %l4, %g1, %g2
0197     stx     %g2, [%sp + 2047 + 128 + 0x30]  ! TTE
0198
0199     /* TTE index is highest minus loop index.  */
0200     sub     %l7, %l5, %g2
0201     stx     %g2, [%sp + 2047 + 128 + 0x38]
0202
0203     sethi       %hi(p1275buf), %g2
0204     or      %g2, %lo(p1275buf), %g2
0205     ldx     [%g2 + 0x08], %o1
0206     call        %o1
0207      add        %sp, (2047 + 128), %o0
0208
0209     add     %l5, 1, %l5
0210     cmp     %l5, %l6
0211     bne,pt      %xcc, 3b
0212      nop
0213
0214     sethi       %hi(prom_entry_lock), %g2
0215     stb     %g0, [%g2 + %lo(prom_entry_lock)]
0216
0217     ba,pt       %xcc, after_lock_tlb
0218      nop
0219
0220 niagara_lock_tlb:
0221     sethi       %hi(KERNBASE), %l3
0222     sethi       %hi(kern_locked_tte_data), %l4
0223     ldx     [%l4 + %lo(kern_locked_tte_data)], %l4
0224     clr     %l5
0225     sethi       %hi(num_kernel_image_mappings), %l6
0226     lduw        [%l6 + %lo(num_kernel_image_mappings)], %l6
0227
0228 1:
0229     mov     HV_FAST_MMU_MAP_PERM_ADDR, %o5
0230     sllx        %l5, 22, %g2
0231     add     %l3, %g2, %o0
0232     clr     %o1
0233     add     %l4, %g2, %o2
0234     mov     HV_MMU_IMMU, %o3
0235     ta      HV_FAST_TRAP
0236
0237     mov     HV_FAST_MMU_MAP_PERM_ADDR, %o5
0238     sllx        %l5, 22, %g2
0239     add     %l3, %g2, %o0
0240     clr     %o1
0241     add     %l4, %g2, %o2
0242     mov     HV_MMU_DMMU, %o3
0243     ta      HV_FAST_TRAP
0244
0245     add     %l5, 1, %l5
0246     cmp     %l5, %l6
0247     bne,pt      %xcc, 1b
0248      nop
0249
0250 after_lock_tlb:
0251     wrpr        %g0, (PSTATE_PRIV | PSTATE_PEF), %pstate
0252     wr      %g0, 0, %fprs
0253
0254     wr      %g0, ASI_P, %asi
0255
0256     mov     PRIMARY_CONTEXT, %g7
0257
0258 661:    stxa        %g0, [%g7] ASI_DMMU
0259     .section    .sun4v_1insn_patch, "ax"
0260     .word       661b
0261     stxa        %g0, [%g7] ASI_MMU
0262     .previous
0263
0264     membar      #Sync
0265     mov     SECONDARY_CONTEXT, %g7
0266
0267 661:    stxa        %g0, [%g7] ASI_DMMU
0268     .section    .sun4v_1insn_patch, "ax"
0269     .word       661b
0270     stxa        %g0, [%g7] ASI_MMU
0271     .previous
0272
0273     membar      #Sync
0274
0275     /* Everything we do here, until we properly take over the
0276      * trap table, must be done with extreme care.  We cannot
0277      * make any references to %g6 (current thread pointer),
0278      * %g4 (current task pointer), or %g5 (base of current cpu's
0279      * per-cpu area) until we properly take over the trap table
0280      * from the firmware and hypervisor.
0281      *
0282      * Get onto temporary stack which is in the locked kernel image.
0283      */
0284     sethi       %hi(tramp_stack), %g1
0285     or      %g1, %lo(tramp_stack), %g1
0286     add     %g1, TRAMP_STACK_SIZE, %g1
0287     sub     %g1, STACKFRAME_SZ + STACK_BIAS + 256, %sp
0288     mov     0, %fp
0289
0290     /* Put garbage in these registers to trap any access to them.  */
0291     set     0xdeadbeef, %g4
0292     set     0xdeadbeef, %g5
0293     set     0xdeadbeef, %g6
0294
0295     call        init_irqwork_curcpu
0296      nop
0297
0298     sethi       %hi(tlb_type), %g3
0299     lduw        [%g3 + %lo(tlb_type)], %g2
0300     cmp     %g2, 3
0301     bne,pt      %icc, 1f
0302      nop
0303
0304     call        hard_smp_processor_id
0305      nop
0306
0307     call        sun4v_register_mondo_queues
0308      nop
0309
0310 1:  call        init_cur_cpu_trap
0311      ldx        [%l0], %o0
0312
0313     /* Start using proper page size encodings in ctx register.  */
0314     sethi       %hi(sparc64_kern_pri_context), %g3
0315     ldx     [%g3 + %lo(sparc64_kern_pri_context)], %g2
0316     mov     PRIMARY_CONTEXT, %g1
0317
0318 661:    stxa        %g2, [%g1] ASI_DMMU
0319     .section    .sun4v_1insn_patch, "ax"
0320     .word       661b
0321     stxa        %g2, [%g1] ASI_MMU
0322     .previous
0323
0324     membar      #Sync
0325
0326     wrpr        %g0, 0, %wstate
0327
0328     sethi       %hi(prom_entry_lock), %g2
0329 1:  ldstub      [%g2 + %lo(prom_entry_lock)], %g1
0330     brnz,pn     %g1, 1b
0331      nop
0332
0333     /* As a hack, put &init_thread_union into %g6.
0334      * prom_world() loads from here to restore the %asi
0335      * register.
0336      */
0337     sethi       %hi(init_thread_union), %g6
0338     or      %g6, %lo(init_thread_union), %g6
0339
0340     sethi       %hi(is_sun4v), %o0
0341     lduw        [%o0 + %lo(is_sun4v)], %o0
0342     brz,pt      %o0, 2f
0343      nop
0344
0345     TRAP_LOAD_TRAP_BLOCK(%g2, %g3)
0346     add     %g2, TRAP_PER_CPU_FAULT_INFO, %g2
0347     stxa        %g2, [%g0] ASI_SCRATCHPAD
0348
0349     /* Compute physical address:
0350      *
0351      * paddr = kern_base + (mmfsa_vaddr - KERNBASE)
0352      */
0353     sethi       %hi(KERNBASE), %g3
0354     sub     %g2, %g3, %g2
0355     sethi       %hi(kern_base), %g3
0356     ldx     [%g3 + %lo(kern_base)], %g3
0357     add     %g2, %g3, %o1
0358     sethi       %hi(sparc64_ttable_tl0), %o0
0359
0360     set     prom_set_trap_table_name, %g2
0361     stx     %g2, [%sp + 2047 + 128 + 0x00]
0362     mov     2, %g2
0363     stx     %g2, [%sp + 2047 + 128 + 0x08]
0364     mov     0, %g2
0365     stx     %g2, [%sp + 2047 + 128 + 0x10]
0366     stx     %o0, [%sp + 2047 + 128 + 0x18]
0367     stx     %o1, [%sp + 2047 + 128 + 0x20]
0368     sethi       %hi(p1275buf), %g2
0369     or      %g2, %lo(p1275buf), %g2
0370     ldx     [%g2 + 0x08], %o1
0371     call        %o1
0372      add        %sp, (2047 + 128), %o0
0373
0374     ba,pt       %xcc, 3f
0375      nop
0376
0377 2:  sethi       %hi(sparc64_ttable_tl0), %o0
0378     set     prom_set_trap_table_name, %g2
0379     stx     %g2, [%sp + 2047 + 128 + 0x00]
0380     mov     1, %g2
0381     stx     %g2, [%sp + 2047 + 128 + 0x08]
0382     mov     0, %g2
0383     stx     %g2, [%sp + 2047 + 128 + 0x10]
0384     stx     %o0, [%sp + 2047 + 128 + 0x18]
0385     sethi       %hi(p1275buf), %g2
0386     or      %g2, %lo(p1275buf), %g2
0387     ldx     [%g2 + 0x08], %o1
0388     call        %o1
0389      add        %sp, (2047 + 128), %o0
0390
0391 3:  sethi       %hi(prom_entry_lock), %g2
0392     stb     %g0, [%g2 + %lo(prom_entry_lock)]
0393
0394     ldx     [%l0], %g6
0395     ldx     [%g6 + TI_TASK], %g4
0396
0397     mov     1, %g5
0398     sllx        %g5, THREAD_SHIFT, %g5
0399     sub     %g5, (STACKFRAME_SZ + STACK_BIAS), %g5
0400     add     %g6, %g5, %sp
0401
0402     rdpr        %pstate, %o1
0403     or      %o1, PSTATE_IE, %o1
0404     wrpr        %o1, 0, %pstate
0405
0406     call        smp_callin
0407      nop
0408
0409     call        cpu_panic
0410      nop
0411 1:  b,a,pt      %xcc, 1b
0412
0413     .align      8
0414 sparc64_cpu_startup_end: