mips/kernel/cps-vec.S

0001 /* SPDX-License-Identifier: GPL-2.0-or-later */
0002 /*
0003  * Copyright (C) 2013 Imagination Technologies
0004  * Author: Paul Burton <paul.burton@mips.com>
0005  */
0006
0007 #include <asm/addrspace.h>
0008 #include <asm/asm.h>
0009 #include <asm/asm-offsets.h>
0010 #include <asm/asmmacro.h>
0011 #include <asm/cacheops.h>
0012 #include <asm/eva.h>
0013 #include <asm/mipsregs.h>
0014 #include <asm/mipsmtregs.h>
0015 #include <asm/pm.h>
0016
0017 #define GCR_CPC_BASE_OFS    0x0088
0018 #define GCR_CL_COHERENCE_OFS    0x2008
0019 #define GCR_CL_ID_OFS       0x2028
0020
0021 #define CPC_CL_VC_STOP_OFS  0x2020
0022 #define CPC_CL_VC_RUN_OFS   0x2028
0023
0024 .extern mips_cm_base
0025
0026 .set noreorder
0027
0028 #ifdef CONFIG_64BIT
0029 # define STATUS_BITDEPS     ST0_KX
0030 #else
0031 # define STATUS_BITDEPS     0
0032 #endif
0033
0034 #ifdef CONFIG_MIPS_CPS_NS16550
0035
0036 #define DUMP_EXCEP(name)        \
0037     PTR_LA  a0, 8f;         \
0038     jal mips_cps_bev_dump;  \
0039      nop;               \
0040     TEXT(name)
0041
0042 #else /* !CONFIG_MIPS_CPS_NS16550 */
0043
0044 #define DUMP_EXCEP(name)
0045
0046 #endif /* !CONFIG_MIPS_CPS_NS16550 */
0047
0048     /*
0049      * Set dest to non-zero if the core supports the MT ASE, else zero. If
0050      * MT is not supported then branch to nomt.
0051      */
0052     .macro  has_mt  dest, nomt
0053     mfc0    \dest, CP0_CONFIG, 1
0054     bgez    \dest, \nomt
0055      mfc0   \dest, CP0_CONFIG, 2
0056     bgez    \dest, \nomt
0057      mfc0   \dest, CP0_CONFIG, 3
0058     andi    \dest, \dest, MIPS_CONF3_MT
0059     beqz    \dest, \nomt
0060      nop
0061     .endm
0062
0063     /*
0064      * Set dest to non-zero if the core supports MIPSr6 multithreading
0065      * (ie. VPs), else zero. If MIPSr6 multithreading is not supported then
0066      * branch to nomt.
0067      */
0068     .macro  has_vp  dest, nomt
0069     mfc0    \dest, CP0_CONFIG, 1
0070     bgez    \dest, \nomt
0071      mfc0   \dest, CP0_CONFIG, 2
0072     bgez    \dest, \nomt
0073      mfc0   \dest, CP0_CONFIG, 3
0074     bgez    \dest, \nomt
0075      mfc0   \dest, CP0_CONFIG, 4
0076     bgez    \dest, \nomt
0077      mfc0   \dest, CP0_CONFIG, 5
0078     andi    \dest, \dest, MIPS_CONF5_VP
0079     beqz    \dest, \nomt
0080      nop
0081     .endm
0082
0083     /* Calculate an uncached address for the CM GCRs */
0084     .macro  cmgcrb  dest
0085     .set    push
0086     .set    noat
0087     MFC0    $1, CP0_CMGCRBASE
0088     PTR_SLL $1, $1, 4
0089     PTR_LI  \dest, UNCAC_BASE
0090     PTR_ADDU \dest, \dest, $1
0091     .set    pop
0092     .endm
0093
0094 .balign 0x1000
0095
0096 LEAF(mips_cps_core_entry)
0097     /*
0098      * These first 4 bytes will be patched by cps_smp_setup to load the
0099      * CCA to use into register s0.
0100      */
0101     .word   0
0102
0103     /* Check whether we're here due to an NMI */
0104     mfc0    k0, CP0_STATUS
0105     and k0, k0, ST0_NMI
0106     beqz    k0, not_nmi
0107      nop
0108
0109     /* This is an NMI */
0110     PTR_LA  k0, nmi_handler
0111     jr  k0
0112      nop
0113
0114 not_nmi:
0115     /* Setup Cause */
0116     li  t0, CAUSEF_IV
0117     mtc0    t0, CP0_CAUSE
0118
0119     /* Setup Status */
0120     li  t0, ST0_CU1 | ST0_CU0 | ST0_BEV | STATUS_BITDEPS
0121     mtc0    t0, CP0_STATUS
0122
0123     /* Skip cache & coherence setup if we're already coherent */
0124     cmgcrb  v1
0125     lw  s7, GCR_CL_COHERENCE_OFS(v1)
0126     bnez    s7, 1f
0127      nop
0128
0129     /* Initialize the L1 caches */
0130     jal mips_cps_cache_init
0131      nop
0132
0133     /* Enter the coherent domain */
0134     li  t0, 0xff
0135     sw  t0, GCR_CL_COHERENCE_OFS(v1)
0136     ehb
0137
0138     /* Set Kseg0 CCA to that in s0 */
0139 1:  mfc0    t0, CP0_CONFIG
0140     ori t0, 0x7
0141     xori    t0, 0x7
0142     or  t0, t0, s0
0143     mtc0    t0, CP0_CONFIG
0144     ehb
0145
0146     /* Jump to kseg0 */
0147     PTR_LA  t0, 1f
0148     jr  t0
0149      nop
0150
0151     /*
0152      * We're up, cached & coherent. Perform any EVA initialization necessary
0153      * before we access memory.
0154      */
0155 1:  eva_init
0156
0157     /* Retrieve boot configuration pointers */
0158     jal mips_cps_get_bootcfg
0159      nop
0160
0161     /* Skip core-level init if we started up coherent */
0162     bnez    s7, 1f
0163      nop
0164
0165     /* Perform any further required core-level initialisation */
0166     jal mips_cps_core_init
0167      nop
0168
0169     /*
0170      * Boot any other VPEs within this core that should be online, and
0171      * deactivate this VPE if it should be offline.
0172      */
0173     move    a1, t9
0174     jal mips_cps_boot_vpes
0175      move   a0, v0
0176
0177     /* Off we go! */
0178 1:  PTR_L   t1, VPEBOOTCFG_PC(v1)
0179     PTR_L   gp, VPEBOOTCFG_GP(v1)
0180     PTR_L   sp, VPEBOOTCFG_SP(v1)
0181     jr  t1
0182      nop
0183     END(mips_cps_core_entry)
0184
0185 .org 0x200
0186 LEAF(excep_tlbfill)
0187     DUMP_EXCEP("TLB Fill")
0188     b   .
0189      nop
0190     END(excep_tlbfill)
0191
0192 .org 0x280
0193 LEAF(excep_xtlbfill)
0194     DUMP_EXCEP("XTLB Fill")
0195     b   .
0196      nop
0197     END(excep_xtlbfill)
0198
0199 .org 0x300
0200 LEAF(excep_cache)
0201     DUMP_EXCEP("Cache")
0202     b   .
0203      nop
0204     END(excep_cache)
0205
0206 .org 0x380
0207 LEAF(excep_genex)
0208     DUMP_EXCEP("General")
0209     b   .
0210      nop
0211     END(excep_genex)
0212
0213 .org 0x400
0214 LEAF(excep_intex)
0215     DUMP_EXCEP("Interrupt")
0216     b   .
0217      nop
0218     END(excep_intex)
0219
0220 .org 0x480
0221 LEAF(excep_ejtag)
0222     PTR_LA  k0, ejtag_debug_handler
0223     jr  k0
0224      nop
0225     END(excep_ejtag)
0226
0227 LEAF(mips_cps_core_init)
0228 #ifdef CONFIG_MIPS_MT_SMP
0229     /* Check that the core implements the MT ASE */
0230     has_mt  t0, 3f
0231
0232     .set    push
0233     .set    MIPS_ISA_LEVEL_RAW
0234     .set    mt
0235
0236     /* Only allow 1 TC per VPE to execute... */
0237     dmt
0238
0239     /* ...and for the moment only 1 VPE */
0240     dvpe
0241     PTR_LA  t1, 1f
0242     jr.hb   t1
0243      nop
0244
0245     /* Enter VPE configuration state */
0246 1:  mfc0    t0, CP0_MVPCONTROL
0247     ori t0, t0, MVPCONTROL_VPC
0248     mtc0    t0, CP0_MVPCONTROL
0249
0250     /* Retrieve the number of VPEs within the core */
0251     mfc0    t0, CP0_MVPCONF0
0252     srl t0, t0, MVPCONF0_PVPE_SHIFT
0253     andi    t0, t0, (MVPCONF0_PVPE >> MVPCONF0_PVPE_SHIFT)
0254     addiu   ta3, t0, 1
0255
0256     /* If there's only 1, we're done */
0257     beqz    t0, 2f
0258      nop
0259
0260     /* Loop through each VPE within this core */
0261     li  ta1, 1
0262
0263 1:  /* Operate on the appropriate TC */
0264     mtc0    ta1, CP0_VPECONTROL
0265     ehb
0266
0267     /* Bind TC to VPE (1:1 TC:VPE mapping) */
0268     mttc0   ta1, CP0_TCBIND
0269
0270     /* Set exclusive TC, non-active, master */
0271     li  t0, VPECONF0_MVP
0272     sll t1, ta1, VPECONF0_XTC_SHIFT
0273     or  t0, t0, t1
0274     mttc0   t0, CP0_VPECONF0
0275
0276     /* Set TC non-active, non-allocatable */
0277     mttc0   zero, CP0_TCSTATUS
0278
0279     /* Set TC halted */
0280     li  t0, TCHALT_H
0281     mttc0   t0, CP0_TCHALT
0282
0283     /* Next VPE */
0284     addiu   ta1, ta1, 1
0285     slt t0, ta1, ta3
0286     bnez    t0, 1b
0287      nop
0288
0289     /* Leave VPE configuration state */
0290 2:  mfc0    t0, CP0_MVPCONTROL
0291     xori    t0, t0, MVPCONTROL_VPC
0292     mtc0    t0, CP0_MVPCONTROL
0293
0294 3:  .set    pop
0295 #endif
0296     jr  ra
0297      nop
0298     END(mips_cps_core_init)
0299
0300 /**
0301  * mips_cps_get_bootcfg() - retrieve boot configuration pointers
0302  *
0303  * Returns: pointer to struct core_boot_config in v0, pointer to
0304  *          struct vpe_boot_config in v1, VPE ID in t9
0305  */
0306 LEAF(mips_cps_get_bootcfg)
0307     /* Calculate a pointer to this cores struct core_boot_config */
0308     cmgcrb  t0
0309     lw  t0, GCR_CL_ID_OFS(t0)
0310     li  t1, COREBOOTCFG_SIZE
0311     mul t0, t0, t1
0312     PTR_LA  t1, mips_cps_core_bootcfg
0313     PTR_L   t1, 0(t1)
0314     PTR_ADDU v0, t0, t1
0315
0316     /* Calculate this VPEs ID. If the core doesn't support MT use 0 */
0317     li  t9, 0
0318 #if defined(CONFIG_CPU_MIPSR6)
0319     has_vp  ta2, 1f
0320
0321     /*
0322      * Assume non-contiguous numbering. Perhaps some day we'll need
0323      * to handle contiguous VP numbering, but no such systems yet
0324      * exist.
0325      */
0326     mfc0    t9, CP0_GLOBALNUMBER
0327     andi    t9, t9, MIPS_GLOBALNUMBER_VP
0328 #elif defined(CONFIG_MIPS_MT_SMP)
0329     has_mt  ta2, 1f
0330
0331     /* Find the number of VPEs present in the core */
0332     mfc0    t1, CP0_MVPCONF0
0333     srl t1, t1, MVPCONF0_PVPE_SHIFT
0334     andi    t1, t1, MVPCONF0_PVPE >> MVPCONF0_PVPE_SHIFT
0335     addiu   t1, t1, 1
0336
0337     /* Calculate a mask for the VPE ID from EBase.CPUNum */
0338     clz t1, t1
0339     li  t2, 31
0340     subu    t1, t2, t1
0341     li  t2, 1
0342     sll t1, t2, t1
0343     addiu   t1, t1, -1
0344
0345     /* Retrieve the VPE ID from EBase.CPUNum */
0346     mfc0    t9, $15, 1
0347     and t9, t9, t1
0348 #endif
0349
0350 1:  /* Calculate a pointer to this VPEs struct vpe_boot_config */
0351     li  t1, VPEBOOTCFG_SIZE
0352     mul v1, t9, t1
0353     PTR_L   ta3, COREBOOTCFG_VPECONFIG(v0)
0354     PTR_ADDU v1, v1, ta3
0355
0356     jr  ra
0357      nop
0358     END(mips_cps_get_bootcfg)
0359
0360 LEAF(mips_cps_boot_vpes)
0361     lw  ta2, COREBOOTCFG_VPEMASK(a0)
0362     PTR_L   ta3, COREBOOTCFG_VPECONFIG(a0)
0363
0364 #if defined(CONFIG_CPU_MIPSR6)
0365
0366     has_vp  t0, 5f
0367
0368     /* Find base address of CPC */
0369     cmgcrb  t3
0370     PTR_L   t1, GCR_CPC_BASE_OFS(t3)
0371     PTR_LI  t2, ~0x7fff
0372     and t1, t1, t2
0373     PTR_LI  t2, UNCAC_BASE
0374     PTR_ADD t1, t1, t2
0375
0376     /* Start any other VPs that ought to be running */
0377     PTR_S   ta2, CPC_CL_VC_RUN_OFS(t1)
0378
0379     /* Ensure this VP stops running if it shouldn't be */
0380     not ta2
0381     PTR_S   ta2, CPC_CL_VC_STOP_OFS(t1)
0382     ehb
0383
0384 #elif defined(CONFIG_MIPS_MT)
0385
0386     /* If the core doesn't support MT then return */
0387     has_mt  t0, 5f
0388
0389     /* Enter VPE configuration state */
0390     .set    push
0391     .set    MIPS_ISA_LEVEL_RAW
0392     .set    mt
0393     dvpe
0394     .set    pop
0395
0396     PTR_LA  t1, 1f
0397     jr.hb   t1
0398      nop
0399 1:  mfc0    t1, CP0_MVPCONTROL
0400     ori t1, t1, MVPCONTROL_VPC
0401     mtc0    t1, CP0_MVPCONTROL
0402     ehb
0403
0404     /* Loop through each VPE */
0405     move    t8, ta2
0406     li  ta1, 0
0407
0408     /* Check whether the VPE should be running. If not, skip it */
0409 1:  andi    t0, ta2, 1
0410     beqz    t0, 2f
0411      nop
0412
0413     /* Operate on the appropriate TC */
0414     mfc0    t0, CP0_VPECONTROL
0415     ori t0, t0, VPECONTROL_TARGTC
0416     xori    t0, t0, VPECONTROL_TARGTC
0417     or  t0, t0, ta1
0418     mtc0    t0, CP0_VPECONTROL
0419     ehb
0420
0421     .set    push
0422     .set    MIPS_ISA_LEVEL_RAW
0423     .set    mt
0424
0425     /* Skip the VPE if its TC is not halted */
0426     mftc0   t0, CP0_TCHALT
0427     beqz    t0, 2f
0428      nop
0429
0430     /* Calculate a pointer to the VPEs struct vpe_boot_config */
0431     li  t0, VPEBOOTCFG_SIZE
0432     mul t0, t0, ta1
0433     addu    t0, t0, ta3
0434
0435     /* Set the TC restart PC */
0436     lw  t1, VPEBOOTCFG_PC(t0)
0437     mttc0   t1, CP0_TCRESTART
0438
0439     /* Set the TC stack pointer */
0440     lw  t1, VPEBOOTCFG_SP(t0)
0441     mttgpr  t1, sp
0442
0443     /* Set the TC global pointer */
0444     lw  t1, VPEBOOTCFG_GP(t0)
0445     mttgpr  t1, gp
0446
0447     /* Copy config from this VPE */
0448     mfc0    t0, CP0_CONFIG
0449     mttc0   t0, CP0_CONFIG
0450
0451     /*
0452      * Copy the EVA config from this VPE if the CPU supports it.
0453      * CONFIG3 must exist to be running MT startup - just read it.
0454      */
0455     mfc0    t0, CP0_CONFIG, 3
0456     and t0, t0, MIPS_CONF3_SC
0457     beqz    t0, 3f
0458      nop
0459     mfc0    t0, CP0_SEGCTL0
0460     mttc0   t0, CP0_SEGCTL0
0461     mfc0    t0, CP0_SEGCTL1
0462     mttc0   t0, CP0_SEGCTL1
0463     mfc0    t0, CP0_SEGCTL2
0464     mttc0   t0, CP0_SEGCTL2
0465 3:
0466     /* Ensure no software interrupts are pending */
0467     mttc0   zero, CP0_CAUSE
0468     mttc0   zero, CP0_STATUS
0469
0470     /* Set TC active, not interrupt exempt */
0471     mftc0   t0, CP0_TCSTATUS
0472     li  t1, ~TCSTATUS_IXMT
0473     and t0, t0, t1
0474     ori t0, t0, TCSTATUS_A
0475     mttc0   t0, CP0_TCSTATUS
0476
0477     /* Clear the TC halt bit */
0478     mttc0   zero, CP0_TCHALT
0479
0480     /* Set VPE active */
0481     mftc0   t0, CP0_VPECONF0
0482     ori t0, t0, VPECONF0_VPA
0483     mttc0   t0, CP0_VPECONF0
0484
0485     /* Next VPE */
0486 2:  srl ta2, ta2, 1
0487     addiu   ta1, ta1, 1
0488     bnez    ta2, 1b
0489      nop
0490
0491     /* Leave VPE configuration state */
0492     mfc0    t1, CP0_MVPCONTROL
0493     xori    t1, t1, MVPCONTROL_VPC
0494     mtc0    t1, CP0_MVPCONTROL
0495     ehb
0496     evpe
0497
0498     .set    pop
0499
0500     /* Check whether this VPE is meant to be running */
0501     li  t0, 1
0502     sll t0, t0, a1
0503     and t0, t0, t8
0504     bnez    t0, 2f
0505      nop
0506
0507     /* This VPE should be offline, halt the TC */
0508     li  t0, TCHALT_H
0509     mtc0    t0, CP0_TCHALT
0510     PTR_LA  t0, 1f
0511 1:  jr.hb   t0
0512      nop
0513
0514 2:
0515
0516 #endif /* CONFIG_MIPS_MT_SMP */
0517
0518     /* Return */
0519 5:  jr  ra
0520      nop
0521     END(mips_cps_boot_vpes)
0522
0523 LEAF(mips_cps_cache_init)
0524     /*
0525      * Clear the bits used to index the caches. Note that the architecture
0526      * dictates that writing to any of TagLo or TagHi selects 0 or 2 should
0527      * be valid for all MIPS32 CPUs, even those for which said writes are
0528      * unnecessary.
0529      */
0530     mtc0    zero, CP0_TAGLO, 0
0531     mtc0    zero, CP0_TAGHI, 0
0532     mtc0    zero, CP0_TAGLO, 2
0533     mtc0    zero, CP0_TAGHI, 2
0534     ehb
0535
0536     /* Primary cache configuration is indicated by Config1 */
0537     mfc0    v0, CP0_CONFIG, 1
0538
0539     /* Detect I-cache line size */
0540     _EXT    t0, v0, MIPS_CONF1_IL_SHF, MIPS_CONF1_IL_SZ
0541     beqz    t0, icache_done
0542      li t1, 2
0543     sllv    t0, t1, t0
0544
0545     /* Detect I-cache size */
0546     _EXT    t1, v0, MIPS_CONF1_IS_SHF, MIPS_CONF1_IS_SZ
0547     xori    t2, t1, 0x7
0548     beqz    t2, 1f
0549      li t3, 32
0550     addiu   t1, t1, 1
0551     sllv    t1, t3, t1
0552 1:  /* At this point t1 == I-cache sets per way */
0553     _EXT    t2, v0, MIPS_CONF1_IA_SHF, MIPS_CONF1_IA_SZ
0554     addiu   t2, t2, 1
0555     mul t1, t1, t0
0556     mul t1, t1, t2
0557
0558     li  a0, CKSEG0
0559     PTR_ADD a1, a0, t1
0560 1:  cache   Index_Store_Tag_I, 0(a0)
0561     PTR_ADD a0, a0, t0
0562     bne a0, a1, 1b
0563      nop
0564 icache_done:
0565
0566     /* Detect D-cache line size */
0567     _EXT    t0, v0, MIPS_CONF1_DL_SHF, MIPS_CONF1_DL_SZ
0568     beqz    t0, dcache_done
0569      li t1, 2
0570     sllv    t0, t1, t0
0571
0572     /* Detect D-cache size */
0573     _EXT    t1, v0, MIPS_CONF1_DS_SHF, MIPS_CONF1_DS_SZ
0574     xori    t2, t1, 0x7
0575     beqz    t2, 1f
0576      li t3, 32
0577     addiu   t1, t1, 1
0578     sllv    t1, t3, t1
0579 1:  /* At this point t1 == D-cache sets per way */
0580     _EXT    t2, v0, MIPS_CONF1_DA_SHF, MIPS_CONF1_DA_SZ
0581     addiu   t2, t2, 1
0582     mul t1, t1, t0
0583     mul t1, t1, t2
0584
0585     li  a0, CKSEG0
0586     PTR_ADDU a1, a0, t1
0587     PTR_SUBU a1, a1, t0
0588 1:  cache   Index_Store_Tag_D, 0(a0)
0589     bne a0, a1, 1b
0590      PTR_ADD a0, a0, t0
0591 dcache_done:
0592
0593     jr  ra
0594      nop
0595     END(mips_cps_cache_init)
0596
0597 #if defined(CONFIG_MIPS_CPS_PM) && defined(CONFIG_CPU_PM)
0598
0599     /* Calculate a pointer to this CPUs struct mips_static_suspend_state */
0600     .macro  psstate dest
0601     .set    push
0602     .set    noat
0603     lw  $1, TI_CPU(gp)
0604     sll $1, $1, LONGLOG
0605     PTR_LA  \dest, __per_cpu_offset
0606     addu    $1, $1, \dest
0607     lw  $1, 0($1)
0608     PTR_LA  \dest, cps_cpu_state
0609     addu    \dest, \dest, $1
0610     .set    pop
0611     .endm
0612
0613 LEAF(mips_cps_pm_save)
0614     /* Save CPU state */
0615     SUSPEND_SAVE_REGS
0616     psstate t1
0617     SUSPEND_SAVE_STATIC
0618     jr  v0
0619      nop
0620     END(mips_cps_pm_save)
0621
0622 LEAF(mips_cps_pm_restore)
0623     /* Restore CPU state */
0624     psstate t1
0625     RESUME_RESTORE_STATIC
0626     RESUME_RESTORE_REGS_RETURN
0627     END(mips_cps_pm_restore)
0628
0629 #endif /* CONFIG_MIPS_CPS_PM && CONFIG_CPU_PM */