arm/common/mcpm_head.S

0001 /* SPDX-License-Identifier: GPL-2.0-only */
0002 /*
0003  * arch/arm/common/mcpm_head.S -- kernel entry point for multi-cluster PM
0004  *
0005  * Created by:  Nicolas Pitre, March 2012
0006  * Copyright:   (C) 2012-2013  Linaro Limited
0007  *
0008  * Refer to Documentation/arm/cluster-pm-race-avoidance.rst
0009  * for details of the synchronisation algorithms used here.
0010  */
0011
0012 #include <linux/linkage.h>
0013 #include <asm/mcpm.h>
0014 #include <asm/assembler.h>
0015
0016 #include "vlock.h"
0017
0018 .if MCPM_SYNC_CLUSTER_CPUS
0019 .error "cpus must be the first member of struct mcpm_sync_struct"
0020 .endif
0021
0022     .macro  pr_dbg  string
0023 #if defined(CONFIG_DEBUG_LL) && defined(DEBUG)
0024     b   1901f
0025 1902:   .asciz  "CPU"
0026 1903:   .asciz  " cluster"
0027 1904:   .asciz  ": \string"
0028     .align
0029 1901:   adr r0, 1902b
0030     bl  printascii
0031     mov r0, r9
0032     bl  printhex2
0033     adr r0, 1903b
0034     bl  printascii
0035     mov r0, r10
0036     bl  printhex2
0037     adr r0, 1904b
0038     bl  printascii
0039 #endif
0040     .endm
0041
0042     .arm
0043     .align
0044
0045 ENTRY(mcpm_entry_point)
0046
0047  ARM_BE8(setend        be)
0048  THUMB( badr    r12, 1f     )
0049  THUMB( bx  r12     )
0050  THUMB( .thumb          )
0051 1:
0052     mrc p15, 0, r0, c0, c0, 5       @ MPIDR
0053     ubfx    r9, r0, #0, #8          @ r9 = cpu
0054     ubfx    r10, r0, #8, #8         @ r10 = cluster
0055     mov r3, #MAX_CPUS_PER_CLUSTER
0056     mla r4, r3, r10, r9         @ r4 = canonical CPU index
0057     cmp r4, #(MAX_CPUS_PER_CLUSTER * MAX_NR_CLUSTERS)
0058     blo 2f
0059
0060     /* We didn't expect this CPU.  Try to cheaply make it quiet. */
0061 1:  wfi
0062     wfe
0063     b   1b
0064
0065 2:  pr_dbg  "kernel mcpm_entry_point\n"
0066
0067     /*
0068      * MMU is off so we need to get to various variables in a
0069      * position independent way.
0070      */
0071     adr r5, 3f
0072     ldmia   r5, {r0, r6, r7, r8, r11}
0073     add r0, r5, r0          @ r0 = mcpm_entry_early_pokes
0074     add r6, r5, r6          @ r6 = mcpm_entry_vectors
0075     ldr r7, [r5, r7]            @ r7 = mcpm_power_up_setup_phys
0076     add r8, r5, r8          @ r8 = mcpm_sync
0077     add r11, r5, r11            @ r11 = first_man_locks
0078
0079     @ Perform an early poke, if any
0080     add r0, r0, r4, lsl #3
0081     ldmia   r0, {r0, r1}
0082     teq r0, #0
0083     strne   r1, [r0]
0084
0085     mov r0, #MCPM_SYNC_CLUSTER_SIZE
0086     mla r8, r0, r10, r8         @ r8 = sync cluster base
0087
0088     @ Signal that this CPU is coming UP:
0089     mov r0, #CPU_COMING_UP
0090     mov r5, #MCPM_SYNC_CPU_SIZE
0091     mla r5, r9, r5, r8          @ r5 = sync cpu address
0092     strb    r0, [r5]
0093
0094     @ At this point, the cluster cannot unexpectedly enter the GOING_DOWN
0095     @ state, because there is at least one active CPU (this CPU).
0096
0097     mov r0, #VLOCK_SIZE
0098     mla r11, r0, r10, r11       @ r11 = cluster first man lock
0099     mov r0, r11
0100     mov r1, r9              @ cpu
0101     bl  vlock_trylock           @ implies DMB
0102
0103     cmp r0, #0              @ failed to get the lock?
0104     bne mcpm_setup_wait     @ wait for cluster setup if so
0105
0106     ldrb    r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER]
0107     cmp r0, #CLUSTER_UP         @ cluster already up?
0108     bne mcpm_setup          @ if not, set up the cluster
0109
0110     @ Otherwise, release the first man lock and skip setup:
0111     mov r0, r11
0112     bl  vlock_unlock
0113     b   mcpm_setup_complete
0114
0115 mcpm_setup:
0116     @ Control dependency implies strb not observable before previous ldrb.
0117
0118     @ Signal that the cluster is being brought up:
0119     mov r0, #INBOUND_COMING_UP
0120     strb    r0, [r8, #MCPM_SYNC_CLUSTER_INBOUND]
0121     dmb
0122
0123     @ Any CPU trying to take the cluster into CLUSTER_GOING_DOWN from this
0124     @ point onwards will observe INBOUND_COMING_UP and abort.
0125
0126     @ Wait for any previously-pending cluster teardown operations to abort
0127     @ or complete:
0128 mcpm_teardown_wait:
0129     ldrb    r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER]
0130     cmp r0, #CLUSTER_GOING_DOWN
0131     bne first_man_setup
0132     wfe
0133     b   mcpm_teardown_wait
0134
0135 first_man_setup:
0136     dmb
0137
0138     @ If the outbound gave up before teardown started, skip cluster setup:
0139
0140     cmp r0, #CLUSTER_UP
0141     beq mcpm_setup_leave
0142
0143     @ power_up_setup is now responsible for setting up the cluster:
0144
0145     cmp r7, #0
0146     mov r0, #1      @ second (cluster) affinity level
0147     blxne   r7      @ Call power_up_setup if defined
0148     dmb
0149
0150     mov r0, #CLUSTER_UP
0151     strb    r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER]
0152     dmb
0153
0154 mcpm_setup_leave:
0155     @ Leave the cluster setup critical section:
0156
0157     mov r0, #INBOUND_NOT_COMING_UP
0158     strb    r0, [r8, #MCPM_SYNC_CLUSTER_INBOUND]
0159     dsb st
0160     sev
0161
0162     mov r0, r11
0163     bl  vlock_unlock    @ implies DMB
0164     b   mcpm_setup_complete
0165
0166     @ In the contended case, non-first men wait here for cluster setup
0167     @ to complete:
0168 mcpm_setup_wait:
0169     ldrb    r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER]
0170     cmp r0, #CLUSTER_UP
0171     wfene
0172     bne mcpm_setup_wait
0173     dmb
0174
0175 mcpm_setup_complete:
0176     @ If a platform-specific CPU setup hook is needed, it is
0177     @ called from here.
0178
0179     cmp r7, #0
0180     mov r0, #0      @ first (CPU) affinity level
0181     blxne   r7      @ Call power_up_setup if defined
0182     dmb
0183
0184     @ Mark the CPU as up:
0185
0186     mov r0, #CPU_UP
0187     strb    r0, [r5]
0188
0189     @ Observability order of CPU_UP and opening of the gate does not matter.
0190
0191 mcpm_entry_gated:
0192     ldr r5, [r6, r4, lsl #2]        @ r5 = CPU entry vector
0193     cmp r5, #0
0194     wfeeq
0195     beq mcpm_entry_gated
0196     dmb
0197
0198     pr_dbg  "released\n"
0199     bx  r5
0200
0201     .align  2
0202
0203 3:  .word   mcpm_entry_early_pokes - .
0204     .word   mcpm_entry_vectors - 3b
0205     .word   mcpm_power_up_setup_phys - 3b
0206     .word   mcpm_sync - 3b
0207     .word   first_man_locks - 3b
0208
0209 ENDPROC(mcpm_entry_point)
0210
0211     .bss
0212
0213     .align  CACHE_WRITEBACK_ORDER
0214     .type   first_man_locks, #object
0215 first_man_locks:
0216     .space  VLOCK_SIZE * MAX_NR_CLUSTERS
0217     .align  CACHE_WRITEBACK_ORDER
0218
0219     .type   mcpm_entry_vectors, #object
0220 ENTRY(mcpm_entry_vectors)
0221     .space  4 * MAX_NR_CLUSTERS * MAX_CPUS_PER_CLUSTER
0222
0223     .type   mcpm_entry_early_pokes, #object
0224 ENTRY(mcpm_entry_early_pokes)
0225     .space  8 * MAX_NR_CLUSTERS * MAX_CPUS_PER_CLUSTER
0226
0227     .type   mcpm_power_up_setup_phys, #object
0228 ENTRY(mcpm_power_up_setup_phys)
0229     .space  4       @ set by mcpm_sync_init()