platforms/powernv/subcore.c

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  * Copyright 2013, Michael (Ellerman|Neuling), IBM Corporation.
0004  */
0005
0006 #define pr_fmt(fmt) "powernv: " fmt
0007
0008 #include <linux/kernel.h>
0009 #include <linux/cpu.h>
0010 #include <linux/cpumask.h>
0011 #include <linux/device.h>
0012 #include <linux/gfp.h>
0013 #include <linux/smp.h>
0014 #include <linux/stop_machine.h>
0015
0016 #include <asm/cputhreads.h>
0017 #include <asm/cpuidle.h>
0018 #include <asm/kvm_ppc.h>
0019 #include <asm/machdep.h>
0020 #include <asm/opal.h>
0021 #include <asm/smp.h>
0022
0023 #include "subcore.h"
0024 #include "powernv.h"
0025
0026
0027 /*
0028  * Split/unsplit procedure:
0029  *
0030  * A core can be in one of three states, unsplit, 2-way split, and 4-way split.
0031  *
0032  * The mapping to subcores_per_core is simple:
0033  *
0034  *  State       | subcores_per_core
0035  *  ------------|------------------
0036  *  Unsplit     |        1
0037  *  2-way split |        2
0038  *  4-way split |        4
0039  *
0040  * The core is split along thread boundaries, the mapping between subcores and
0041  * threads is as follows:
0042  *
0043  *  Unsplit:
0044  *          ----------------------------
0045  *  Subcore |            0             |
0046  *          ----------------------------
0047  *  Thread  |  0  1  2  3  4  5  6  7  |
0048  *          ----------------------------
0049  *
0050  *  2-way split:
0051  *          -------------------------------------
0052  *  Subcore |        0        |        1        |
0053  *          -------------------------------------
0054  *  Thread  |  0   1   2   3  |  4   5   6   7  |
0055  *          -------------------------------------
0056  *
0057  *  4-way split:
0058  *          -----------------------------------------
0059  *  Subcore |    0    |    1    |    2    |    3    |
0060  *          -----------------------------------------
0061  *  Thread  |  0   1  |  2   3  |  4   5  |  6   7  |
0062  *          -----------------------------------------
0063  *
0064  *
0065  * Transitions
0066  * -----------
0067  *
0068  * It is not possible to transition between either of the split states, the
0069  * core must first be unsplit. The legal transitions are:
0070  *
0071  *  -----------          ---------------
0072  *  |         |  <---->  | 2-way split |
0073  *  |         |          ---------------
0074  *  | Unsplit |
0075  *  |         |          ---------------
0076  *  |         |  <---->  | 4-way split |
0077  *  -----------          ---------------
0078  *
0079  * Unsplitting
0080  * -----------
0081  *
0082  * Unsplitting is the simpler procedure. It requires thread 0 to request the
0083  * unsplit while all other threads NAP.
0084  *
0085  * Thread 0 clears HID0_POWER8_DYNLPARDIS (Dynamic LPAR Disable). This tells
0086  * the hardware that if all threads except 0 are napping, the hardware should
0087  * unsplit the core.
0088  *
0089  * Non-zero threads are sent to a NAP loop, they don't exit the loop until they
0090  * see the core unsplit.
0091  *
0092  * Core 0 spins waiting for the hardware to see all the other threads napping
0093  * and perform the unsplit.
0094  *
0095  * Once thread 0 sees the unsplit, it IPIs the secondary threads to wake them
0096  * out of NAP. They will then see the core unsplit and exit the NAP loop.
0097  *
0098  * Splitting
0099  * ---------
0100  *
0101  * The basic splitting procedure is fairly straight forward. However it is
0102  * complicated by the fact that after the split occurs, the newly created
0103  * subcores are not in a fully initialised state.
0104  *
0105  * Most notably the subcores do not have the correct value for SDR1, which
0106  * means they must not be running in virtual mode when the split occurs. The
0107  * subcores have separate timebases SPRs but these are pre-synchronised by
0108  * opal.
0109  *
0110  * To begin with secondary threads are sent to an assembly routine. There they
0111  * switch to real mode, so they are immune to the uninitialised SDR1 value.
0112  * Once in real mode they indicate that they are in real mode, and spin waiting
0113  * to see the core split.
0114  *
0115  * Thread 0 waits to see that all secondaries are in real mode, and then begins
0116  * the splitting procedure. It firstly sets HID0_POWER8_DYNLPARDIS, which
0117  * prevents the hardware from unsplitting. Then it sets the appropriate HID bit
0118  * to request the split, and spins waiting to see that the split has happened.
0119  *
0120  * Concurrently the secondaries will notice the split. When they do they set up
0121  * their SPRs, notably SDR1, and then they can return to virtual mode and exit
0122  * the procedure.
0123  */
0124
0125 /* Initialised at boot by subcore_init() */
0126 static int subcores_per_core;
0127
0128 /*
0129  * Used to communicate to offline cpus that we want them to pop out of the
0130  * offline loop and do a split or unsplit.
0131  *
0132  * 0 - no split happening
0133  * 1 - unsplit in progress
0134  * 2 - split to 2 in progress
0135  * 4 - split to 4 in progress
0136  */
0137 static int new_split_mode;
0138
0139 static cpumask_var_t cpu_offline_mask;
0140
0141 struct split_state {
0142     u8 step;
0143     u8 master;
0144 };
0145
0146 static DEFINE_PER_CPU(struct split_state, split_state);
0147
0148 static void wait_for_sync_step(int step)
0149 {
0150     int i, cpu = smp_processor_id();
0151
0152     for (i = cpu + 1; i < cpu + threads_per_core; i++)
0153         while(per_cpu(split_state, i).step < step)
0154             barrier();
0155
0156     /* Order the wait loop vs any subsequent loads/stores. */
0157     mb();
0158 }
0159
0160 static void update_hid_in_slw(u64 hid0)
0161 {
0162     u64 idle_states = pnv_get_supported_cpuidle_states();
0163
0164     if (idle_states & OPAL_PM_WINKLE_ENABLED) {
0165         /* OPAL call to patch slw with the new HID0 value */
0166         u64 cpu_pir = hard_smp_processor_id();
0167
0168         opal_slw_set_reg(cpu_pir, SPRN_HID0, hid0);
0169     }
0170 }
0171
0172 static inline void update_power8_hid0(unsigned long hid0)
0173 {
0174     /*
0175      *  The HID0 update on Power8 should at the very least be
0176      *  preceded by a SYNC instruction followed by an ISYNC
0177      *  instruction
0178      */
0179     asm volatile("sync; mtspr %0,%1; isync":: "i"(SPRN_HID0), "r"(hid0));
0180 }
0181
0182 static void unsplit_core(void)
0183 {
0184     u64 hid0, mask;
0185     int i, cpu;
0186
0187     mask = HID0_POWER8_2LPARMODE | HID0_POWER8_4LPARMODE;
0188
0189     cpu = smp_processor_id();
0190     if (cpu_thread_in_core(cpu) != 0) {
0191         while (mfspr(SPRN_HID0) & mask)
0192             power7_idle_type(PNV_THREAD_NAP);
0193
0194         per_cpu(split_state, cpu).step = SYNC_STEP_UNSPLIT;
0195         return;
0196     }
0197
0198     hid0 = mfspr(SPRN_HID0);
0199     hid0 &= ~HID0_POWER8_DYNLPARDIS;
0200     update_power8_hid0(hid0);
0201     update_hid_in_slw(hid0);
0202
0203     while (mfspr(SPRN_HID0) & mask)
0204         cpu_relax();
0205
0206     /* Wake secondaries out of NAP */
0207     for (i = cpu + 1; i < cpu + threads_per_core; i++)
0208         smp_send_reschedule(i);
0209
0210     wait_for_sync_step(SYNC_STEP_UNSPLIT);
0211 }
0212
0213 static void split_core(int new_mode)
0214 {
0215     struct {  u64 value; u64 mask; } split_parms[2] = {
0216         { HID0_POWER8_1TO2LPAR, HID0_POWER8_2LPARMODE },
0217         { HID0_POWER8_1TO4LPAR, HID0_POWER8_4LPARMODE }
0218     };
0219     int i, cpu;
0220     u64 hid0;
0221
0222     /* Convert new_mode (2 or 4) into an index into our parms array */
0223     i = (new_mode >> 1) - 1;
0224     BUG_ON(i < 0 || i > 1);
0225
0226     cpu = smp_processor_id();
0227     if (cpu_thread_in_core(cpu) != 0) {
0228         split_core_secondary_loop(&per_cpu(split_state, cpu).step);
0229         return;
0230     }
0231
0232     wait_for_sync_step(SYNC_STEP_REAL_MODE);
0233
0234     /* Write new mode */
0235     hid0  = mfspr(SPRN_HID0);
0236     hid0 |= HID0_POWER8_DYNLPARDIS | split_parms[i].value;
0237     update_power8_hid0(hid0);
0238     update_hid_in_slw(hid0);
0239
0240     /* Wait for it to happen */
0241     while (!(mfspr(SPRN_HID0) & split_parms[i].mask))
0242         cpu_relax();
0243 }
0244
0245 static void cpu_do_split(int new_mode)
0246 {
0247     /*
0248      * At boot subcores_per_core will be 0, so we will always unsplit at
0249      * boot. In the usual case where the core is already unsplit it's a
0250      * nop, and this just ensures the kernel's notion of the mode is
0251      * consistent with the hardware.
0252      */
0253     if (subcores_per_core != 1)
0254         unsplit_core();
0255
0256     if (new_mode != 1)
0257         split_core(new_mode);
0258
0259     mb();
0260     per_cpu(split_state, smp_processor_id()).step = SYNC_STEP_FINISHED;
0261 }
0262
0263 bool cpu_core_split_required(void)
0264 {
0265     smp_rmb();
0266
0267     if (!new_split_mode)
0268         return false;
0269
0270     cpu_do_split(new_split_mode);
0271
0272     return true;
0273 }
0274
0275 void update_subcore_sibling_mask(void)
0276 {
0277     int cpu;
0278     /*
0279      * sibling mask for the first cpu. Left shift this by required bits
0280      * to get sibling mask for the rest of the cpus.
0281      */
0282     int sibling_mask_first_cpu =  (1 << threads_per_subcore) - 1;
0283
0284     for_each_possible_cpu(cpu) {
0285         int tid = cpu_thread_in_core(cpu);
0286         int offset = (tid / threads_per_subcore) * threads_per_subcore;
0287         int mask = sibling_mask_first_cpu << offset;
0288
0289         paca_ptrs[cpu]->subcore_sibling_mask = mask;
0290
0291     }
0292 }
0293
0294 static int cpu_update_split_mode(void *data)
0295 {
0296     int cpu, new_mode = *(int *)data;
0297
0298     if (this_cpu_ptr(&split_state)->master) {
0299         new_split_mode = new_mode;
0300         smp_wmb();
0301
0302         cpumask_andnot(cpu_offline_mask, cpu_present_mask,
0303                    cpu_online_mask);
0304
0305         /* This should work even though the cpu is offline */
0306         for_each_cpu(cpu, cpu_offline_mask)
0307             smp_send_reschedule(cpu);
0308     }
0309
0310     cpu_do_split(new_mode);
0311
0312     if (this_cpu_ptr(&split_state)->master) {
0313         /* Wait for all cpus to finish before we touch subcores_per_core */
0314         for_each_present_cpu(cpu) {
0315             if (cpu >= setup_max_cpus)
0316                 break;
0317
0318             while(per_cpu(split_state, cpu).step < SYNC_STEP_FINISHED)
0319                 barrier();
0320         }
0321
0322         new_split_mode = 0;
0323
0324         /* Make the new mode public */
0325         subcores_per_core = new_mode;
0326         threads_per_subcore = threads_per_core / subcores_per_core;
0327         update_subcore_sibling_mask();
0328
0329         /* Make sure the new mode is written before we exit */
0330         mb();
0331     }
0332
0333     return 0;
0334 }
0335
0336 static int set_subcores_per_core(int new_mode)
0337 {
0338     struct split_state *state;
0339     int cpu;
0340
0341     if (kvm_hv_mode_active()) {
0342         pr_err("Unable to change split core mode while KVM active.\n");
0343         return -EBUSY;
0344     }
0345
0346     /*
0347      * We are only called at boot, or from the sysfs write. If that ever
0348      * changes we'll need a lock here.
0349      */
0350     BUG_ON(new_mode < 1 || new_mode > 4 || new_mode == 3);
0351
0352     for_each_present_cpu(cpu) {
0353         state = &per_cpu(split_state, cpu);
0354         state->step = SYNC_STEP_INITIAL;
0355         state->master = 0;
0356     }
0357
0358     cpus_read_lock();
0359
0360     /* This cpu will update the globals before exiting stop machine */
0361     this_cpu_ptr(&split_state)->master = 1;
0362
0363     /* Ensure state is consistent before we call the other cpus */
0364     mb();
0365
0366     stop_machine_cpuslocked(cpu_update_split_mode, &new_mode,
0367                 cpu_online_mask);
0368
0369     cpus_read_unlock();
0370
0371     return 0;
0372 }
0373
0374 static ssize_t __used store_subcores_per_core(struct device *dev,
0375         struct device_attribute *attr, const char *buf,
0376         size_t count)
0377 {
0378     unsigned long val;
0379     int rc;
0380
0381     /* We are serialised by the attribute lock */
0382
0383     rc = sscanf(buf, "%lx", &val);
0384     if (rc != 1)
0385         return -EINVAL;
0386
0387     switch (val) {
0388     case 1:
0389     case 2:
0390     case 4:
0391         if (subcores_per_core == val)
0392             /* Nothing to do */
0393             goto out;
0394         break;
0395     default:
0396         return -EINVAL;
0397     }
0398
0399     rc = set_subcores_per_core(val);
0400     if (rc)
0401         return rc;
0402
0403 out:
0404     return count;
0405 }
0406
0407 static ssize_t show_subcores_per_core(struct device *dev,
0408         struct device_attribute *attr, char *buf)
0409 {
0410     return sprintf(buf, "%x\n", subcores_per_core);
0411 }
0412
0413 static DEVICE_ATTR(subcores_per_core, 0644,
0414         show_subcores_per_core, store_subcores_per_core);
0415
0416 static int subcore_init(void)
0417 {
0418     unsigned pvr_ver;
0419
0420     pvr_ver = PVR_VER(mfspr(SPRN_PVR));
0421
0422     if (pvr_ver != PVR_POWER8 &&
0423         pvr_ver != PVR_POWER8E &&
0424         pvr_ver != PVR_POWER8NVL)
0425         return 0;
0426
0427     /*
0428      * We need all threads in a core to be present to split/unsplit so
0429          * continue only if max_cpus are aligned to threads_per_core.
0430      */
0431     if (setup_max_cpus % threads_per_core)
0432         return 0;
0433
0434     BUG_ON(!alloc_cpumask_var(&cpu_offline_mask, GFP_KERNEL));
0435
0436     set_subcores_per_core(1);
0437
0438     return device_create_file(cpu_subsys.dev_root,
0439                   &dev_attr_subcores_per_core);
0440 }
0441 machine_device_initcall(powernv, subcore_init);