Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  *  SMP related functions
0004  *
0005  *    Copyright IBM Corp. 1999, 2012
0006  *    Author(s): Denis Joseph Barrow,
0007  *       Martin Schwidefsky <schwidefsky@de.ibm.com>,
0008  *
0009  *  based on other smp stuff by
0010  *    (c) 1995 Alan Cox, CymruNET Ltd  <alan@cymru.net>
0011  *    (c) 1998 Ingo Molnar
0012  *
0013  * The code outside of smp.c uses logical cpu numbers, only smp.c does
0014  * the translation of logical to physical cpu ids. All new code that
0015  * operates on physical cpu numbers needs to go into smp.c.
0016  */
0017 
0018 #define KMSG_COMPONENT "cpu"
0019 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
0020 
0021 #include <linux/workqueue.h>
0022 #include <linux/memblock.h>
0023 #include <linux/export.h>
0024 #include <linux/init.h>
0025 #include <linux/mm.h>
0026 #include <linux/err.h>
0027 #include <linux/spinlock.h>
0028 #include <linux/kernel_stat.h>
0029 #include <linux/delay.h>
0030 #include <linux/interrupt.h>
0031 #include <linux/irqflags.h>
0032 #include <linux/irq_work.h>
0033 #include <linux/cpu.h>
0034 #include <linux/slab.h>
0035 #include <linux/sched/hotplug.h>
0036 #include <linux/sched/task_stack.h>
0037 #include <linux/crash_dump.h>
0038 #include <linux/kprobes.h>
0039 #include <asm/asm-offsets.h>
0040 #include <asm/diag.h>
0041 #include <asm/switch_to.h>
0042 #include <asm/facility.h>
0043 #include <asm/ipl.h>
0044 #include <asm/setup.h>
0045 #include <asm/irq.h>
0046 #include <asm/tlbflush.h>
0047 #include <asm/vtimer.h>
0048 #include <asm/lowcore.h>
0049 #include <asm/sclp.h>
0050 #include <asm/debug.h>
0051 #include <asm/os_info.h>
0052 #include <asm/sigp.h>
0053 #include <asm/idle.h>
0054 #include <asm/nmi.h>
0055 #include <asm/stacktrace.h>
0056 #include <asm/topology.h>
0057 #include <asm/vdso.h>
0058 #include "entry.h"
0059 
0060 enum {
0061     ec_schedule = 0,
0062     ec_call_function_single,
0063     ec_stop_cpu,
0064     ec_mcck_pending,
0065     ec_irq_work,
0066 };
0067 
0068 enum {
0069     CPU_STATE_STANDBY,
0070     CPU_STATE_CONFIGURED,
0071 };
0072 
0073 static DEFINE_PER_CPU(struct cpu *, cpu_device);
0074 
0075 struct pcpu {
0076     unsigned long ec_mask;      /* bit mask for ec_xxx functions */
0077     unsigned long ec_clk;       /* sigp timestamp for ec_xxx */
0078     signed char state;      /* physical cpu state */
0079     signed char polarization;   /* physical polarization */
0080     u16 address;            /* physical cpu address */
0081 };
0082 
0083 static u8 boot_core_type;
0084 static struct pcpu pcpu_devices[NR_CPUS];
0085 
0086 unsigned int smp_cpu_mt_shift;
0087 EXPORT_SYMBOL(smp_cpu_mt_shift);
0088 
0089 unsigned int smp_cpu_mtid;
0090 EXPORT_SYMBOL(smp_cpu_mtid);
0091 
0092 #ifdef CONFIG_CRASH_DUMP
0093 __vector128 __initdata boot_cpu_vector_save_area[__NUM_VXRS];
0094 #endif
0095 
0096 static unsigned int smp_max_threads __initdata = -1U;
0097 cpumask_t cpu_setup_mask;
0098 
0099 static int __init early_nosmt(char *s)
0100 {
0101     smp_max_threads = 1;
0102     return 0;
0103 }
0104 early_param("nosmt", early_nosmt);
0105 
0106 static int __init early_smt(char *s)
0107 {
0108     get_option(&s, &smp_max_threads);
0109     return 0;
0110 }
0111 early_param("smt", early_smt);
0112 
0113 /*
0114  * The smp_cpu_state_mutex must be held when changing the state or polarization
0115  * member of a pcpu data structure within the pcpu_devices arreay.
0116  */
0117 DEFINE_MUTEX(smp_cpu_state_mutex);
0118 
0119 /*
0120  * Signal processor helper functions.
0121  */
0122 static inline int __pcpu_sigp_relax(u16 addr, u8 order, unsigned long parm)
0123 {
0124     int cc;
0125 
0126     while (1) {
0127         cc = __pcpu_sigp(addr, order, parm, NULL);
0128         if (cc != SIGP_CC_BUSY)
0129             return cc;
0130         cpu_relax();
0131     }
0132 }
0133 
0134 static int pcpu_sigp_retry(struct pcpu *pcpu, u8 order, u32 parm)
0135 {
0136     int cc, retry;
0137 
0138     for (retry = 0; ; retry++) {
0139         cc = __pcpu_sigp(pcpu->address, order, parm, NULL);
0140         if (cc != SIGP_CC_BUSY)
0141             break;
0142         if (retry >= 3)
0143             udelay(10);
0144     }
0145     return cc;
0146 }
0147 
0148 static inline int pcpu_stopped(struct pcpu *pcpu)
0149 {
0150     u32 status;
0151 
0152     if (__pcpu_sigp(pcpu->address, SIGP_SENSE,
0153             0, &status) != SIGP_CC_STATUS_STORED)
0154         return 0;
0155     return !!(status & (SIGP_STATUS_CHECK_STOP|SIGP_STATUS_STOPPED));
0156 }
0157 
0158 static inline int pcpu_running(struct pcpu *pcpu)
0159 {
0160     if (__pcpu_sigp(pcpu->address, SIGP_SENSE_RUNNING,
0161             0, NULL) != SIGP_CC_STATUS_STORED)
0162         return 1;
0163     /* Status stored condition code is equivalent to cpu not running. */
0164     return 0;
0165 }
0166 
0167 /*
0168  * Find struct pcpu by cpu address.
0169  */
0170 static struct pcpu *pcpu_find_address(const struct cpumask *mask, u16 address)
0171 {
0172     int cpu;
0173 
0174     for_each_cpu(cpu, mask)
0175         if (pcpu_devices[cpu].address == address)
0176             return pcpu_devices + cpu;
0177     return NULL;
0178 }
0179 
0180 static void pcpu_ec_call(struct pcpu *pcpu, int ec_bit)
0181 {
0182     int order;
0183 
0184     if (test_and_set_bit(ec_bit, &pcpu->ec_mask))
0185         return;
0186     order = pcpu_running(pcpu) ? SIGP_EXTERNAL_CALL : SIGP_EMERGENCY_SIGNAL;
0187     pcpu->ec_clk = get_tod_clock_fast();
0188     pcpu_sigp_retry(pcpu, order, 0);
0189 }
0190 
0191 static int pcpu_alloc_lowcore(struct pcpu *pcpu, int cpu)
0192 {
0193     unsigned long async_stack, nodat_stack, mcck_stack;
0194     struct lowcore *lc;
0195 
0196     lc = (struct lowcore *) __get_free_pages(GFP_KERNEL | GFP_DMA, LC_ORDER);
0197     nodat_stack = __get_free_pages(GFP_KERNEL, THREAD_SIZE_ORDER);
0198     async_stack = stack_alloc();
0199     mcck_stack = stack_alloc();
0200     if (!lc || !nodat_stack || !async_stack || !mcck_stack)
0201         goto out;
0202     memcpy(lc, &S390_lowcore, 512);
0203     memset((char *) lc + 512, 0, sizeof(*lc) - 512);
0204     lc->async_stack = async_stack + STACK_INIT_OFFSET;
0205     lc->nodat_stack = nodat_stack + STACK_INIT_OFFSET;
0206     lc->mcck_stack = mcck_stack + STACK_INIT_OFFSET;
0207     lc->cpu_nr = cpu;
0208     lc->spinlock_lockval = arch_spin_lockval(cpu);
0209     lc->spinlock_index = 0;
0210     lc->return_lpswe = gen_lpswe(__LC_RETURN_PSW);
0211     lc->return_mcck_lpswe = gen_lpswe(__LC_RETURN_MCCK_PSW);
0212     lc->preempt_count = PREEMPT_DISABLED;
0213     if (nmi_alloc_mcesa(&lc->mcesad))
0214         goto out;
0215     lowcore_ptr[cpu] = lc;
0216     pcpu_sigp_retry(pcpu, SIGP_SET_PREFIX, __pa(lc));
0217     return 0;
0218 
0219 out:
0220     stack_free(mcck_stack);
0221     stack_free(async_stack);
0222     free_pages(nodat_stack, THREAD_SIZE_ORDER);
0223     free_pages((unsigned long) lc, LC_ORDER);
0224     return -ENOMEM;
0225 }
0226 
0227 static void pcpu_free_lowcore(struct pcpu *pcpu)
0228 {
0229     unsigned long async_stack, nodat_stack, mcck_stack;
0230     struct lowcore *lc;
0231     int cpu;
0232 
0233     cpu = pcpu - pcpu_devices;
0234     lc = lowcore_ptr[cpu];
0235     nodat_stack = lc->nodat_stack - STACK_INIT_OFFSET;
0236     async_stack = lc->async_stack - STACK_INIT_OFFSET;
0237     mcck_stack = lc->mcck_stack - STACK_INIT_OFFSET;
0238     pcpu_sigp_retry(pcpu, SIGP_SET_PREFIX, 0);
0239     lowcore_ptr[cpu] = NULL;
0240     nmi_free_mcesa(&lc->mcesad);
0241     stack_free(async_stack);
0242     stack_free(mcck_stack);
0243     free_pages(nodat_stack, THREAD_SIZE_ORDER);
0244     free_pages((unsigned long) lc, LC_ORDER);
0245 }
0246 
0247 static void pcpu_prepare_secondary(struct pcpu *pcpu, int cpu)
0248 {
0249     struct lowcore *lc = lowcore_ptr[cpu];
0250 
0251     cpumask_set_cpu(cpu, &init_mm.context.cpu_attach_mask);
0252     cpumask_set_cpu(cpu, mm_cpumask(&init_mm));
0253     lc->cpu_nr = cpu;
0254     lc->restart_flags = RESTART_FLAG_CTLREGS;
0255     lc->spinlock_lockval = arch_spin_lockval(cpu);
0256     lc->spinlock_index = 0;
0257     lc->percpu_offset = __per_cpu_offset[cpu];
0258     lc->kernel_asce = S390_lowcore.kernel_asce;
0259     lc->user_asce = s390_invalid_asce;
0260     lc->machine_flags = S390_lowcore.machine_flags;
0261     lc->user_timer = lc->system_timer =
0262         lc->steal_timer = lc->avg_steal_timer = 0;
0263     __ctl_store(lc->cregs_save_area, 0, 15);
0264     lc->cregs_save_area[1] = lc->kernel_asce;
0265     lc->cregs_save_area[7] = lc->user_asce;
0266     save_access_regs((unsigned int *) lc->access_regs_save_area);
0267     arch_spin_lock_setup(cpu);
0268 }
0269 
0270 static void pcpu_attach_task(struct pcpu *pcpu, struct task_struct *tsk)
0271 {
0272     struct lowcore *lc;
0273     int cpu;
0274 
0275     cpu = pcpu - pcpu_devices;
0276     lc = lowcore_ptr[cpu];
0277     lc->kernel_stack = (unsigned long) task_stack_page(tsk)
0278         + THREAD_SIZE - STACK_FRAME_OVERHEAD - sizeof(struct pt_regs);
0279     lc->current_task = (unsigned long) tsk;
0280     lc->lpp = LPP_MAGIC;
0281     lc->current_pid = tsk->pid;
0282     lc->user_timer = tsk->thread.user_timer;
0283     lc->guest_timer = tsk->thread.guest_timer;
0284     lc->system_timer = tsk->thread.system_timer;
0285     lc->hardirq_timer = tsk->thread.hardirq_timer;
0286     lc->softirq_timer = tsk->thread.softirq_timer;
0287     lc->steal_timer = 0;
0288 }
0289 
0290 static void pcpu_start_fn(struct pcpu *pcpu, void (*func)(void *), void *data)
0291 {
0292     struct lowcore *lc;
0293     int cpu;
0294 
0295     cpu = pcpu - pcpu_devices;
0296     lc = lowcore_ptr[cpu];
0297     lc->restart_stack = lc->kernel_stack;
0298     lc->restart_fn = (unsigned long) func;
0299     lc->restart_data = (unsigned long) data;
0300     lc->restart_source = -1U;
0301     pcpu_sigp_retry(pcpu, SIGP_RESTART, 0);
0302 }
0303 
0304 typedef void (pcpu_delegate_fn)(void *);
0305 
0306 /*
0307  * Call function via PSW restart on pcpu and stop the current cpu.
0308  */
0309 static void __pcpu_delegate(pcpu_delegate_fn *func, void *data)
0310 {
0311     func(data); /* should not return */
0312 }
0313 
0314 static void pcpu_delegate(struct pcpu *pcpu,
0315               pcpu_delegate_fn *func,
0316               void *data, unsigned long stack)
0317 {
0318     struct lowcore *lc = lowcore_ptr[pcpu - pcpu_devices];
0319     unsigned int source_cpu = stap();
0320 
0321     __load_psw_mask(PSW_KERNEL_BITS | PSW_MASK_DAT);
0322     if (pcpu->address == source_cpu) {
0323         call_on_stack(2, stack, void, __pcpu_delegate,
0324                   pcpu_delegate_fn *, func, void *, data);
0325     }
0326     /* Stop target cpu (if func returns this stops the current cpu). */
0327     pcpu_sigp_retry(pcpu, SIGP_STOP, 0);
0328     /* Restart func on the target cpu and stop the current cpu. */
0329     if (lc) {
0330         lc->restart_stack = stack;
0331         lc->restart_fn = (unsigned long)func;
0332         lc->restart_data = (unsigned long)data;
0333         lc->restart_source = source_cpu;
0334     } else {
0335         put_abs_lowcore(restart_stack, stack);
0336         put_abs_lowcore(restart_fn, (unsigned long)func);
0337         put_abs_lowcore(restart_data, (unsigned long)data);
0338         put_abs_lowcore(restart_source, source_cpu);
0339     }
0340     __bpon();
0341     asm volatile(
0342         "0: sigp    0,%0,%2 # sigp restart to target cpu\n"
0343         "   brc 2,0b    # busy, try again\n"
0344         "1: sigp    0,%1,%3 # sigp stop to current cpu\n"
0345         "   brc 2,1b    # busy, try again\n"
0346         : : "d" (pcpu->address), "d" (source_cpu),
0347             "K" (SIGP_RESTART), "K" (SIGP_STOP)
0348         : "0", "1", "cc");
0349     for (;;) ;
0350 }
0351 
0352 /*
0353  * Enable additional logical cpus for multi-threading.
0354  */
0355 static int pcpu_set_smt(unsigned int mtid)
0356 {
0357     int cc;
0358 
0359     if (smp_cpu_mtid == mtid)
0360         return 0;
0361     cc = __pcpu_sigp(0, SIGP_SET_MULTI_THREADING, mtid, NULL);
0362     if (cc == 0) {
0363         smp_cpu_mtid = mtid;
0364         smp_cpu_mt_shift = 0;
0365         while (smp_cpu_mtid >= (1U << smp_cpu_mt_shift))
0366             smp_cpu_mt_shift++;
0367         pcpu_devices[0].address = stap();
0368     }
0369     return cc;
0370 }
0371 
0372 /*
0373  * Call function on an online CPU.
0374  */
0375 void smp_call_online_cpu(void (*func)(void *), void *data)
0376 {
0377     struct pcpu *pcpu;
0378 
0379     /* Use the current cpu if it is online. */
0380     pcpu = pcpu_find_address(cpu_online_mask, stap());
0381     if (!pcpu)
0382         /* Use the first online cpu. */
0383         pcpu = pcpu_devices + cpumask_first(cpu_online_mask);
0384     pcpu_delegate(pcpu, func, data, (unsigned long) restart_stack);
0385 }
0386 
0387 /*
0388  * Call function on the ipl CPU.
0389  */
0390 void smp_call_ipl_cpu(void (*func)(void *), void *data)
0391 {
0392     struct lowcore *lc = lowcore_ptr[0];
0393 
0394     if (pcpu_devices[0].address == stap())
0395         lc = &S390_lowcore;
0396 
0397     pcpu_delegate(&pcpu_devices[0], func, data,
0398               lc->nodat_stack);
0399 }
0400 
0401 int smp_find_processor_id(u16 address)
0402 {
0403     int cpu;
0404 
0405     for_each_present_cpu(cpu)
0406         if (pcpu_devices[cpu].address == address)
0407             return cpu;
0408     return -1;
0409 }
0410 
0411 void schedule_mcck_handler(void)
0412 {
0413     pcpu_ec_call(pcpu_devices + smp_processor_id(), ec_mcck_pending);
0414 }
0415 
0416 bool notrace arch_vcpu_is_preempted(int cpu)
0417 {
0418     if (test_cpu_flag_of(CIF_ENABLED_WAIT, cpu))
0419         return false;
0420     if (pcpu_running(pcpu_devices + cpu))
0421         return false;
0422     return true;
0423 }
0424 EXPORT_SYMBOL(arch_vcpu_is_preempted);
0425 
0426 void notrace smp_yield_cpu(int cpu)
0427 {
0428     if (!MACHINE_HAS_DIAG9C)
0429         return;
0430     diag_stat_inc_norecursion(DIAG_STAT_X09C);
0431     asm volatile("diag %0,0,0x9c"
0432              : : "d" (pcpu_devices[cpu].address));
0433 }
0434 EXPORT_SYMBOL_GPL(smp_yield_cpu);
0435 
0436 /*
0437  * Send cpus emergency shutdown signal. This gives the cpus the
0438  * opportunity to complete outstanding interrupts.
0439  */
0440 void notrace smp_emergency_stop(void)
0441 {
0442     static arch_spinlock_t lock = __ARCH_SPIN_LOCK_UNLOCKED;
0443     static cpumask_t cpumask;
0444     u64 end;
0445     int cpu;
0446 
0447     arch_spin_lock(&lock);
0448     cpumask_copy(&cpumask, cpu_online_mask);
0449     cpumask_clear_cpu(smp_processor_id(), &cpumask);
0450 
0451     end = get_tod_clock() + (1000000UL << 12);
0452     for_each_cpu(cpu, &cpumask) {
0453         struct pcpu *pcpu = pcpu_devices + cpu;
0454         set_bit(ec_stop_cpu, &pcpu->ec_mask);
0455         while (__pcpu_sigp(pcpu->address, SIGP_EMERGENCY_SIGNAL,
0456                    0, NULL) == SIGP_CC_BUSY &&
0457                get_tod_clock() < end)
0458             cpu_relax();
0459     }
0460     while (get_tod_clock() < end) {
0461         for_each_cpu(cpu, &cpumask)
0462             if (pcpu_stopped(pcpu_devices + cpu))
0463                 cpumask_clear_cpu(cpu, &cpumask);
0464         if (cpumask_empty(&cpumask))
0465             break;
0466         cpu_relax();
0467     }
0468     arch_spin_unlock(&lock);
0469 }
0470 NOKPROBE_SYMBOL(smp_emergency_stop);
0471 
0472 /*
0473  * Stop all cpus but the current one.
0474  */
0475 void smp_send_stop(void)
0476 {
0477     int cpu;
0478 
0479     /* Disable all interrupts/machine checks */
0480     __load_psw_mask(PSW_KERNEL_BITS | PSW_MASK_DAT);
0481     trace_hardirqs_off();
0482 
0483     debug_set_critical();
0484 
0485     if (oops_in_progress)
0486         smp_emergency_stop();
0487 
0488     /* stop all processors */
0489     for_each_online_cpu(cpu) {
0490         if (cpu == smp_processor_id())
0491             continue;
0492         pcpu_sigp_retry(pcpu_devices + cpu, SIGP_STOP, 0);
0493         while (!pcpu_stopped(pcpu_devices + cpu))
0494             cpu_relax();
0495     }
0496 }
0497 
0498 /*
0499  * This is the main routine where commands issued by other
0500  * cpus are handled.
0501  */
0502 static void smp_handle_ext_call(void)
0503 {
0504     unsigned long bits;
0505 
0506     /* handle bit signal external calls */
0507     bits = xchg(&pcpu_devices[smp_processor_id()].ec_mask, 0);
0508     if (test_bit(ec_stop_cpu, &bits))
0509         smp_stop_cpu();
0510     if (test_bit(ec_schedule, &bits))
0511         scheduler_ipi();
0512     if (test_bit(ec_call_function_single, &bits))
0513         generic_smp_call_function_single_interrupt();
0514     if (test_bit(ec_mcck_pending, &bits))
0515         __s390_handle_mcck();
0516     if (test_bit(ec_irq_work, &bits))
0517         irq_work_run();
0518 }
0519 
0520 static void do_ext_call_interrupt(struct ext_code ext_code,
0521                   unsigned int param32, unsigned long param64)
0522 {
0523     inc_irq_stat(ext_code.code == 0x1202 ? IRQEXT_EXC : IRQEXT_EMS);
0524     smp_handle_ext_call();
0525 }
0526 
0527 void arch_send_call_function_ipi_mask(const struct cpumask *mask)
0528 {
0529     int cpu;
0530 
0531     for_each_cpu(cpu, mask)
0532         pcpu_ec_call(pcpu_devices + cpu, ec_call_function_single);
0533 }
0534 
0535 void arch_send_call_function_single_ipi(int cpu)
0536 {
0537     pcpu_ec_call(pcpu_devices + cpu, ec_call_function_single);
0538 }
0539 
0540 /*
0541  * this function sends a 'reschedule' IPI to another CPU.
0542  * it goes straight through and wastes no time serializing
0543  * anything. Worst case is that we lose a reschedule ...
0544  */
0545 void smp_send_reschedule(int cpu)
0546 {
0547     pcpu_ec_call(pcpu_devices + cpu, ec_schedule);
0548 }
0549 
0550 #ifdef CONFIG_IRQ_WORK
0551 void arch_irq_work_raise(void)
0552 {
0553     pcpu_ec_call(pcpu_devices + smp_processor_id(), ec_irq_work);
0554 }
0555 #endif
0556 
0557 /*
0558  * parameter area for the set/clear control bit callbacks
0559  */
0560 struct ec_creg_mask_parms {
0561     unsigned long orval;
0562     unsigned long andval;
0563     int cr;
0564 };
0565 
0566 /*
0567  * callback for setting/clearing control bits
0568  */
0569 static void smp_ctl_bit_callback(void *info)
0570 {
0571     struct ec_creg_mask_parms *pp = info;
0572     unsigned long cregs[16];
0573 
0574     __ctl_store(cregs, 0, 15);
0575     cregs[pp->cr] = (cregs[pp->cr] & pp->andval) | pp->orval;
0576     __ctl_load(cregs, 0, 15);
0577 }
0578 
0579 static DEFINE_SPINLOCK(ctl_lock);
0580 
0581 void smp_ctl_set_clear_bit(int cr, int bit, bool set)
0582 {
0583     struct ec_creg_mask_parms parms = { .cr = cr, };
0584     u64 ctlreg;
0585 
0586     if (set) {
0587         parms.orval = 1UL << bit;
0588         parms.andval = -1UL;
0589     } else {
0590         parms.orval = 0;
0591         parms.andval = ~(1UL << bit);
0592     }
0593     spin_lock(&ctl_lock);
0594     get_abs_lowcore(ctlreg, cregs_save_area[cr]);
0595     ctlreg = (ctlreg & parms.andval) | parms.orval;
0596     put_abs_lowcore(cregs_save_area[cr], ctlreg);
0597     spin_unlock(&ctl_lock);
0598     on_each_cpu(smp_ctl_bit_callback, &parms, 1);
0599 }
0600 EXPORT_SYMBOL(smp_ctl_set_clear_bit);
0601 
0602 #ifdef CONFIG_CRASH_DUMP
0603 
0604 int smp_store_status(int cpu)
0605 {
0606     struct lowcore *lc;
0607     struct pcpu *pcpu;
0608     unsigned long pa;
0609 
0610     pcpu = pcpu_devices + cpu;
0611     lc = lowcore_ptr[cpu];
0612     pa = __pa(&lc->floating_pt_save_area);
0613     if (__pcpu_sigp_relax(pcpu->address, SIGP_STORE_STATUS_AT_ADDRESS,
0614                   pa) != SIGP_CC_ORDER_CODE_ACCEPTED)
0615         return -EIO;
0616     if (!MACHINE_HAS_VX && !MACHINE_HAS_GS)
0617         return 0;
0618     pa = lc->mcesad & MCESA_ORIGIN_MASK;
0619     if (MACHINE_HAS_GS)
0620         pa |= lc->mcesad & MCESA_LC_MASK;
0621     if (__pcpu_sigp_relax(pcpu->address, SIGP_STORE_ADDITIONAL_STATUS,
0622                   pa) != SIGP_CC_ORDER_CODE_ACCEPTED)
0623         return -EIO;
0624     return 0;
0625 }
0626 
0627 /*
0628  * Collect CPU state of the previous, crashed system.
0629  * There are four cases:
0630  * 1) standard zfcp/nvme dump
0631  *    condition: OLDMEM_BASE == NULL && is_ipl_type_dump() == true
0632  *    The state for all CPUs except the boot CPU needs to be collected
0633  *    with sigp stop-and-store-status. The boot CPU state is located in
0634  *    the absolute lowcore of the memory stored in the HSA. The zcore code
0635  *    will copy the boot CPU state from the HSA.
0636  * 2) stand-alone kdump for SCSI/NVMe (zfcp/nvme dump with swapped memory)
0637  *    condition: OLDMEM_BASE != NULL && is_ipl_type_dump() == true
0638  *    The state for all CPUs except the boot CPU needs to be collected
0639  *    with sigp stop-and-store-status. The firmware or the boot-loader
0640  *    stored the registers of the boot CPU in the absolute lowcore in the
0641  *    memory of the old system.
0642  * 3) kdump and the old kernel did not store the CPU state,
0643  *    or stand-alone kdump for DASD
0644  *    condition: OLDMEM_BASE != NULL && !is_kdump_kernel()
0645  *    The state for all CPUs except the boot CPU needs to be collected
0646  *    with sigp stop-and-store-status. The kexec code or the boot-loader
0647  *    stored the registers of the boot CPU in the memory of the old system.
0648  * 4) kdump and the old kernel stored the CPU state
0649  *    condition: OLDMEM_BASE != NULL && is_kdump_kernel()
0650  *    This case does not exist for s390 anymore, setup_arch explicitly
0651  *    deactivates the elfcorehdr= kernel parameter
0652  */
0653 static __init void smp_save_cpu_vxrs(struct save_area *sa, u16 addr,
0654                      bool is_boot_cpu, __vector128 *vxrs)
0655 {
0656     if (is_boot_cpu)
0657         vxrs = boot_cpu_vector_save_area;
0658     else
0659         __pcpu_sigp_relax(addr, SIGP_STORE_ADDITIONAL_STATUS, __pa(vxrs));
0660     save_area_add_vxrs(sa, vxrs);
0661 }
0662 
0663 static __init void smp_save_cpu_regs(struct save_area *sa, u16 addr,
0664                      bool is_boot_cpu, void *regs)
0665 {
0666     if (is_boot_cpu)
0667         copy_oldmem_kernel(regs, __LC_FPREGS_SAVE_AREA, 512);
0668     else
0669         __pcpu_sigp_relax(addr, SIGP_STORE_STATUS_AT_ADDRESS, __pa(regs));
0670     save_area_add_regs(sa, regs);
0671 }
0672 
0673 void __init smp_save_dump_cpus(void)
0674 {
0675     int addr, boot_cpu_addr, max_cpu_addr;
0676     struct save_area *sa;
0677     bool is_boot_cpu;
0678     void *page;
0679 
0680     if (!(oldmem_data.start || is_ipl_type_dump()))
0681         /* No previous system present, normal boot. */
0682         return;
0683     /* Allocate a page as dumping area for the store status sigps */
0684     page = memblock_alloc_low(PAGE_SIZE, PAGE_SIZE);
0685     if (!page)
0686         panic("ERROR: Failed to allocate %lx bytes below %lx\n",
0687               PAGE_SIZE, 1UL << 31);
0688 
0689     /* Set multi-threading state to the previous system. */
0690     pcpu_set_smt(sclp.mtid_prev);
0691     boot_cpu_addr = stap();
0692     max_cpu_addr = SCLP_MAX_CORES << sclp.mtid_prev;
0693     for (addr = 0; addr <= max_cpu_addr; addr++) {
0694         if (__pcpu_sigp_relax(addr, SIGP_SENSE, 0) ==
0695             SIGP_CC_NOT_OPERATIONAL)
0696             continue;
0697         is_boot_cpu = (addr == boot_cpu_addr);
0698         /* Allocate save area */
0699         sa = save_area_alloc(is_boot_cpu);
0700         if (!sa)
0701             panic("could not allocate memory for save area\n");
0702         if (MACHINE_HAS_VX)
0703             /* Get the vector registers */
0704             smp_save_cpu_vxrs(sa, addr, is_boot_cpu, page);
0705         /*
0706          * For a zfcp/nvme dump OLDMEM_BASE == NULL and the registers
0707          * of the boot CPU are stored in the HSA. To retrieve
0708          * these registers an SCLP request is required which is
0709          * done by drivers/s390/char/zcore.c:init_cpu_info()
0710          */
0711         if (!is_boot_cpu || oldmem_data.start)
0712             /* Get the CPU registers */
0713             smp_save_cpu_regs(sa, addr, is_boot_cpu, page);
0714     }
0715     memblock_free(page, PAGE_SIZE);
0716     diag_amode31_ops.diag308_reset();
0717     pcpu_set_smt(0);
0718 }
0719 #endif /* CONFIG_CRASH_DUMP */
0720 
0721 void smp_cpu_set_polarization(int cpu, int val)
0722 {
0723     pcpu_devices[cpu].polarization = val;
0724 }
0725 
0726 int smp_cpu_get_polarization(int cpu)
0727 {
0728     return pcpu_devices[cpu].polarization;
0729 }
0730 
0731 int smp_cpu_get_cpu_address(int cpu)
0732 {
0733     return pcpu_devices[cpu].address;
0734 }
0735 
0736 static void __ref smp_get_core_info(struct sclp_core_info *info, int early)
0737 {
0738     static int use_sigp_detection;
0739     int address;
0740 
0741     if (use_sigp_detection || sclp_get_core_info(info, early)) {
0742         use_sigp_detection = 1;
0743         for (address = 0;
0744              address < (SCLP_MAX_CORES << smp_cpu_mt_shift);
0745              address += (1U << smp_cpu_mt_shift)) {
0746             if (__pcpu_sigp_relax(address, SIGP_SENSE, 0) ==
0747                 SIGP_CC_NOT_OPERATIONAL)
0748                 continue;
0749             info->core[info->configured].core_id =
0750                 address >> smp_cpu_mt_shift;
0751             info->configured++;
0752         }
0753         info->combined = info->configured;
0754     }
0755 }
0756 
0757 static int smp_add_present_cpu(int cpu);
0758 
0759 static int smp_add_core(struct sclp_core_entry *core, cpumask_t *avail,
0760             bool configured, bool early)
0761 {
0762     struct pcpu *pcpu;
0763     int cpu, nr, i;
0764     u16 address;
0765 
0766     nr = 0;
0767     if (sclp.has_core_type && core->type != boot_core_type)
0768         return nr;
0769     cpu = cpumask_first(avail);
0770     address = core->core_id << smp_cpu_mt_shift;
0771     for (i = 0; (i <= smp_cpu_mtid) && (cpu < nr_cpu_ids); i++) {
0772         if (pcpu_find_address(cpu_present_mask, address + i))
0773             continue;
0774         pcpu = pcpu_devices + cpu;
0775         pcpu->address = address + i;
0776         if (configured)
0777             pcpu->state = CPU_STATE_CONFIGURED;
0778         else
0779             pcpu->state = CPU_STATE_STANDBY;
0780         smp_cpu_set_polarization(cpu, POLARIZATION_UNKNOWN);
0781         set_cpu_present(cpu, true);
0782         if (!early && smp_add_present_cpu(cpu) != 0)
0783             set_cpu_present(cpu, false);
0784         else
0785             nr++;
0786         cpumask_clear_cpu(cpu, avail);
0787         cpu = cpumask_next(cpu, avail);
0788     }
0789     return nr;
0790 }
0791 
0792 static int __smp_rescan_cpus(struct sclp_core_info *info, bool early)
0793 {
0794     struct sclp_core_entry *core;
0795     static cpumask_t avail;
0796     bool configured;
0797     u16 core_id;
0798     int nr, i;
0799 
0800     cpus_read_lock();
0801     mutex_lock(&smp_cpu_state_mutex);
0802     nr = 0;
0803     cpumask_xor(&avail, cpu_possible_mask, cpu_present_mask);
0804     /*
0805      * Add IPL core first (which got logical CPU number 0) to make sure
0806      * that all SMT threads get subsequent logical CPU numbers.
0807      */
0808     if (early) {
0809         core_id = pcpu_devices[0].address >> smp_cpu_mt_shift;
0810         for (i = 0; i < info->configured; i++) {
0811             core = &info->core[i];
0812             if (core->core_id == core_id) {
0813                 nr += smp_add_core(core, &avail, true, early);
0814                 break;
0815             }
0816         }
0817     }
0818     for (i = 0; i < info->combined; i++) {
0819         configured = i < info->configured;
0820         nr += smp_add_core(&info->core[i], &avail, configured, early);
0821     }
0822     mutex_unlock(&smp_cpu_state_mutex);
0823     cpus_read_unlock();
0824     return nr;
0825 }
0826 
0827 void __init smp_detect_cpus(void)
0828 {
0829     unsigned int cpu, mtid, c_cpus, s_cpus;
0830     struct sclp_core_info *info;
0831     u16 address;
0832 
0833     /* Get CPU information */
0834     info = memblock_alloc(sizeof(*info), 8);
0835     if (!info)
0836         panic("%s: Failed to allocate %zu bytes align=0x%x\n",
0837               __func__, sizeof(*info), 8);
0838     smp_get_core_info(info, 1);
0839     /* Find boot CPU type */
0840     if (sclp.has_core_type) {
0841         address = stap();
0842         for (cpu = 0; cpu < info->combined; cpu++)
0843             if (info->core[cpu].core_id == address) {
0844                 /* The boot cpu dictates the cpu type. */
0845                 boot_core_type = info->core[cpu].type;
0846                 break;
0847             }
0848         if (cpu >= info->combined)
0849             panic("Could not find boot CPU type");
0850     }
0851 
0852     /* Set multi-threading state for the current system */
0853     mtid = boot_core_type ? sclp.mtid : sclp.mtid_cp;
0854     mtid = (mtid < smp_max_threads) ? mtid : smp_max_threads - 1;
0855     pcpu_set_smt(mtid);
0856 
0857     /* Print number of CPUs */
0858     c_cpus = s_cpus = 0;
0859     for (cpu = 0; cpu < info->combined; cpu++) {
0860         if (sclp.has_core_type &&
0861             info->core[cpu].type != boot_core_type)
0862             continue;
0863         if (cpu < info->configured)
0864             c_cpus += smp_cpu_mtid + 1;
0865         else
0866             s_cpus += smp_cpu_mtid + 1;
0867     }
0868     pr_info("%d configured CPUs, %d standby CPUs\n", c_cpus, s_cpus);
0869 
0870     /* Add CPUs present at boot */
0871     __smp_rescan_cpus(info, true);
0872     memblock_free(info, sizeof(*info));
0873 }
0874 
0875 /*
0876  *  Activate a secondary processor.
0877  */
0878 static void smp_start_secondary(void *cpuvoid)
0879 {
0880     int cpu = raw_smp_processor_id();
0881 
0882     S390_lowcore.last_update_clock = get_tod_clock();
0883     S390_lowcore.restart_stack = (unsigned long)restart_stack;
0884     S390_lowcore.restart_fn = (unsigned long)do_restart;
0885     S390_lowcore.restart_data = 0;
0886     S390_lowcore.restart_source = -1U;
0887     S390_lowcore.restart_flags = 0;
0888     restore_access_regs(S390_lowcore.access_regs_save_area);
0889     cpu_init();
0890     rcu_cpu_starting(cpu);
0891     init_cpu_timer();
0892     vtime_init();
0893     vdso_getcpu_init();
0894     pfault_init();
0895     cpumask_set_cpu(cpu, &cpu_setup_mask);
0896     update_cpu_masks();
0897     notify_cpu_starting(cpu);
0898     if (topology_cpu_dedicated(cpu))
0899         set_cpu_flag(CIF_DEDICATED_CPU);
0900     else
0901         clear_cpu_flag(CIF_DEDICATED_CPU);
0902     set_cpu_online(cpu, true);
0903     inc_irq_stat(CPU_RST);
0904     local_irq_enable();
0905     cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
0906 }
0907 
0908 /* Upping and downing of CPUs */
0909 int __cpu_up(unsigned int cpu, struct task_struct *tidle)
0910 {
0911     struct pcpu *pcpu = pcpu_devices + cpu;
0912     int rc;
0913 
0914     if (pcpu->state != CPU_STATE_CONFIGURED)
0915         return -EIO;
0916     if (pcpu_sigp_retry(pcpu, SIGP_INITIAL_CPU_RESET, 0) !=
0917         SIGP_CC_ORDER_CODE_ACCEPTED)
0918         return -EIO;
0919 
0920     rc = pcpu_alloc_lowcore(pcpu, cpu);
0921     if (rc)
0922         return rc;
0923     pcpu_prepare_secondary(pcpu, cpu);
0924     pcpu_attach_task(pcpu, tidle);
0925     pcpu_start_fn(pcpu, smp_start_secondary, NULL);
0926     /* Wait until cpu puts itself in the online & active maps */
0927     while (!cpu_online(cpu))
0928         cpu_relax();
0929     return 0;
0930 }
0931 
0932 static unsigned int setup_possible_cpus __initdata;
0933 
0934 static int __init _setup_possible_cpus(char *s)
0935 {
0936     get_option(&s, &setup_possible_cpus);
0937     return 0;
0938 }
0939 early_param("possible_cpus", _setup_possible_cpus);
0940 
0941 int __cpu_disable(void)
0942 {
0943     unsigned long cregs[16];
0944     int cpu;
0945 
0946     /* Handle possible pending IPIs */
0947     smp_handle_ext_call();
0948     cpu = smp_processor_id();
0949     set_cpu_online(cpu, false);
0950     cpumask_clear_cpu(cpu, &cpu_setup_mask);
0951     update_cpu_masks();
0952     /* Disable pseudo page faults on this cpu. */
0953     pfault_fini();
0954     /* Disable interrupt sources via control register. */
0955     __ctl_store(cregs, 0, 15);
0956     cregs[0]  &= ~0x0000ee70UL; /* disable all external interrupts */
0957     cregs[6]  &= ~0xff000000UL; /* disable all I/O interrupts */
0958     cregs[14] &= ~0x1f000000UL; /* disable most machine checks */
0959     __ctl_load(cregs, 0, 15);
0960     clear_cpu_flag(CIF_NOHZ_DELAY);
0961     return 0;
0962 }
0963 
0964 void __cpu_die(unsigned int cpu)
0965 {
0966     struct pcpu *pcpu;
0967 
0968     /* Wait until target cpu is down */
0969     pcpu = pcpu_devices + cpu;
0970     while (!pcpu_stopped(pcpu))
0971         cpu_relax();
0972     pcpu_free_lowcore(pcpu);
0973     cpumask_clear_cpu(cpu, mm_cpumask(&init_mm));
0974     cpumask_clear_cpu(cpu, &init_mm.context.cpu_attach_mask);
0975 }
0976 
0977 void __noreturn cpu_die(void)
0978 {
0979     idle_task_exit();
0980     __bpon();
0981     pcpu_sigp_retry(pcpu_devices + smp_processor_id(), SIGP_STOP, 0);
0982     for (;;) ;
0983 }
0984 
0985 void __init smp_fill_possible_mask(void)
0986 {
0987     unsigned int possible, sclp_max, cpu;
0988 
0989     sclp_max = max(sclp.mtid, sclp.mtid_cp) + 1;
0990     sclp_max = min(smp_max_threads, sclp_max);
0991     sclp_max = (sclp.max_cores * sclp_max) ?: nr_cpu_ids;
0992     possible = setup_possible_cpus ?: nr_cpu_ids;
0993     possible = min(possible, sclp_max);
0994     for (cpu = 0; cpu < possible && cpu < nr_cpu_ids; cpu++)
0995         set_cpu_possible(cpu, true);
0996 }
0997 
0998 void __init smp_prepare_cpus(unsigned int max_cpus)
0999 {
1000     /* request the 0x1201 emergency signal external interrupt */
1001     if (register_external_irq(EXT_IRQ_EMERGENCY_SIG, do_ext_call_interrupt))
1002         panic("Couldn't request external interrupt 0x1201");
1003     /* request the 0x1202 external call external interrupt */
1004     if (register_external_irq(EXT_IRQ_EXTERNAL_CALL, do_ext_call_interrupt))
1005         panic("Couldn't request external interrupt 0x1202");
1006 }
1007 
1008 void __init smp_prepare_boot_cpu(void)
1009 {
1010     struct pcpu *pcpu = pcpu_devices;
1011 
1012     WARN_ON(!cpu_present(0) || !cpu_online(0));
1013     pcpu->state = CPU_STATE_CONFIGURED;
1014     S390_lowcore.percpu_offset = __per_cpu_offset[0];
1015     smp_cpu_set_polarization(0, POLARIZATION_UNKNOWN);
1016 }
1017 
1018 void __init smp_setup_processor_id(void)
1019 {
1020     pcpu_devices[0].address = stap();
1021     S390_lowcore.cpu_nr = 0;
1022     S390_lowcore.spinlock_lockval = arch_spin_lockval(0);
1023     S390_lowcore.spinlock_index = 0;
1024 }
1025 
1026 /*
1027  * the frequency of the profiling timer can be changed
1028  * by writing a multiplier value into /proc/profile.
1029  *
1030  * usually you want to run this on all CPUs ;)
1031  */
1032 int setup_profiling_timer(unsigned int multiplier)
1033 {
1034     return 0;
1035 }
1036 
1037 static ssize_t cpu_configure_show(struct device *dev,
1038                   struct device_attribute *attr, char *buf)
1039 {
1040     ssize_t count;
1041 
1042     mutex_lock(&smp_cpu_state_mutex);
1043     count = sprintf(buf, "%d\n", pcpu_devices[dev->id].state);
1044     mutex_unlock(&smp_cpu_state_mutex);
1045     return count;
1046 }
1047 
1048 static ssize_t cpu_configure_store(struct device *dev,
1049                    struct device_attribute *attr,
1050                    const char *buf, size_t count)
1051 {
1052     struct pcpu *pcpu;
1053     int cpu, val, rc, i;
1054     char delim;
1055 
1056     if (sscanf(buf, "%d %c", &val, &delim) != 1)
1057         return -EINVAL;
1058     if (val != 0 && val != 1)
1059         return -EINVAL;
1060     cpus_read_lock();
1061     mutex_lock(&smp_cpu_state_mutex);
1062     rc = -EBUSY;
1063     /* disallow configuration changes of online cpus and cpu 0 */
1064     cpu = dev->id;
1065     cpu = smp_get_base_cpu(cpu);
1066     if (cpu == 0)
1067         goto out;
1068     for (i = 0; i <= smp_cpu_mtid; i++)
1069         if (cpu_online(cpu + i))
1070             goto out;
1071     pcpu = pcpu_devices + cpu;
1072     rc = 0;
1073     switch (val) {
1074     case 0:
1075         if (pcpu->state != CPU_STATE_CONFIGURED)
1076             break;
1077         rc = sclp_core_deconfigure(pcpu->address >> smp_cpu_mt_shift);
1078         if (rc)
1079             break;
1080         for (i = 0; i <= smp_cpu_mtid; i++) {
1081             if (cpu + i >= nr_cpu_ids || !cpu_present(cpu + i))
1082                 continue;
1083             pcpu[i].state = CPU_STATE_STANDBY;
1084             smp_cpu_set_polarization(cpu + i,
1085                          POLARIZATION_UNKNOWN);
1086         }
1087         topology_expect_change();
1088         break;
1089     case 1:
1090         if (pcpu->state != CPU_STATE_STANDBY)
1091             break;
1092         rc = sclp_core_configure(pcpu->address >> smp_cpu_mt_shift);
1093         if (rc)
1094             break;
1095         for (i = 0; i <= smp_cpu_mtid; i++) {
1096             if (cpu + i >= nr_cpu_ids || !cpu_present(cpu + i))
1097                 continue;
1098             pcpu[i].state = CPU_STATE_CONFIGURED;
1099             smp_cpu_set_polarization(cpu + i,
1100                          POLARIZATION_UNKNOWN);
1101         }
1102         topology_expect_change();
1103         break;
1104     default:
1105         break;
1106     }
1107 out:
1108     mutex_unlock(&smp_cpu_state_mutex);
1109     cpus_read_unlock();
1110     return rc ? rc : count;
1111 }
1112 static DEVICE_ATTR(configure, 0644, cpu_configure_show, cpu_configure_store);
1113 
1114 static ssize_t show_cpu_address(struct device *dev,
1115                 struct device_attribute *attr, char *buf)
1116 {
1117     return sprintf(buf, "%d\n", pcpu_devices[dev->id].address);
1118 }
1119 static DEVICE_ATTR(address, 0444, show_cpu_address, NULL);
1120 
1121 static struct attribute *cpu_common_attrs[] = {
1122     &dev_attr_configure.attr,
1123     &dev_attr_address.attr,
1124     NULL,
1125 };
1126 
1127 static struct attribute_group cpu_common_attr_group = {
1128     .attrs = cpu_common_attrs,
1129 };
1130 
1131 static struct attribute *cpu_online_attrs[] = {
1132     &dev_attr_idle_count.attr,
1133     &dev_attr_idle_time_us.attr,
1134     NULL,
1135 };
1136 
1137 static struct attribute_group cpu_online_attr_group = {
1138     .attrs = cpu_online_attrs,
1139 };
1140 
1141 static int smp_cpu_online(unsigned int cpu)
1142 {
1143     struct device *s = &per_cpu(cpu_device, cpu)->dev;
1144 
1145     return sysfs_create_group(&s->kobj, &cpu_online_attr_group);
1146 }
1147 
1148 static int smp_cpu_pre_down(unsigned int cpu)
1149 {
1150     struct device *s = &per_cpu(cpu_device, cpu)->dev;
1151 
1152     sysfs_remove_group(&s->kobj, &cpu_online_attr_group);
1153     return 0;
1154 }
1155 
1156 static int smp_add_present_cpu(int cpu)
1157 {
1158     struct device *s;
1159     struct cpu *c;
1160     int rc;
1161 
1162     c = kzalloc(sizeof(*c), GFP_KERNEL);
1163     if (!c)
1164         return -ENOMEM;
1165     per_cpu(cpu_device, cpu) = c;
1166     s = &c->dev;
1167     c->hotpluggable = 1;
1168     rc = register_cpu(c, cpu);
1169     if (rc)
1170         goto out;
1171     rc = sysfs_create_group(&s->kobj, &cpu_common_attr_group);
1172     if (rc)
1173         goto out_cpu;
1174     rc = topology_cpu_init(c);
1175     if (rc)
1176         goto out_topology;
1177     return 0;
1178 
1179 out_topology:
1180     sysfs_remove_group(&s->kobj, &cpu_common_attr_group);
1181 out_cpu:
1182     unregister_cpu(c);
1183 out:
1184     return rc;
1185 }
1186 
1187 int __ref smp_rescan_cpus(void)
1188 {
1189     struct sclp_core_info *info;
1190     int nr;
1191 
1192     info = kzalloc(sizeof(*info), GFP_KERNEL);
1193     if (!info)
1194         return -ENOMEM;
1195     smp_get_core_info(info, 0);
1196     nr = __smp_rescan_cpus(info, false);
1197     kfree(info);
1198     if (nr)
1199         topology_schedule_update();
1200     return 0;
1201 }
1202 
1203 static ssize_t __ref rescan_store(struct device *dev,
1204                   struct device_attribute *attr,
1205                   const char *buf,
1206                   size_t count)
1207 {
1208     int rc;
1209 
1210     rc = lock_device_hotplug_sysfs();
1211     if (rc)
1212         return rc;
1213     rc = smp_rescan_cpus();
1214     unlock_device_hotplug();
1215     return rc ? rc : count;
1216 }
1217 static DEVICE_ATTR_WO(rescan);
1218 
1219 static int __init s390_smp_init(void)
1220 {
1221     int cpu, rc = 0;
1222 
1223     rc = device_create_file(cpu_subsys.dev_root, &dev_attr_rescan);
1224     if (rc)
1225         return rc;
1226     for_each_present_cpu(cpu) {
1227         rc = smp_add_present_cpu(cpu);
1228         if (rc)
1229             goto out;
1230     }
1231 
1232     rc = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "s390/smp:online",
1233                    smp_cpu_online, smp_cpu_pre_down);
1234     rc = rc <= 0 ? rc : 0;
1235 out:
1236     return rc;
1237 }
1238 subsys_initcall(s390_smp_init);
1239 
1240 static __always_inline void set_new_lowcore(struct lowcore *lc)
1241 {
1242     union register_pair dst, src;
1243     u32 pfx;
1244 
1245     src.even = (unsigned long) &S390_lowcore;
1246     src.odd  = sizeof(S390_lowcore);
1247     dst.even = (unsigned long) lc;
1248     dst.odd  = sizeof(*lc);
1249     pfx = __pa(lc);
1250 
1251     asm volatile(
1252         "   mvcl    %[dst],%[src]\n"
1253         "   spx %[pfx]\n"
1254         : [dst] "+&d" (dst.pair), [src] "+&d" (src.pair)
1255         : [pfx] "Q" (pfx)
1256         : "memory", "cc");
1257 }
1258 
1259 static int __init smp_reinit_ipl_cpu(void)
1260 {
1261     unsigned long async_stack, nodat_stack, mcck_stack;
1262     struct lowcore *lc, *lc_ipl;
1263     unsigned long flags, cr0;
1264     u64 mcesad;
1265 
1266     lc_ipl = lowcore_ptr[0];
1267     lc = (struct lowcore *) __get_free_pages(GFP_KERNEL | GFP_DMA, LC_ORDER);
1268     nodat_stack = __get_free_pages(GFP_KERNEL, THREAD_SIZE_ORDER);
1269     async_stack = stack_alloc();
1270     mcck_stack = stack_alloc();
1271     if (!lc || !nodat_stack || !async_stack || !mcck_stack || nmi_alloc_mcesa(&mcesad))
1272         panic("Couldn't allocate memory");
1273 
1274     local_irq_save(flags);
1275     local_mcck_disable();
1276     set_new_lowcore(lc);
1277     S390_lowcore.nodat_stack = nodat_stack + STACK_INIT_OFFSET;
1278     S390_lowcore.async_stack = async_stack + STACK_INIT_OFFSET;
1279     S390_lowcore.mcck_stack = mcck_stack + STACK_INIT_OFFSET;
1280     __ctl_store(cr0, 0, 0);
1281     __ctl_clear_bit(0, 28); /* disable lowcore protection */
1282     S390_lowcore.mcesad = mcesad;
1283     __ctl_load(cr0, 0, 0);
1284     lowcore_ptr[0] = lc;
1285     local_mcck_enable();
1286     local_irq_restore(flags);
1287 
1288     free_pages(lc_ipl->async_stack - STACK_INIT_OFFSET, THREAD_SIZE_ORDER);
1289     memblock_free_late(__pa(lc_ipl->mcck_stack - STACK_INIT_OFFSET), THREAD_SIZE);
1290     memblock_free_late(__pa(lc_ipl), sizeof(*lc_ipl));
1291 
1292     return 0;
1293 }
1294 early_initcall(smp_reinit_ipl_cpu);
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!