Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  * SGI NMI support routines
0004  *
0005  * (C) Copyright 2020 Hewlett Packard Enterprise Development LP
0006  * Copyright (C) 2007-2017 Silicon Graphics, Inc. All rights reserved.
0007  * Copyright (c) Mike Travis
0008  */
0009 
0010 #include <linux/cpu.h>
0011 #include <linux/delay.h>
0012 #include <linux/kdb.h>
0013 #include <linux/kexec.h>
0014 #include <linux/kgdb.h>
0015 #include <linux/moduleparam.h>
0016 #include <linux/nmi.h>
0017 #include <linux/sched.h>
0018 #include <linux/sched/debug.h>
0019 #include <linux/slab.h>
0020 #include <linux/clocksource.h>
0021 
0022 #include <asm/apic.h>
0023 #include <asm/current.h>
0024 #include <asm/kdebug.h>
0025 #include <asm/local64.h>
0026 #include <asm/nmi.h>
0027 #include <asm/reboot.h>
0028 #include <asm/traps.h>
0029 #include <asm/uv/uv.h>
0030 #include <asm/uv/uv_hub.h>
0031 #include <asm/uv/uv_mmrs.h>
0032 
0033 /*
0034  * UV handler for NMI
0035  *
0036  * Handle system-wide NMI events generated by the global 'power nmi' command.
0037  *
0038  * Basic operation is to field the NMI interrupt on each CPU and wait
0039  * until all CPU's have arrived into the nmi handler.  If some CPU's do not
0040  * make it into the handler, try and force them in with the IPI(NMI) signal.
0041  *
0042  * We also have to lessen UV Hub MMR accesses as much as possible as this
0043  * disrupts the UV Hub's primary mission of directing NumaLink traffic and
0044  * can cause system problems to occur.
0045  *
0046  * To do this we register our primary NMI notifier on the NMI_UNKNOWN
0047  * chain.  This reduces the number of false NMI calls when the perf
0048  * tools are running which generate an enormous number of NMIs per
0049  * second (~4M/s for 1024 CPU threads).  Our secondary NMI handler is
0050  * very short as it only checks that if it has been "pinged" with the
0051  * IPI(NMI) signal as mentioned above, and does not read the UV Hub's MMR.
0052  *
0053  */
0054 
0055 static struct uv_hub_nmi_s **uv_hub_nmi_list;
0056 
0057 DEFINE_PER_CPU(struct uv_cpu_nmi_s, uv_cpu_nmi);
0058 
0059 /* Newer SMM NMI handler, not present in all systems */
0060 static unsigned long uvh_nmi_mmrx;      /* UVH_EVENT_OCCURRED0/1 */
0061 static unsigned long uvh_nmi_mmrx_clear;    /* UVH_EVENT_OCCURRED0/1_ALIAS */
0062 static int uvh_nmi_mmrx_shift;          /* UVH_EVENT_OCCURRED0/1_EXTIO_INT0_SHFT */
0063 static char *uvh_nmi_mmrx_type;         /* "EXTIO_INT0" */
0064 
0065 /* Non-zero indicates newer SMM NMI handler present */
0066 static unsigned long uvh_nmi_mmrx_supported;    /* UVH_EXTIO_INT0_BROADCAST */
0067 
0068 /* Indicates to BIOS that we want to use the newer SMM NMI handler */
0069 static unsigned long uvh_nmi_mmrx_req;      /* UVH_BIOS_KERNEL_MMR_ALIAS_2 */
0070 static int uvh_nmi_mmrx_req_shift;      /* 62 */
0071 
0072 /* UV hubless values */
0073 #define NMI_CONTROL_PORT    0x70
0074 #define NMI_DUMMY_PORT      0x71
0075 #define PAD_OWN_GPP_D_0     0x2c
0076 #define GPI_NMI_STS_GPP_D_0 0x164
0077 #define GPI_NMI_ENA_GPP_D_0 0x174
0078 #define STS_GPP_D_0_MASK    0x1
0079 #define PAD_CFG_DW0_GPP_D_0 0x4c0
0080 #define GPIROUTNMI      (1ul << 17)
0081 #define PCH_PCR_GPIO_1_BASE 0xfdae0000ul
0082 #define PCH_PCR_GPIO_ADDRESS(offset) (int *)((u64)(pch_base) | (u64)(offset))
0083 
0084 static u64 *pch_base;
0085 static unsigned long nmi_mmr;
0086 static unsigned long nmi_mmr_clear;
0087 static unsigned long nmi_mmr_pending;
0088 
0089 static atomic_t uv_in_nmi;
0090 static atomic_t uv_nmi_cpu = ATOMIC_INIT(-1);
0091 static atomic_t uv_nmi_cpus_in_nmi = ATOMIC_INIT(-1);
0092 static atomic_t uv_nmi_slave_continue;
0093 static cpumask_var_t uv_nmi_cpu_mask;
0094 
0095 static atomic_t uv_nmi_kexec_failed;
0096 
0097 /* Values for uv_nmi_slave_continue */
0098 #define SLAVE_CLEAR 0
0099 #define SLAVE_CONTINUE  1
0100 #define SLAVE_EXIT  2
0101 
0102 /*
0103  * Default is all stack dumps go to the console and buffer.
0104  * Lower level to send to log buffer only.
0105  */
0106 static int uv_nmi_loglevel = CONSOLE_LOGLEVEL_DEFAULT;
0107 module_param_named(dump_loglevel, uv_nmi_loglevel, int, 0644);
0108 
0109 /*
0110  * The following values show statistics on how perf events are affecting
0111  * this system.
0112  */
0113 static int param_get_local64(char *buffer, const struct kernel_param *kp)
0114 {
0115     return sprintf(buffer, "%lu\n", local64_read((local64_t *)kp->arg));
0116 }
0117 
0118 static int param_set_local64(const char *val, const struct kernel_param *kp)
0119 {
0120     /* Clear on any write */
0121     local64_set((local64_t *)kp->arg, 0);
0122     return 0;
0123 }
0124 
0125 static const struct kernel_param_ops param_ops_local64 = {
0126     .get = param_get_local64,
0127     .set = param_set_local64,
0128 };
0129 #define param_check_local64(name, p) __param_check(name, p, local64_t)
0130 
0131 static local64_t uv_nmi_count;
0132 module_param_named(nmi_count, uv_nmi_count, local64, 0644);
0133 
0134 static local64_t uv_nmi_misses;
0135 module_param_named(nmi_misses, uv_nmi_misses, local64, 0644);
0136 
0137 static local64_t uv_nmi_ping_count;
0138 module_param_named(ping_count, uv_nmi_ping_count, local64, 0644);
0139 
0140 static local64_t uv_nmi_ping_misses;
0141 module_param_named(ping_misses, uv_nmi_ping_misses, local64, 0644);
0142 
0143 /*
0144  * Following values allow tuning for large systems under heavy loading
0145  */
0146 static int uv_nmi_initial_delay = 100;
0147 module_param_named(initial_delay, uv_nmi_initial_delay, int, 0644);
0148 
0149 static int uv_nmi_slave_delay = 100;
0150 module_param_named(slave_delay, uv_nmi_slave_delay, int, 0644);
0151 
0152 static int uv_nmi_loop_delay = 100;
0153 module_param_named(loop_delay, uv_nmi_loop_delay, int, 0644);
0154 
0155 static int uv_nmi_trigger_delay = 10000;
0156 module_param_named(trigger_delay, uv_nmi_trigger_delay, int, 0644);
0157 
0158 static int uv_nmi_wait_count = 100;
0159 module_param_named(wait_count, uv_nmi_wait_count, int, 0644);
0160 
0161 static int uv_nmi_retry_count = 500;
0162 module_param_named(retry_count, uv_nmi_retry_count, int, 0644);
0163 
0164 static bool uv_pch_intr_enable = true;
0165 static bool uv_pch_intr_now_enabled;
0166 module_param_named(pch_intr_enable, uv_pch_intr_enable, bool, 0644);
0167 
0168 static bool uv_pch_init_enable = true;
0169 module_param_named(pch_init_enable, uv_pch_init_enable, bool, 0644);
0170 
0171 static int uv_nmi_debug;
0172 module_param_named(debug, uv_nmi_debug, int, 0644);
0173 
0174 #define nmi_debug(fmt, ...)             \
0175     do {                        \
0176         if (uv_nmi_debug)           \
0177             pr_info(fmt, ##__VA_ARGS__);    \
0178     } while (0)
0179 
0180 /* Valid NMI Actions */
0181 #define ACTION_LEN  16
0182 static struct nmi_action {
0183     char    *action;
0184     char    *desc;
0185 } valid_acts[] = {
0186     {   "kdump",    "do kernel crash dump"          },
0187     {   "dump",     "dump process stack for each cpu"   },
0188     {   "ips",      "dump Inst Ptr info for each cpu"   },
0189     {   "kdb",      "enter KDB (needs kgdboc= assignment)"  },
0190     {   "kgdb",     "enter KGDB (needs gdb target remote)"  },
0191     {   "health",   "check if CPUs respond to NMI"      },
0192 };
0193 typedef char action_t[ACTION_LEN];
0194 static action_t uv_nmi_action = { "dump" };
0195 
0196 static int param_get_action(char *buffer, const struct kernel_param *kp)
0197 {
0198     return sprintf(buffer, "%s\n", uv_nmi_action);
0199 }
0200 
0201 static int param_set_action(const char *val, const struct kernel_param *kp)
0202 {
0203     int i;
0204     int n = ARRAY_SIZE(valid_acts);
0205     char arg[ACTION_LEN], *p;
0206 
0207     /* (remove possible '\n') */
0208     strncpy(arg, val, ACTION_LEN - 1);
0209     arg[ACTION_LEN - 1] = '\0';
0210     p = strchr(arg, '\n');
0211     if (p)
0212         *p = '\0';
0213 
0214     for (i = 0; i < n; i++)
0215         if (!strcmp(arg, valid_acts[i].action))
0216             break;
0217 
0218     if (i < n) {
0219         strcpy(uv_nmi_action, arg);
0220         pr_info("UV: New NMI action:%s\n", uv_nmi_action);
0221         return 0;
0222     }
0223 
0224     pr_err("UV: Invalid NMI action:%s, valid actions are:\n", arg);
0225     for (i = 0; i < n; i++)
0226         pr_err("UV: %-8s - %s\n",
0227             valid_acts[i].action, valid_acts[i].desc);
0228     return -EINVAL;
0229 }
0230 
0231 static const struct kernel_param_ops param_ops_action = {
0232     .get = param_get_action,
0233     .set = param_set_action,
0234 };
0235 #define param_check_action(name, p) __param_check(name, p, action_t)
0236 
0237 module_param_named(action, uv_nmi_action, action, 0644);
0238 
0239 static inline bool uv_nmi_action_is(const char *action)
0240 {
0241     return (strncmp(uv_nmi_action, action, strlen(action)) == 0);
0242 }
0243 
0244 /* Setup which NMI support is present in system */
0245 static void uv_nmi_setup_mmrs(void)
0246 {
0247     bool new_nmi_method_only = false;
0248 
0249     /* First determine arch specific MMRs to handshake with BIOS */
0250     if (UVH_EVENT_OCCURRED0_EXTIO_INT0_MASK) {  /* UV2,3,4 setup */
0251         uvh_nmi_mmrx = UVH_EVENT_OCCURRED0;
0252         uvh_nmi_mmrx_clear = UVH_EVENT_OCCURRED0_ALIAS;
0253         uvh_nmi_mmrx_shift = UVH_EVENT_OCCURRED0_EXTIO_INT0_SHFT;
0254         uvh_nmi_mmrx_type = "OCRD0-EXTIO_INT0";
0255 
0256         uvh_nmi_mmrx_supported = UVH_EXTIO_INT0_BROADCAST;
0257         uvh_nmi_mmrx_req = UVH_BIOS_KERNEL_MMR_ALIAS_2;
0258         uvh_nmi_mmrx_req_shift = 62;
0259 
0260     } else if (UVH_EVENT_OCCURRED1_EXTIO_INT0_MASK) { /* UV5+ setup */
0261         uvh_nmi_mmrx = UVH_EVENT_OCCURRED1;
0262         uvh_nmi_mmrx_clear = UVH_EVENT_OCCURRED1_ALIAS;
0263         uvh_nmi_mmrx_shift = UVH_EVENT_OCCURRED1_EXTIO_INT0_SHFT;
0264         uvh_nmi_mmrx_type = "OCRD1-EXTIO_INT0";
0265 
0266         new_nmi_method_only = true;     /* Newer nmi always valid on UV5+ */
0267         uvh_nmi_mmrx_req = 0;           /* no request bit to clear */
0268 
0269     } else {
0270         pr_err("UV:%s:NMI support not available on this system\n", __func__);
0271         return;
0272     }
0273 
0274     /* Then find out if new NMI is supported */
0275     if (new_nmi_method_only || uv_read_local_mmr(uvh_nmi_mmrx_supported)) {
0276         if (uvh_nmi_mmrx_req)
0277             uv_write_local_mmr(uvh_nmi_mmrx_req,
0278                         1UL << uvh_nmi_mmrx_req_shift);
0279         nmi_mmr = uvh_nmi_mmrx;
0280         nmi_mmr_clear = uvh_nmi_mmrx_clear;
0281         nmi_mmr_pending = 1UL << uvh_nmi_mmrx_shift;
0282         pr_info("UV: SMI NMI support: %s\n", uvh_nmi_mmrx_type);
0283     } else {
0284         nmi_mmr = UVH_NMI_MMR;
0285         nmi_mmr_clear = UVH_NMI_MMR_CLEAR;
0286         nmi_mmr_pending = 1UL << UVH_NMI_MMR_SHIFT;
0287         pr_info("UV: SMI NMI support: %s\n", UVH_NMI_MMR_TYPE);
0288     }
0289 }
0290 
0291 /* Read NMI MMR and check if NMI flag was set by BMC. */
0292 static inline int uv_nmi_test_mmr(struct uv_hub_nmi_s *hub_nmi)
0293 {
0294     hub_nmi->nmi_value = uv_read_local_mmr(nmi_mmr);
0295     atomic_inc(&hub_nmi->read_mmr_count);
0296     return !!(hub_nmi->nmi_value & nmi_mmr_pending);
0297 }
0298 
0299 static inline void uv_local_mmr_clear_nmi(void)
0300 {
0301     uv_write_local_mmr(nmi_mmr_clear, nmi_mmr_pending);
0302 }
0303 
0304 /*
0305  * UV hubless NMI handler functions
0306  */
0307 static inline void uv_reassert_nmi(void)
0308 {
0309     /* (from arch/x86/include/asm/mach_traps.h) */
0310     outb(0x8f, NMI_CONTROL_PORT);
0311     inb(NMI_DUMMY_PORT);        /* dummy read */
0312     outb(0x0f, NMI_CONTROL_PORT);
0313     inb(NMI_DUMMY_PORT);        /* dummy read */
0314 }
0315 
0316 static void uv_init_hubless_pch_io(int offset, int mask, int data)
0317 {
0318     int *addr = PCH_PCR_GPIO_ADDRESS(offset);
0319     int readd = readl(addr);
0320 
0321     if (mask) {         /* OR in new data */
0322         int writed = (readd & ~mask) | data;
0323 
0324         nmi_debug("UV:PCH: %p = %x & %x | %x (%x)\n",
0325             addr, readd, ~mask, data, writed);
0326         writel(writed, addr);
0327     } else if (readd & data) {  /* clear status bit */
0328         nmi_debug("UV:PCH: %p = %x\n", addr, data);
0329         writel(data, addr);
0330     }
0331 
0332     (void)readl(addr);      /* flush write data */
0333 }
0334 
0335 static void uv_nmi_setup_hubless_intr(void)
0336 {
0337     uv_pch_intr_now_enabled = uv_pch_intr_enable;
0338 
0339     uv_init_hubless_pch_io(
0340         PAD_CFG_DW0_GPP_D_0, GPIROUTNMI,
0341         uv_pch_intr_now_enabled ? GPIROUTNMI : 0);
0342 
0343     nmi_debug("UV:NMI: GPP_D_0 interrupt %s\n",
0344         uv_pch_intr_now_enabled ? "enabled" : "disabled");
0345 }
0346 
0347 static struct init_nmi {
0348     unsigned int    offset;
0349     unsigned int    mask;
0350     unsigned int    data;
0351 } init_nmi[] = {
0352     {   /* HOSTSW_OWN_GPP_D_0 */
0353     .offset = 0x84,
0354     .mask = 0x1,
0355     .data = 0x0,    /* ACPI Mode */
0356     },
0357 
0358 /* Clear status: */
0359     {   /* GPI_INT_STS_GPP_D_0 */
0360     .offset = 0x104,
0361     .mask = 0x0,
0362     .data = 0x1,    /* Clear Status */
0363     },
0364     {   /* GPI_GPE_STS_GPP_D_0 */
0365     .offset = 0x124,
0366     .mask = 0x0,
0367     .data = 0x1,    /* Clear Status */
0368     },
0369     {   /* GPI_SMI_STS_GPP_D_0 */
0370     .offset = 0x144,
0371     .mask = 0x0,
0372     .data = 0x1,    /* Clear Status */
0373     },
0374     {   /* GPI_NMI_STS_GPP_D_0 */
0375     .offset = 0x164,
0376     .mask = 0x0,
0377     .data = 0x1,    /* Clear Status */
0378     },
0379 
0380 /* Disable interrupts: */
0381     {   /* GPI_INT_EN_GPP_D_0 */
0382     .offset = 0x114,
0383     .mask = 0x1,
0384     .data = 0x0,    /* Disable interrupt generation */
0385     },
0386     {   /* GPI_GPE_EN_GPP_D_0 */
0387     .offset = 0x134,
0388     .mask = 0x1,
0389     .data = 0x0,    /* Disable interrupt generation */
0390     },
0391     {   /* GPI_SMI_EN_GPP_D_0 */
0392     .offset = 0x154,
0393     .mask = 0x1,
0394     .data = 0x0,    /* Disable interrupt generation */
0395     },
0396     {   /* GPI_NMI_EN_GPP_D_0 */
0397     .offset = 0x174,
0398     .mask = 0x1,
0399     .data = 0x0,    /* Disable interrupt generation */
0400     },
0401 
0402 /* Setup GPP_D_0 Pad Config: */
0403     {   /* PAD_CFG_DW0_GPP_D_0 */
0404     .offset = 0x4c0,
0405     .mask = 0xffffffff,
0406     .data = 0x82020100,
0407 /*
0408  *  31:30 Pad Reset Config (PADRSTCFG): = 2h  # PLTRST# (default)
0409  *
0410  *  29    RX Pad State Select (RXPADSTSEL): = 0 # Raw RX pad state directly
0411  *                                                from RX buffer (default)
0412  *
0413  *  28    RX Raw Override to '1' (RXRAW1): = 0 # No Override
0414  *
0415  *  26:25 RX Level/Edge Configuration (RXEVCFG):
0416  *      = 0h # Level
0417  *      = 1h # Edge
0418  *
0419  *  23    RX Invert (RXINV): = 0 # No Inversion (signal active high)
0420  *
0421  *  20    GPIO Input Route IOxAPIC (GPIROUTIOXAPIC):
0422  * = 0 # Routing does not cause peripheral IRQ...
0423  *     # (we want an NMI not an IRQ)
0424  *
0425  *  19    GPIO Input Route SCI (GPIROUTSCI): = 0 # Routing does not cause SCI.
0426  *  18    GPIO Input Route SMI (GPIROUTSMI): = 0 # Routing does not cause SMI.
0427  *  17    GPIO Input Route NMI (GPIROUTNMI): = 1 # Routing can cause NMI.
0428  *
0429  *  11:10 Pad Mode (PMODE1/0): = 0h = GPIO control the Pad.
0430  *   9    GPIO RX Disable (GPIORXDIS):
0431  * = 0 # Enable the input buffer (active low enable)
0432  *
0433  *   8    GPIO TX Disable (GPIOTXDIS):
0434  * = 1 # Disable the output buffer; i.e. Hi-Z
0435  *
0436  *   1 GPIO RX State (GPIORXSTATE): This is the current internal RX pad state..
0437  *   0 GPIO TX State (GPIOTXSTATE):
0438  * = 0 # (Leave at default)
0439  */
0440     },
0441 
0442 /* Pad Config DW1 */
0443     {   /* PAD_CFG_DW1_GPP_D_0 */
0444     .offset = 0x4c4,
0445     .mask = 0x3c00,
0446     .data = 0,  /* Termination = none (default) */
0447     },
0448 };
0449 
0450 static void uv_init_hubless_pch_d0(void)
0451 {
0452     int i, read;
0453 
0454     read = *PCH_PCR_GPIO_ADDRESS(PAD_OWN_GPP_D_0);
0455     if (read != 0) {
0456         pr_info("UV: Hubless NMI already configured\n");
0457         return;
0458     }
0459 
0460     nmi_debug("UV: Initializing UV Hubless NMI on PCH\n");
0461     for (i = 0; i < ARRAY_SIZE(init_nmi); i++) {
0462         uv_init_hubless_pch_io(init_nmi[i].offset,
0463                     init_nmi[i].mask,
0464                     init_nmi[i].data);
0465     }
0466 }
0467 
0468 static int uv_nmi_test_hubless(struct uv_hub_nmi_s *hub_nmi)
0469 {
0470     int *pstat = PCH_PCR_GPIO_ADDRESS(GPI_NMI_STS_GPP_D_0);
0471     int status = *pstat;
0472 
0473     hub_nmi->nmi_value = status;
0474     atomic_inc(&hub_nmi->read_mmr_count);
0475 
0476     if (!(status & STS_GPP_D_0_MASK))   /* Not a UV external NMI */
0477         return 0;
0478 
0479     *pstat = STS_GPP_D_0_MASK;  /* Is a UV NMI: clear GPP_D_0 status */
0480     (void)*pstat;           /* Flush write */
0481 
0482     return 1;
0483 }
0484 
0485 static int uv_test_nmi(struct uv_hub_nmi_s *hub_nmi)
0486 {
0487     if (hub_nmi->hub_present)
0488         return uv_nmi_test_mmr(hub_nmi);
0489 
0490     if (hub_nmi->pch_owner)     /* Only PCH owner can check status */
0491         return uv_nmi_test_hubless(hub_nmi);
0492 
0493     return -1;
0494 }
0495 
0496 /*
0497  * If first CPU in on this hub, set hub_nmi "in_nmi" and "owner" values and
0498  * return true.  If first CPU in on the system, set global "in_nmi" flag.
0499  */
0500 static int uv_set_in_nmi(int cpu, struct uv_hub_nmi_s *hub_nmi)
0501 {
0502     int first = atomic_add_unless(&hub_nmi->in_nmi, 1, 1);
0503 
0504     if (first) {
0505         atomic_set(&hub_nmi->cpu_owner, cpu);
0506         if (atomic_add_unless(&uv_in_nmi, 1, 1))
0507             atomic_set(&uv_nmi_cpu, cpu);
0508 
0509         atomic_inc(&hub_nmi->nmi_count);
0510     }
0511     return first;
0512 }
0513 
0514 /* Check if this is a system NMI event */
0515 static int uv_check_nmi(struct uv_hub_nmi_s *hub_nmi)
0516 {
0517     int cpu = smp_processor_id();
0518     int nmi = 0;
0519     int nmi_detected = 0;
0520 
0521     local64_inc(&uv_nmi_count);
0522     this_cpu_inc(uv_cpu_nmi.queries);
0523 
0524     do {
0525         nmi = atomic_read(&hub_nmi->in_nmi);
0526         if (nmi)
0527             break;
0528 
0529         if (raw_spin_trylock(&hub_nmi->nmi_lock)) {
0530             nmi_detected = uv_test_nmi(hub_nmi);
0531 
0532             /* Check flag for UV external NMI */
0533             if (nmi_detected > 0) {
0534                 uv_set_in_nmi(cpu, hub_nmi);
0535                 nmi = 1;
0536                 break;
0537             }
0538 
0539             /* A non-PCH node in a hubless system waits for NMI */
0540             else if (nmi_detected < 0)
0541                 goto slave_wait;
0542 
0543             /* MMR/PCH NMI flag is clear */
0544             raw_spin_unlock(&hub_nmi->nmi_lock);
0545 
0546         } else {
0547 
0548             /* Wait a moment for the HUB NMI locker to set flag */
0549 slave_wait:     cpu_relax();
0550             udelay(uv_nmi_slave_delay);
0551 
0552             /* Re-check hub in_nmi flag */
0553             nmi = atomic_read(&hub_nmi->in_nmi);
0554             if (nmi)
0555                 break;
0556         }
0557 
0558         /*
0559          * Check if this BMC missed setting the MMR NMI flag (or)
0560          * UV hubless system where only PCH owner can check flag
0561          */
0562         if (!nmi) {
0563             nmi = atomic_read(&uv_in_nmi);
0564             if (nmi)
0565                 uv_set_in_nmi(cpu, hub_nmi);
0566         }
0567 
0568         /* If we're holding the hub lock, release it now */
0569         if (nmi_detected < 0)
0570             raw_spin_unlock(&hub_nmi->nmi_lock);
0571 
0572     } while (0);
0573 
0574     if (!nmi)
0575         local64_inc(&uv_nmi_misses);
0576 
0577     return nmi;
0578 }
0579 
0580 /* Need to reset the NMI MMR register, but only once per hub. */
0581 static inline void uv_clear_nmi(int cpu)
0582 {
0583     struct uv_hub_nmi_s *hub_nmi = uv_hub_nmi;
0584 
0585     if (cpu == atomic_read(&hub_nmi->cpu_owner)) {
0586         atomic_set(&hub_nmi->cpu_owner, -1);
0587         atomic_set(&hub_nmi->in_nmi, 0);
0588         if (hub_nmi->hub_present)
0589             uv_local_mmr_clear_nmi();
0590         else
0591             uv_reassert_nmi();
0592         raw_spin_unlock(&hub_nmi->nmi_lock);
0593     }
0594 }
0595 
0596 /* Ping non-responding CPU's attempting to force them into the NMI handler */
0597 static void uv_nmi_nr_cpus_ping(void)
0598 {
0599     int cpu;
0600 
0601     for_each_cpu(cpu, uv_nmi_cpu_mask)
0602         uv_cpu_nmi_per(cpu).pinging = 1;
0603 
0604     apic->send_IPI_mask(uv_nmi_cpu_mask, APIC_DM_NMI);
0605 }
0606 
0607 /* Clean up flags for CPU's that ignored both NMI and ping */
0608 static void uv_nmi_cleanup_mask(void)
0609 {
0610     int cpu;
0611 
0612     for_each_cpu(cpu, uv_nmi_cpu_mask) {
0613         uv_cpu_nmi_per(cpu).pinging =  0;
0614         uv_cpu_nmi_per(cpu).state = UV_NMI_STATE_OUT;
0615         cpumask_clear_cpu(cpu, uv_nmi_cpu_mask);
0616     }
0617 }
0618 
0619 /* Loop waiting as CPU's enter NMI handler */
0620 static int uv_nmi_wait_cpus(int first)
0621 {
0622     int i, j, k, n = num_online_cpus();
0623     int last_k = 0, waiting = 0;
0624     int cpu = smp_processor_id();
0625 
0626     if (first) {
0627         cpumask_copy(uv_nmi_cpu_mask, cpu_online_mask);
0628         k = 0;
0629     } else {
0630         k = n - cpumask_weight(uv_nmi_cpu_mask);
0631     }
0632 
0633     /* PCH NMI causes only one CPU to respond */
0634     if (first && uv_pch_intr_now_enabled) {
0635         cpumask_clear_cpu(cpu, uv_nmi_cpu_mask);
0636         return n - k - 1;
0637     }
0638 
0639     udelay(uv_nmi_initial_delay);
0640     for (i = 0; i < uv_nmi_retry_count; i++) {
0641         int loop_delay = uv_nmi_loop_delay;
0642 
0643         for_each_cpu(j, uv_nmi_cpu_mask) {
0644             if (uv_cpu_nmi_per(j).state) {
0645                 cpumask_clear_cpu(j, uv_nmi_cpu_mask);
0646                 if (++k >= n)
0647                     break;
0648             }
0649         }
0650         if (k >= n) {       /* all in? */
0651             k = n;
0652             break;
0653         }
0654         if (last_k != k) {  /* abort if no new CPU's coming in */
0655             last_k = k;
0656             waiting = 0;
0657         } else if (++waiting > uv_nmi_wait_count)
0658             break;
0659 
0660         /* Extend delay if waiting only for CPU 0: */
0661         if (waiting && (n - k) == 1 &&
0662             cpumask_test_cpu(0, uv_nmi_cpu_mask))
0663             loop_delay *= 100;
0664 
0665         udelay(loop_delay);
0666     }
0667     atomic_set(&uv_nmi_cpus_in_nmi, k);
0668     return n - k;
0669 }
0670 
0671 /* Wait until all slave CPU's have entered UV NMI handler */
0672 static void uv_nmi_wait(int master)
0673 {
0674     /* Indicate this CPU is in: */
0675     this_cpu_write(uv_cpu_nmi.state, UV_NMI_STATE_IN);
0676 
0677     /* If not the first CPU in (the master), then we are a slave CPU */
0678     if (!master)
0679         return;
0680 
0681     do {
0682         /* Wait for all other CPU's to gather here */
0683         if (!uv_nmi_wait_cpus(1))
0684             break;
0685 
0686         /* If not all made it in, send IPI NMI to them */
0687         pr_alert("UV: Sending NMI IPI to %d CPUs: %*pbl\n",
0688              cpumask_weight(uv_nmi_cpu_mask),
0689              cpumask_pr_args(uv_nmi_cpu_mask));
0690 
0691         uv_nmi_nr_cpus_ping();
0692 
0693         /* If all CPU's are in, then done */
0694         if (!uv_nmi_wait_cpus(0))
0695             break;
0696 
0697         pr_alert("UV: %d CPUs not in NMI loop: %*pbl\n",
0698              cpumask_weight(uv_nmi_cpu_mask),
0699              cpumask_pr_args(uv_nmi_cpu_mask));
0700     } while (0);
0701 
0702     pr_alert("UV: %d of %d CPUs in NMI\n",
0703         atomic_read(&uv_nmi_cpus_in_nmi), num_online_cpus());
0704 }
0705 
0706 /* Dump Instruction Pointer header */
0707 static void uv_nmi_dump_cpu_ip_hdr(void)
0708 {
0709     pr_info("\nUV: %4s %6s %-32s %s   (Note: PID 0 not listed)\n",
0710         "CPU", "PID", "COMMAND", "IP");
0711 }
0712 
0713 /* Dump Instruction Pointer info */
0714 static void uv_nmi_dump_cpu_ip(int cpu, struct pt_regs *regs)
0715 {
0716     pr_info("UV: %4d %6d %-32.32s %pS",
0717         cpu, current->pid, current->comm, (void *)regs->ip);
0718 }
0719 
0720 /*
0721  * Dump this CPU's state.  If action was set to "kdump" and the crash_kexec
0722  * failed, then we provide "dump" as an alternate action.  Action "dump" now
0723  * also includes the show "ips" (instruction pointers) action whereas the
0724  * action "ips" only displays instruction pointers for the non-idle CPU's.
0725  * This is an abbreviated form of the "ps" command.
0726  */
0727 static void uv_nmi_dump_state_cpu(int cpu, struct pt_regs *regs)
0728 {
0729     const char *dots = " ................................. ";
0730 
0731     if (cpu == 0)
0732         uv_nmi_dump_cpu_ip_hdr();
0733 
0734     if (current->pid != 0 || !uv_nmi_action_is("ips"))
0735         uv_nmi_dump_cpu_ip(cpu, regs);
0736 
0737     if (uv_nmi_action_is("dump")) {
0738         pr_info("UV:%sNMI process trace for CPU %d\n", dots, cpu);
0739         show_regs(regs);
0740     }
0741 
0742     this_cpu_write(uv_cpu_nmi.state, UV_NMI_STATE_DUMP_DONE);
0743 }
0744 
0745 /* Trigger a slave CPU to dump it's state */
0746 static void uv_nmi_trigger_dump(int cpu)
0747 {
0748     int retry = uv_nmi_trigger_delay;
0749 
0750     if (uv_cpu_nmi_per(cpu).state != UV_NMI_STATE_IN)
0751         return;
0752 
0753     uv_cpu_nmi_per(cpu).state = UV_NMI_STATE_DUMP;
0754     do {
0755         cpu_relax();
0756         udelay(10);
0757         if (uv_cpu_nmi_per(cpu).state
0758                 != UV_NMI_STATE_DUMP)
0759             return;
0760     } while (--retry > 0);
0761 
0762     pr_crit("UV: CPU %d stuck in process dump function\n", cpu);
0763     uv_cpu_nmi_per(cpu).state = UV_NMI_STATE_DUMP_DONE;
0764 }
0765 
0766 /* Wait until all CPU's ready to exit */
0767 static void uv_nmi_sync_exit(int master)
0768 {
0769     atomic_dec(&uv_nmi_cpus_in_nmi);
0770     if (master) {
0771         while (atomic_read(&uv_nmi_cpus_in_nmi) > 0)
0772             cpu_relax();
0773         atomic_set(&uv_nmi_slave_continue, SLAVE_CLEAR);
0774     } else {
0775         while (atomic_read(&uv_nmi_slave_continue))
0776             cpu_relax();
0777     }
0778 }
0779 
0780 /* Current "health" check is to check which CPU's are responsive */
0781 static void uv_nmi_action_health(int cpu, struct pt_regs *regs, int master)
0782 {
0783     if (master) {
0784         int in = atomic_read(&uv_nmi_cpus_in_nmi);
0785         int out = num_online_cpus() - in;
0786 
0787         pr_alert("UV: NMI CPU health check (non-responding:%d)\n", out);
0788         atomic_set(&uv_nmi_slave_continue, SLAVE_EXIT);
0789     } else {
0790         while (!atomic_read(&uv_nmi_slave_continue))
0791             cpu_relax();
0792     }
0793     uv_nmi_sync_exit(master);
0794 }
0795 
0796 /* Walk through CPU list and dump state of each */
0797 static void uv_nmi_dump_state(int cpu, struct pt_regs *regs, int master)
0798 {
0799     if (master) {
0800         int tcpu;
0801         int ignored = 0;
0802         int saved_console_loglevel = console_loglevel;
0803 
0804         pr_alert("UV: tracing %s for %d CPUs from CPU %d\n",
0805             uv_nmi_action_is("ips") ? "IPs" : "processes",
0806             atomic_read(&uv_nmi_cpus_in_nmi), cpu);
0807 
0808         console_loglevel = uv_nmi_loglevel;
0809         atomic_set(&uv_nmi_slave_continue, SLAVE_EXIT);
0810         for_each_online_cpu(tcpu) {
0811             if (cpumask_test_cpu(tcpu, uv_nmi_cpu_mask))
0812                 ignored++;
0813             else if (tcpu == cpu)
0814                 uv_nmi_dump_state_cpu(tcpu, regs);
0815             else
0816                 uv_nmi_trigger_dump(tcpu);
0817         }
0818         if (ignored)
0819             pr_alert("UV: %d CPUs ignored NMI\n", ignored);
0820 
0821         console_loglevel = saved_console_loglevel;
0822         pr_alert("UV: process trace complete\n");
0823     } else {
0824         while (!atomic_read(&uv_nmi_slave_continue))
0825             cpu_relax();
0826         while (this_cpu_read(uv_cpu_nmi.state) != UV_NMI_STATE_DUMP)
0827             cpu_relax();
0828         uv_nmi_dump_state_cpu(cpu, regs);
0829     }
0830     uv_nmi_sync_exit(master);
0831 }
0832 
0833 static void uv_nmi_touch_watchdogs(void)
0834 {
0835     touch_softlockup_watchdog_sync();
0836     clocksource_touch_watchdog();
0837     rcu_cpu_stall_reset();
0838     touch_nmi_watchdog();
0839 }
0840 
0841 static void uv_nmi_kdump(int cpu, int main, struct pt_regs *regs)
0842 {
0843     /* Check if kdump kernel loaded for both main and secondary CPUs */
0844     if (!kexec_crash_image) {
0845         if (main)
0846             pr_err("UV: NMI error: kdump kernel not loaded\n");
0847         return;
0848     }
0849 
0850     /* Call crash to dump system state */
0851     if (main) {
0852         pr_emerg("UV: NMI executing crash_kexec on CPU%d\n", cpu);
0853         crash_kexec(regs);
0854 
0855         pr_emerg("UV: crash_kexec unexpectedly returned\n");
0856         atomic_set(&uv_nmi_kexec_failed, 1);
0857 
0858     } else { /* secondary */
0859 
0860         /* If kdump kernel fails, secondaries will exit this loop */
0861         while (atomic_read(&uv_nmi_kexec_failed) == 0) {
0862 
0863             /* Once shootdown cpus starts, they do not return */
0864             run_crash_ipi_callback(regs);
0865 
0866             mdelay(10);
0867         }
0868     }
0869 }
0870 
0871 #ifdef CONFIG_KGDB
0872 #ifdef CONFIG_KGDB_KDB
0873 static inline int uv_nmi_kdb_reason(void)
0874 {
0875     return KDB_REASON_SYSTEM_NMI;
0876 }
0877 #else /* !CONFIG_KGDB_KDB */
0878 static inline int uv_nmi_kdb_reason(void)
0879 {
0880     /* Ensure user is expecting to attach gdb remote */
0881     if (uv_nmi_action_is("kgdb"))
0882         return 0;
0883 
0884     pr_err("UV: NMI error: KDB is not enabled in this kernel\n");
0885     return -1;
0886 }
0887 #endif /* CONFIG_KGDB_KDB */
0888 
0889 /*
0890  * Call KGDB/KDB from NMI handler
0891  *
0892  * Note that if both KGDB and KDB are configured, then the action of 'kgdb' or
0893  * 'kdb' has no affect on which is used.  See the KGDB documentation for further
0894  * information.
0895  */
0896 static void uv_call_kgdb_kdb(int cpu, struct pt_regs *regs, int master)
0897 {
0898     if (master) {
0899         int reason = uv_nmi_kdb_reason();
0900         int ret;
0901 
0902         if (reason < 0)
0903             return;
0904 
0905         /* Call KGDB NMI handler as MASTER */
0906         ret = kgdb_nmicallin(cpu, X86_TRAP_NMI, regs, reason,
0907                 &uv_nmi_slave_continue);
0908         if (ret) {
0909             pr_alert("KGDB returned error, is kgdboc set?\n");
0910             atomic_set(&uv_nmi_slave_continue, SLAVE_EXIT);
0911         }
0912     } else {
0913         /* Wait for KGDB signal that it's ready for slaves to enter */
0914         int sig;
0915 
0916         do {
0917             cpu_relax();
0918             sig = atomic_read(&uv_nmi_slave_continue);
0919         } while (!sig);
0920 
0921         /* Call KGDB as slave */
0922         if (sig == SLAVE_CONTINUE)
0923             kgdb_nmicallback(cpu, regs);
0924     }
0925     uv_nmi_sync_exit(master);
0926 }
0927 
0928 #else /* !CONFIG_KGDB */
0929 static inline void uv_call_kgdb_kdb(int cpu, struct pt_regs *regs, int master)
0930 {
0931     pr_err("UV: NMI error: KGDB is not enabled in this kernel\n");
0932 }
0933 #endif /* !CONFIG_KGDB */
0934 
0935 /*
0936  * UV NMI handler
0937  */
0938 static int uv_handle_nmi(unsigned int reason, struct pt_regs *regs)
0939 {
0940     struct uv_hub_nmi_s *hub_nmi = uv_hub_nmi;
0941     int cpu = smp_processor_id();
0942     int master = 0;
0943     unsigned long flags;
0944 
0945     local_irq_save(flags);
0946 
0947     /* If not a UV System NMI, ignore */
0948     if (!this_cpu_read(uv_cpu_nmi.pinging) && !uv_check_nmi(hub_nmi)) {
0949         local_irq_restore(flags);
0950         return NMI_DONE;
0951     }
0952 
0953     /* Indicate we are the first CPU into the NMI handler */
0954     master = (atomic_read(&uv_nmi_cpu) == cpu);
0955 
0956     /* If NMI action is "kdump", then attempt to do it */
0957     if (uv_nmi_action_is("kdump")) {
0958         uv_nmi_kdump(cpu, master, regs);
0959 
0960         /* Unexpected return, revert action to "dump" */
0961         if (master)
0962             strncpy(uv_nmi_action, "dump", strlen(uv_nmi_action));
0963     }
0964 
0965     /* Pause as all CPU's enter the NMI handler */
0966     uv_nmi_wait(master);
0967 
0968     /* Process actions other than "kdump": */
0969     if (uv_nmi_action_is("health")) {
0970         uv_nmi_action_health(cpu, regs, master);
0971     } else if (uv_nmi_action_is("ips") || uv_nmi_action_is("dump")) {
0972         uv_nmi_dump_state(cpu, regs, master);
0973     } else if (uv_nmi_action_is("kdb") || uv_nmi_action_is("kgdb")) {
0974         uv_call_kgdb_kdb(cpu, regs, master);
0975     } else {
0976         if (master)
0977             pr_alert("UV: unknown NMI action: %s\n", uv_nmi_action);
0978         uv_nmi_sync_exit(master);
0979     }
0980 
0981     /* Clear per_cpu "in_nmi" flag */
0982     this_cpu_write(uv_cpu_nmi.state, UV_NMI_STATE_OUT);
0983 
0984     /* Clear MMR NMI flag on each hub */
0985     uv_clear_nmi(cpu);
0986 
0987     /* Clear global flags */
0988     if (master) {
0989         if (!cpumask_empty(uv_nmi_cpu_mask))
0990             uv_nmi_cleanup_mask();
0991         atomic_set(&uv_nmi_cpus_in_nmi, -1);
0992         atomic_set(&uv_nmi_cpu, -1);
0993         atomic_set(&uv_in_nmi, 0);
0994         atomic_set(&uv_nmi_kexec_failed, 0);
0995         atomic_set(&uv_nmi_slave_continue, SLAVE_CLEAR);
0996     }
0997 
0998     uv_nmi_touch_watchdogs();
0999     local_irq_restore(flags);
1000 
1001     return NMI_HANDLED;
1002 }
1003 
1004 /*
1005  * NMI handler for pulling in CPU's when perf events are grabbing our NMI
1006  */
1007 static int uv_handle_nmi_ping(unsigned int reason, struct pt_regs *regs)
1008 {
1009     int ret;
1010 
1011     this_cpu_inc(uv_cpu_nmi.queries);
1012     if (!this_cpu_read(uv_cpu_nmi.pinging)) {
1013         local64_inc(&uv_nmi_ping_misses);
1014         return NMI_DONE;
1015     }
1016 
1017     this_cpu_inc(uv_cpu_nmi.pings);
1018     local64_inc(&uv_nmi_ping_count);
1019     ret = uv_handle_nmi(reason, regs);
1020     this_cpu_write(uv_cpu_nmi.pinging, 0);
1021     return ret;
1022 }
1023 
1024 static void uv_register_nmi_notifier(void)
1025 {
1026     if (register_nmi_handler(NMI_UNKNOWN, uv_handle_nmi, 0, "uv"))
1027         pr_warn("UV: NMI handler failed to register\n");
1028 
1029     if (register_nmi_handler(NMI_LOCAL, uv_handle_nmi_ping, 0, "uvping"))
1030         pr_warn("UV: PING NMI handler failed to register\n");
1031 }
1032 
1033 void uv_nmi_init(void)
1034 {
1035     unsigned int value;
1036 
1037     /*
1038      * Unmask NMI on all CPU's
1039      */
1040     value = apic_read(APIC_LVT1) | APIC_DM_NMI;
1041     value &= ~APIC_LVT_MASKED;
1042     apic_write(APIC_LVT1, value);
1043 }
1044 
1045 /* Setup HUB NMI info */
1046 static void __init uv_nmi_setup_common(bool hubbed)
1047 {
1048     int size = sizeof(void *) * (1 << NODES_SHIFT);
1049     int cpu;
1050 
1051     uv_hub_nmi_list = kzalloc(size, GFP_KERNEL);
1052     nmi_debug("UV: NMI hub list @ 0x%p (%d)\n", uv_hub_nmi_list, size);
1053     BUG_ON(!uv_hub_nmi_list);
1054     size = sizeof(struct uv_hub_nmi_s);
1055     for_each_present_cpu(cpu) {
1056         int nid = cpu_to_node(cpu);
1057         if (uv_hub_nmi_list[nid] == NULL) {
1058             uv_hub_nmi_list[nid] = kzalloc_node(size,
1059                                 GFP_KERNEL, nid);
1060             BUG_ON(!uv_hub_nmi_list[nid]);
1061             raw_spin_lock_init(&(uv_hub_nmi_list[nid]->nmi_lock));
1062             atomic_set(&uv_hub_nmi_list[nid]->cpu_owner, -1);
1063             uv_hub_nmi_list[nid]->hub_present = hubbed;
1064             uv_hub_nmi_list[nid]->pch_owner = (nid == 0);
1065         }
1066         uv_hub_nmi_per(cpu) = uv_hub_nmi_list[nid];
1067     }
1068     BUG_ON(!alloc_cpumask_var(&uv_nmi_cpu_mask, GFP_KERNEL));
1069 }
1070 
1071 /* Setup for UV Hub systems */
1072 void __init uv_nmi_setup(void)
1073 {
1074     uv_nmi_setup_mmrs();
1075     uv_nmi_setup_common(true);
1076     uv_register_nmi_notifier();
1077     pr_info("UV: Hub NMI enabled\n");
1078 }
1079 
1080 /* Setup for UV Hubless systems */
1081 void __init uv_nmi_setup_hubless(void)
1082 {
1083     uv_nmi_setup_common(false);
1084     pch_base = xlate_dev_mem_ptr(PCH_PCR_GPIO_1_BASE);
1085     nmi_debug("UV: PCH base:%p from 0x%lx, GPP_D_0\n",
1086         pch_base, PCH_PCR_GPIO_1_BASE);
1087     if (uv_pch_init_enable)
1088         uv_init_hubless_pch_d0();
1089     uv_init_hubless_pch_io(GPI_NMI_ENA_GPP_D_0,
1090                 STS_GPP_D_0_MASK, STS_GPP_D_0_MASK);
1091     uv_nmi_setup_hubless_intr();
1092     /* Ensure NMI enabled in Processor Interface Reg: */
1093     uv_reassert_nmi();
1094     uv_register_nmi_notifier();
1095     pr_info("UV: PCH NMI enabled\n");
1096 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!