powerpc/kernel/irq_64.c

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  *  Derived from arch/i386/kernel/irq.c
0004  *    Copyright (C) 1992 Linus Torvalds
0005  *  Adapted from arch/i386 by Gary Thomas
0006  *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
0007  *  Updated and modified by Cort Dougan <cort@fsmlabs.com>
0008  *    Copyright (C) 1996-2001 Cort Dougan
0009  *  Adapted for Power Macintosh by Paul Mackerras
0010  *    Copyright (C) 1996 Paul Mackerras (paulus@cs.anu.edu.au)
0011  *
0012  * This file contains the code used by various IRQ handling routines:
0013  * asking for different IRQ's should be done through these routines
0014  * instead of just grabbing them. Thus setups with different IRQ numbers
0015  * shouldn't result in any weird surprises, and installing new handlers
0016  * should be easier.
0017  */
0018
0019 #undef DEBUG
0020
0021 #include <linux/export.h>
0022 #include <linux/threads.h>
0023 #include <linux/kernel_stat.h>
0024 #include <linux/signal.h>
0025 #include <linux/sched.h>
0026 #include <linux/ptrace.h>
0027 #include <linux/ioport.h>
0028 #include <linux/interrupt.h>
0029 #include <linux/timex.h>
0030 #include <linux/init.h>
0031 #include <linux/slab.h>
0032 #include <linux/delay.h>
0033 #include <linux/irq.h>
0034 #include <linux/seq_file.h>
0035 #include <linux/cpumask.h>
0036 #include <linux/profile.h>
0037 #include <linux/bitops.h>
0038 #include <linux/list.h>
0039 #include <linux/radix-tree.h>
0040 #include <linux/mutex.h>
0041 #include <linux/pci.h>
0042 #include <linux/debugfs.h>
0043 #include <linux/of.h>
0044 #include <linux/of_irq.h>
0045 #include <linux/vmalloc.h>
0046 #include <linux/pgtable.h>
0047 #include <linux/static_call.h>
0048
0049 #include <linux/uaccess.h>
0050 #include <asm/interrupt.h>
0051 #include <asm/io.h>
0052 #include <asm/irq.h>
0053 #include <asm/cache.h>
0054 #include <asm/ptrace.h>
0055 #include <asm/machdep.h>
0056 #include <asm/udbg.h>
0057 #include <asm/smp.h>
0058 #include <asm/hw_irq.h>
0059 #include <asm/softirq_stack.h>
0060 #include <asm/ppc_asm.h>
0061
0062 #include <asm/paca.h>
0063 #include <asm/firmware.h>
0064 #include <asm/lv1call.h>
0065 #include <asm/dbell.h>
0066 #include <asm/trace.h>
0067 #include <asm/cpu_has_feature.h>
0068
0069 int distribute_irqs = 1;
0070
0071 void replay_soft_interrupts(void)
0072 {
0073     struct pt_regs regs;
0074
0075     /*
0076      * Be careful here, calling these interrupt handlers can cause
0077      * softirqs to be raised, which they may run when calling irq_exit,
0078      * which will cause local_irq_enable() to be run, which can then
0079      * recurse into this function. Don't keep any state across
0080      * interrupt handler calls which may change underneath us.
0081      *
0082      * We use local_paca rather than get_paca() to avoid all the
0083      * debug_smp_processor_id() business in this low level function.
0084      */
0085
0086     ppc_save_regs(&regs);
0087     regs.softe = IRQS_ENABLED;
0088     regs.msr |= MSR_EE;
0089
0090 again:
0091     if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
0092         WARN_ON_ONCE(mfmsr() & MSR_EE);
0093
0094     /*
0095      * Force the delivery of pending soft-disabled interrupts on PS3.
0096      * Any HV call will have this side effect.
0097      */
0098     if (firmware_has_feature(FW_FEATURE_PS3_LV1)) {
0099         u64 tmp, tmp2;
0100         lv1_get_version_info(&tmp, &tmp2);
0101     }
0102
0103     /*
0104      * Check if an hypervisor Maintenance interrupt happened.
0105      * This is a higher priority interrupt than the others, so
0106      * replay it first.
0107      */
0108     if (IS_ENABLED(CONFIG_PPC_BOOK3S) && (local_paca->irq_happened & PACA_IRQ_HMI)) {
0109         local_paca->irq_happened &= ~PACA_IRQ_HMI;
0110         regs.trap = INTERRUPT_HMI;
0111         handle_hmi_exception(&regs);
0112         if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS))
0113             hard_irq_disable();
0114     }
0115
0116     if (local_paca->irq_happened & PACA_IRQ_DEC) {
0117         local_paca->irq_happened &= ~PACA_IRQ_DEC;
0118         regs.trap = INTERRUPT_DECREMENTER;
0119         timer_interrupt(&regs);
0120         if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS))
0121             hard_irq_disable();
0122     }
0123
0124     if (local_paca->irq_happened & PACA_IRQ_EE) {
0125         local_paca->irq_happened &= ~PACA_IRQ_EE;
0126         regs.trap = INTERRUPT_EXTERNAL;
0127         do_IRQ(&regs);
0128         if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS))
0129             hard_irq_disable();
0130     }
0131
0132     if (IS_ENABLED(CONFIG_PPC_DOORBELL) && (local_paca->irq_happened & PACA_IRQ_DBELL)) {
0133         local_paca->irq_happened &= ~PACA_IRQ_DBELL;
0134         regs.trap = INTERRUPT_DOORBELL;
0135         doorbell_exception(&regs);
0136         if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS))
0137             hard_irq_disable();
0138     }
0139
0140     /* Book3E does not support soft-masking PMI interrupts */
0141     if (IS_ENABLED(CONFIG_PPC_BOOK3S) && (local_paca->irq_happened & PACA_IRQ_PMI)) {
0142         local_paca->irq_happened &= ~PACA_IRQ_PMI;
0143         regs.trap = INTERRUPT_PERFMON;
0144         performance_monitor_exception(&regs);
0145         if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS))
0146             hard_irq_disable();
0147     }
0148
0149     if (local_paca->irq_happened & ~PACA_IRQ_HARD_DIS) {
0150         /*
0151          * We are responding to the next interrupt, so interrupt-off
0152          * latencies should be reset here.
0153          */
0154         trace_hardirqs_on();
0155         trace_hardirqs_off();
0156         goto again;
0157     }
0158 }
0159
0160 #if defined(CONFIG_PPC_BOOK3S_64) && defined(CONFIG_PPC_KUAP)
0161 static inline void replay_soft_interrupts_irqrestore(void)
0162 {
0163     unsigned long kuap_state = get_kuap();
0164
0165     /*
0166      * Check if anything calls local_irq_enable/restore() when KUAP is
0167      * disabled (user access enabled). We handle that case here by saving
0168      * and re-locking AMR but we shouldn't get here in the first place,
0169      * hence the warning.
0170      */
0171     kuap_assert_locked();
0172
0173     if (kuap_state != AMR_KUAP_BLOCKED)
0174         set_kuap(AMR_KUAP_BLOCKED);
0175
0176     replay_soft_interrupts();
0177
0178     if (kuap_state != AMR_KUAP_BLOCKED)
0179         set_kuap(kuap_state);
0180 }
0181 #else
0182 #define replay_soft_interrupts_irqrestore() replay_soft_interrupts()
0183 #endif
0184
0185 notrace void arch_local_irq_restore(unsigned long mask)
0186 {
0187     unsigned char irq_happened;
0188
0189     /* Write the new soft-enabled value if it is a disable */
0190     if (mask) {
0191         irq_soft_mask_set(mask);
0192         return;
0193     }
0194
0195     if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
0196         WARN_ON_ONCE(in_nmi() || in_hardirq());
0197
0198     /*
0199      * After the stb, interrupts are unmasked and there are no interrupts
0200      * pending replay. The restart sequence makes this atomic with
0201      * respect to soft-masked interrupts. If this was just a simple code
0202      * sequence, a soft-masked interrupt could become pending right after
0203      * the comparison and before the stb.
0204      *
0205      * This allows interrupts to be unmasked without hard disabling, and
0206      * also without new hard interrupts coming in ahead of pending ones.
0207      */
0208     asm_volatile_goto(
0209 "1:                 \n"
0210 "       lbz 9,%0(13)    \n"
0211 "       cmpwi   9,0     \n"
0212 "       bne %l[happened]    \n"
0213 "       stb 9,%1(13)    \n"
0214 "2:                 \n"
0215         RESTART_TABLE(1b, 2b, 1b)
0216     : : "i" (offsetof(struct paca_struct, irq_happened)),
0217         "i" (offsetof(struct paca_struct, irq_soft_mask))
0218     : "cr0", "r9"
0219     : happened);
0220
0221     if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
0222         WARN_ON_ONCE(!(mfmsr() & MSR_EE));
0223
0224     return;
0225
0226 happened:
0227     irq_happened = READ_ONCE(local_paca->irq_happened);
0228     if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
0229         WARN_ON_ONCE(!irq_happened);
0230
0231     if (irq_happened == PACA_IRQ_HARD_DIS) {
0232         if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
0233             WARN_ON_ONCE(mfmsr() & MSR_EE);
0234         irq_soft_mask_set(IRQS_ENABLED);
0235         local_paca->irq_happened = 0;
0236         __hard_irq_enable();
0237         return;
0238     }
0239
0240     /* Have interrupts to replay, need to hard disable first */
0241     if (!(irq_happened & PACA_IRQ_HARD_DIS)) {
0242         if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG)) {
0243             if (!(mfmsr() & MSR_EE)) {
0244                 /*
0245                  * An interrupt could have come in and cleared
0246                  * MSR[EE] and set IRQ_HARD_DIS, so check
0247                  * IRQ_HARD_DIS again and warn if it is still
0248                  * clear.
0249                  */
0250                 irq_happened = READ_ONCE(local_paca->irq_happened);
0251                 WARN_ON_ONCE(!(irq_happened & PACA_IRQ_HARD_DIS));
0252             }
0253         }
0254         __hard_irq_disable();
0255         local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
0256     } else {
0257         if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG)) {
0258             if (WARN_ON_ONCE(mfmsr() & MSR_EE))
0259                 __hard_irq_disable();
0260         }
0261     }
0262
0263     /*
0264      * Disable preempt here, so that the below preempt_enable will
0265      * perform resched if required (a replayed interrupt may set
0266      * need_resched).
0267      */
0268     preempt_disable();
0269     irq_soft_mask_set(IRQS_ALL_DISABLED);
0270     trace_hardirqs_off();
0271
0272     replay_soft_interrupts_irqrestore();
0273     local_paca->irq_happened = 0;
0274
0275     trace_hardirqs_on();
0276     irq_soft_mask_set(IRQS_ENABLED);
0277     __hard_irq_enable();
0278     preempt_enable();
0279 }
0280 EXPORT_SYMBOL(arch_local_irq_restore);
0281
0282 /*
0283  * This is a helper to use when about to go into idle low-power
0284  * when the latter has the side effect of re-enabling interrupts
0285  * (such as calling H_CEDE under pHyp).
0286  *
0287  * You call this function with interrupts soft-disabled (this is
0288  * already the case when ppc_md.power_save is called). The function
0289  * will return whether to enter power save or just return.
0290  *
0291  * In the former case, it will have notified lockdep of interrupts
0292  * being re-enabled and generally sanitized the lazy irq state,
0293  * and in the latter case it will leave with interrupts hard
0294  * disabled and marked as such, so the local_irq_enable() call
0295  * in arch_cpu_idle() will properly re-enable everything.
0296  */
0297 bool prep_irq_for_idle(void)
0298 {
0299     /*
0300      * First we need to hard disable to ensure no interrupt
0301      * occurs before we effectively enter the low power state
0302      */
0303     __hard_irq_disable();
0304     local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
0305
0306     /*
0307      * If anything happened while we were soft-disabled,
0308      * we return now and do not enter the low power state.
0309      */
0310     if (lazy_irq_pending())
0311         return false;
0312
0313     /* Tell lockdep we are about to re-enable */
0314     trace_hardirqs_on();
0315
0316     /*
0317      * Mark interrupts as soft-enabled and clear the
0318      * PACA_IRQ_HARD_DIS from the pending mask since we
0319      * are about to hard enable as well as a side effect
0320      * of entering the low power state.
0321      */
0322     local_paca->irq_happened &= ~PACA_IRQ_HARD_DIS;
0323     irq_soft_mask_set(IRQS_ENABLED);
0324
0325     /* Tell the caller to enter the low power state */
0326     return true;
0327 }
0328
0329 #ifdef CONFIG_PPC_BOOK3S
0330 /*
0331  * This is for idle sequences that return with IRQs off, but the
0332  * idle state itself wakes on interrupt. Tell the irq tracer that
0333  * IRQs are enabled for the duration of idle so it does not get long
0334  * off times. Must be paired with fini_irq_for_idle_irqsoff.
0335  */
0336 bool prep_irq_for_idle_irqsoff(void)
0337 {
0338     WARN_ON(!irqs_disabled());
0339
0340     /*
0341      * First we need to hard disable to ensure no interrupt
0342      * occurs before we effectively enter the low power state
0343      */
0344     __hard_irq_disable();
0345     local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
0346
0347     /*
0348      * If anything happened while we were soft-disabled,
0349      * we return now and do not enter the low power state.
0350      */
0351     if (lazy_irq_pending())
0352         return false;
0353
0354     /* Tell lockdep we are about to re-enable */
0355     trace_hardirqs_on();
0356
0357     return true;
0358 }
0359
0360 /*
0361  * Take the SRR1 wakeup reason, index into this table to find the
0362  * appropriate irq_happened bit.
0363  *
0364  * Sytem reset exceptions taken in idle state also come through here,
0365  * but they are NMI interrupts so do not need to wait for IRQs to be
0366  * restored, and should be taken as early as practical. These are marked
0367  * with 0xff in the table. The Power ISA specifies 0100b as the system
0368  * reset interrupt reason.
0369  */
0370 #define IRQ_SYSTEM_RESET    0xff
0371
0372 static const u8 srr1_to_lazyirq[0x10] = {
0373     0, 0, 0,
0374     PACA_IRQ_DBELL,
0375     IRQ_SYSTEM_RESET,
0376     PACA_IRQ_DBELL,
0377     PACA_IRQ_DEC,
0378     0,
0379     PACA_IRQ_EE,
0380     PACA_IRQ_EE,
0381     PACA_IRQ_HMI,
0382     0, 0, 0, 0, 0 };
0383
0384 void replay_system_reset(void)
0385 {
0386     struct pt_regs regs;
0387
0388     ppc_save_regs(&regs);
0389     regs.trap = 0x100;
0390     get_paca()->in_nmi = 1;
0391     system_reset_exception(&regs);
0392     get_paca()->in_nmi = 0;
0393 }
0394 EXPORT_SYMBOL_GPL(replay_system_reset);
0395
0396 void irq_set_pending_from_srr1(unsigned long srr1)
0397 {
0398     unsigned int idx = (srr1 & SRR1_WAKEMASK_P8) >> 18;
0399     u8 reason = srr1_to_lazyirq[idx];
0400
0401     /*
0402      * Take the system reset now, which is immediately after registers
0403      * are restored from idle. It's an NMI, so interrupts need not be
0404      * re-enabled before it is taken.
0405      */
0406     if (unlikely(reason == IRQ_SYSTEM_RESET)) {
0407         replay_system_reset();
0408         return;
0409     }
0410
0411     if (reason == PACA_IRQ_DBELL) {
0412         /*
0413          * When doorbell triggers a system reset wakeup, the message
0414          * is not cleared, so if the doorbell interrupt is replayed
0415          * and the IPI handled, the doorbell interrupt would still
0416          * fire when EE is enabled.
0417          *
0418          * To avoid taking the superfluous doorbell interrupt,
0419          * execute a msgclr here before the interrupt is replayed.
0420          */
0421         ppc_msgclr(PPC_DBELL_MSGTYPE);
0422     }
0423
0424     /*
0425      * The 0 index (SRR1[42:45]=b0000) must always evaluate to 0,
0426      * so this can be called unconditionally with the SRR1 wake
0427      * reason as returned by the idle code, which uses 0 to mean no
0428      * interrupt.
0429      *
0430      * If a future CPU was to designate this as an interrupt reason,
0431      * then a new index for no interrupt must be assigned.
0432      */
0433     local_paca->irq_happened |= reason;
0434 }
0435 #endif /* CONFIG_PPC_BOOK3S */
0436
0437 /*
0438  * Force a replay of the external interrupt handler on this CPU.
0439  */
0440 void force_external_irq_replay(void)
0441 {
0442     /*
0443      * This must only be called with interrupts soft-disabled,
0444      * the replay will happen when re-enabling.
0445      */
0446     WARN_ON(!arch_irqs_disabled());
0447
0448     /*
0449      * Interrupts must always be hard disabled before irq_happened is
0450      * modified (to prevent lost update in case of interrupt between
0451      * load and store).
0452      */
0453     __hard_irq_disable();
0454     local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
0455
0456     /* Indicate in the PACA that we have an interrupt to replay */
0457     local_paca->irq_happened |= PACA_IRQ_EE;
0458 }
0459
0460 static int __init setup_noirqdistrib(char *str)
0461 {
0462     distribute_irqs = 0;
0463     return 1;
0464 }
0465
0466 __setup("noirqdistrib", setup_noirqdistrib);