sparc/kernel/pcr.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /* pcr.c: Generic sparc64 performance counter infrastructure.
0003  *
0004  * Copyright (C) 2009 David S. Miller (davem@davemloft.net)
0005  */
0006 #include <linux/kernel.h>
0007 #include <linux/export.h>
0008 #include <linux/init.h>
0009 #include <linux/irq.h>
0010
0011 #include <linux/irq_work.h>
0012 #include <linux/ftrace.h>
0013
0014 #include <asm/pil.h>
0015 #include <asm/pcr.h>
0016 #include <asm/nmi.h>
0017 #include <asm/asi.h>
0018 #include <asm/spitfire.h>
0019
0020 /* This code is shared between various users of the performance
0021  * counters.  Users will be oprofile, pseudo-NMI watchdog, and the
0022  * perf_event support layer.
0023  */
0024
0025 /* Performance counter interrupts run unmasked at PIL level 15.
0026  * Therefore we can't do things like wakeups and other work
0027  * that expects IRQ disabling to be adhered to in locking etc.
0028  *
0029  * Therefore in such situations we defer the work by signalling
0030  * a lower level cpu IRQ.
0031  */
0032 void __irq_entry deferred_pcr_work_irq(int irq, struct pt_regs *regs)
0033 {
0034     struct pt_regs *old_regs;
0035
0036     clear_softint(1 << PIL_DEFERRED_PCR_WORK);
0037
0038     old_regs = set_irq_regs(regs);
0039     irq_enter();
0040 #ifdef CONFIG_IRQ_WORK
0041     irq_work_run();
0042 #endif
0043     irq_exit();
0044     set_irq_regs(old_regs);
0045 }
0046
0047 void arch_irq_work_raise(void)
0048 {
0049     set_softint(1 << PIL_DEFERRED_PCR_WORK);
0050 }
0051
0052 const struct pcr_ops *pcr_ops;
0053 EXPORT_SYMBOL_GPL(pcr_ops);
0054
0055 static u64 direct_pcr_read(unsigned long reg_num)
0056 {
0057     u64 val;
0058
0059     WARN_ON_ONCE(reg_num != 0);
0060     __asm__ __volatile__("rd %%pcr, %0" : "=r" (val));
0061     return val;
0062 }
0063
0064 static void direct_pcr_write(unsigned long reg_num, u64 val)
0065 {
0066     WARN_ON_ONCE(reg_num != 0);
0067     __asm__ __volatile__("wr %0, 0x0, %%pcr" : : "r" (val));
0068 }
0069
0070 static u64 direct_pic_read(unsigned long reg_num)
0071 {
0072     u64 val;
0073
0074     WARN_ON_ONCE(reg_num != 0);
0075     __asm__ __volatile__("rd %%pic, %0" : "=r" (val));
0076     return val;
0077 }
0078
0079 static void direct_pic_write(unsigned long reg_num, u64 val)
0080 {
0081     WARN_ON_ONCE(reg_num != 0);
0082
0083     /* Blackbird errata workaround.  See commentary in
0084      * arch/sparc64/kernel/smp.c:smp_percpu_timer_interrupt()
0085      * for more information.
0086      */
0087     __asm__ __volatile__("ba,pt %%xcc, 99f\n\t"
0088                  " nop\n\t"
0089                  ".align    64\n"
0090               "99:wr    %0, 0x0, %%pic\n\t"
0091                  "rd    %%pic, %%g0" : : "r" (val));
0092 }
0093
0094 static u64 direct_picl_value(unsigned int nmi_hz)
0095 {
0096     u32 delta = local_cpu_data().clock_tick / nmi_hz;
0097
0098     return ((u64)((0 - delta) & 0xffffffff)) << 32;
0099 }
0100
0101 static const struct pcr_ops direct_pcr_ops = {
0102     .read_pcr       = direct_pcr_read,
0103     .write_pcr      = direct_pcr_write,
0104     .read_pic       = direct_pic_read,
0105     .write_pic      = direct_pic_write,
0106     .nmi_picl_value     = direct_picl_value,
0107     .pcr_nmi_enable     = (PCR_PIC_PRIV | PCR_STRACE | PCR_UTRACE),
0108     .pcr_nmi_disable    = PCR_PIC_PRIV,
0109 };
0110
0111 static void n2_pcr_write(unsigned long reg_num, u64 val)
0112 {
0113     unsigned long ret;
0114
0115     WARN_ON_ONCE(reg_num != 0);
0116     if (val & PCR_N2_HTRACE) {
0117         ret = sun4v_niagara2_setperf(HV_N2_PERF_SPARC_CTL, val);
0118         if (ret != HV_EOK)
0119             direct_pcr_write(reg_num, val);
0120     } else
0121         direct_pcr_write(reg_num, val);
0122 }
0123
0124 static u64 n2_picl_value(unsigned int nmi_hz)
0125 {
0126     u32 delta = local_cpu_data().clock_tick / (nmi_hz << 2);
0127
0128     return ((u64)((0 - delta) & 0xffffffff)) << 32;
0129 }
0130
0131 static const struct pcr_ops n2_pcr_ops = {
0132     .read_pcr       = direct_pcr_read,
0133     .write_pcr      = n2_pcr_write,
0134     .read_pic       = direct_pic_read,
0135     .write_pic      = direct_pic_write,
0136     .nmi_picl_value     = n2_picl_value,
0137     .pcr_nmi_enable     = (PCR_PIC_PRIV | PCR_STRACE | PCR_UTRACE |
0138                    PCR_N2_TOE_OV1 |
0139                    (2 << PCR_N2_SL1_SHIFT) |
0140                    (0xff << PCR_N2_MASK1_SHIFT)),
0141     .pcr_nmi_disable    = PCR_PIC_PRIV,
0142 };
0143
0144 static u64 n4_pcr_read(unsigned long reg_num)
0145 {
0146     unsigned long val;
0147
0148     (void) sun4v_vt_get_perfreg(reg_num, &val);
0149
0150     return val;
0151 }
0152
0153 static void n4_pcr_write(unsigned long reg_num, u64 val)
0154 {
0155     (void) sun4v_vt_set_perfreg(reg_num, val);
0156 }
0157
0158 static u64 n4_pic_read(unsigned long reg_num)
0159 {
0160     unsigned long val;
0161
0162     __asm__ __volatile__("ldxa [%1] %2, %0"
0163                  : "=r" (val)
0164                  : "r" (reg_num * 0x8UL), "i" (ASI_PIC));
0165
0166     return val;
0167 }
0168
0169 static void n4_pic_write(unsigned long reg_num, u64 val)
0170 {
0171     __asm__ __volatile__("stxa %0, [%1] %2"
0172                  : /* no outputs */
0173                  : "r" (val), "r" (reg_num * 0x8UL), "i" (ASI_PIC));
0174 }
0175
0176 static u64 n4_picl_value(unsigned int nmi_hz)
0177 {
0178     u32 delta = local_cpu_data().clock_tick / (nmi_hz << 2);
0179
0180     return ((u64)((0 - delta) & 0xffffffff));
0181 }
0182
0183 static const struct pcr_ops n4_pcr_ops = {
0184     .read_pcr       = n4_pcr_read,
0185     .write_pcr      = n4_pcr_write,
0186     .read_pic       = n4_pic_read,
0187     .write_pic      = n4_pic_write,
0188     .nmi_picl_value     = n4_picl_value,
0189     .pcr_nmi_enable     = (PCR_N4_PICNPT | PCR_N4_STRACE |
0190                    PCR_N4_UTRACE | PCR_N4_TOE |
0191                    (26 << PCR_N4_SL_SHIFT)),
0192     .pcr_nmi_disable    = PCR_N4_PICNPT,
0193 };
0194
0195 static u64 n5_pcr_read(unsigned long reg_num)
0196 {
0197     unsigned long val;
0198
0199     (void) sun4v_t5_get_perfreg(reg_num, &val);
0200
0201     return val;
0202 }
0203
0204 static void n5_pcr_write(unsigned long reg_num, u64 val)
0205 {
0206     (void) sun4v_t5_set_perfreg(reg_num, val);
0207 }
0208
0209 static const struct pcr_ops n5_pcr_ops = {
0210     .read_pcr       = n5_pcr_read,
0211     .write_pcr      = n5_pcr_write,
0212     .read_pic       = n4_pic_read,
0213     .write_pic      = n4_pic_write,
0214     .nmi_picl_value     = n4_picl_value,
0215     .pcr_nmi_enable     = (PCR_N4_PICNPT | PCR_N4_STRACE |
0216                    PCR_N4_UTRACE | PCR_N4_TOE |
0217                    (26 << PCR_N4_SL_SHIFT)),
0218     .pcr_nmi_disable    = PCR_N4_PICNPT,
0219 };
0220
0221 static u64 m7_pcr_read(unsigned long reg_num)
0222 {
0223     unsigned long val;
0224
0225     (void) sun4v_m7_get_perfreg(reg_num, &val);
0226
0227     return val;
0228 }
0229
0230 static void m7_pcr_write(unsigned long reg_num, u64 val)
0231 {
0232     (void) sun4v_m7_set_perfreg(reg_num, val);
0233 }
0234
0235 static const struct pcr_ops m7_pcr_ops = {
0236     .read_pcr       = m7_pcr_read,
0237     .write_pcr      = m7_pcr_write,
0238     .read_pic       = n4_pic_read,
0239     .write_pic      = n4_pic_write,
0240     .nmi_picl_value     = n4_picl_value,
0241     .pcr_nmi_enable     = (PCR_N4_PICNPT | PCR_N4_STRACE |
0242                    PCR_N4_UTRACE | PCR_N4_TOE |
0243                    (26 << PCR_N4_SL_SHIFT)),
0244     .pcr_nmi_disable    = PCR_N4_PICNPT,
0245 };
0246
0247 static unsigned long perf_hsvc_group;
0248 static unsigned long perf_hsvc_major;
0249 static unsigned long perf_hsvc_minor;
0250
0251 static int __init register_perf_hsvc(void)
0252 {
0253     unsigned long hverror;
0254
0255     if (tlb_type == hypervisor) {
0256         switch (sun4v_chip_type) {
0257         case SUN4V_CHIP_NIAGARA1:
0258             perf_hsvc_group = HV_GRP_NIAG_PERF;
0259             break;
0260
0261         case SUN4V_CHIP_NIAGARA2:
0262             perf_hsvc_group = HV_GRP_N2_CPU;
0263             break;
0264
0265         case SUN4V_CHIP_NIAGARA3:
0266             perf_hsvc_group = HV_GRP_KT_CPU;
0267             break;
0268
0269         case SUN4V_CHIP_NIAGARA4:
0270             perf_hsvc_group = HV_GRP_VT_CPU;
0271             break;
0272
0273         case SUN4V_CHIP_NIAGARA5:
0274             perf_hsvc_group = HV_GRP_T5_CPU;
0275             break;
0276
0277         case SUN4V_CHIP_SPARC_M7:
0278             perf_hsvc_group = HV_GRP_M7_PERF;
0279             break;
0280
0281         default:
0282             return -ENODEV;
0283         }
0284
0285
0286         perf_hsvc_major = 1;
0287         perf_hsvc_minor = 0;
0288         hverror = sun4v_hvapi_register(perf_hsvc_group,
0289                            perf_hsvc_major,
0290                            &perf_hsvc_minor);
0291         if (hverror) {
0292             pr_err("perfmon: Could not register hvapi(0x%lx).\n",
0293                    hverror);
0294             return -ENODEV;
0295         }
0296     }
0297     return 0;
0298 }
0299
0300 static void __init unregister_perf_hsvc(void)
0301 {
0302     if (tlb_type != hypervisor)
0303         return;
0304     sun4v_hvapi_unregister(perf_hsvc_group);
0305 }
0306
0307 static int __init setup_sun4v_pcr_ops(void)
0308 {
0309     int ret = 0;
0310
0311     switch (sun4v_chip_type) {
0312     case SUN4V_CHIP_NIAGARA1:
0313     case SUN4V_CHIP_NIAGARA2:
0314     case SUN4V_CHIP_NIAGARA3:
0315         pcr_ops = &n2_pcr_ops;
0316         break;
0317
0318     case SUN4V_CHIP_NIAGARA4:
0319         pcr_ops = &n4_pcr_ops;
0320         break;
0321
0322     case SUN4V_CHIP_NIAGARA5:
0323         pcr_ops = &n5_pcr_ops;
0324         break;
0325
0326     case SUN4V_CHIP_SPARC_M7:
0327         pcr_ops = &m7_pcr_ops;
0328         break;
0329
0330     default:
0331         ret = -ENODEV;
0332         break;
0333     }
0334
0335     return ret;
0336 }
0337
0338 int __init pcr_arch_init(void)
0339 {
0340     int err = register_perf_hsvc();
0341
0342     if (err)
0343         return err;
0344
0345     switch (tlb_type) {
0346     case hypervisor:
0347         err = setup_sun4v_pcr_ops();
0348         if (err)
0349             goto out_unregister;
0350         break;
0351
0352     case cheetah:
0353     case cheetah_plus:
0354         pcr_ops = &direct_pcr_ops;
0355         break;
0356
0357     case spitfire:
0358         /* UltraSPARC-I/II and derivatives lack a profile
0359          * counter overflow interrupt so we can't make use of
0360          * their hardware currently.
0361          */
0362         fallthrough;
0363     default:
0364         err = -ENODEV;
0365         goto out_unregister;
0366     }
0367
0368     return nmi_init();
0369
0370 out_unregister:
0371     unregister_perf_hsvc();
0372     return err;
0373 }