Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  * Performance event support - Freescale Embedded Performance Monitor
0004  *
0005  * Copyright 2008-2009 Paul Mackerras, IBM Corporation.
0006  * Copyright 2010 Freescale Semiconductor, Inc.
0007  */
0008 #include <linux/kernel.h>
0009 #include <linux/sched.h>
0010 #include <linux/perf_event.h>
0011 #include <linux/percpu.h>
0012 #include <linux/hardirq.h>
0013 #include <asm/reg_fsl_emb.h>
0014 #include <asm/pmc.h>
0015 #include <asm/machdep.h>
0016 #include <asm/firmware.h>
0017 #include <asm/ptrace.h>
0018 
0019 struct cpu_hw_events {
0020     int n_events;
0021     int disabled;
0022     u8  pmcs_enabled;
0023     struct perf_event *event[MAX_HWEVENTS];
0024 };
0025 static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events);
0026 
0027 static struct fsl_emb_pmu *ppmu;
0028 
0029 /* Number of perf_events counting hardware events */
0030 static atomic_t num_events;
0031 /* Used to avoid races in calling reserve/release_pmc_hardware */
0032 static DEFINE_MUTEX(pmc_reserve_mutex);
0033 
0034 static void perf_event_interrupt(struct pt_regs *regs);
0035 
0036 /*
0037  * Read one performance monitor counter (PMC).
0038  */
0039 static unsigned long read_pmc(int idx)
0040 {
0041     unsigned long val;
0042 
0043     switch (idx) {
0044     case 0:
0045         val = mfpmr(PMRN_PMC0);
0046         break;
0047     case 1:
0048         val = mfpmr(PMRN_PMC1);
0049         break;
0050     case 2:
0051         val = mfpmr(PMRN_PMC2);
0052         break;
0053     case 3:
0054         val = mfpmr(PMRN_PMC3);
0055         break;
0056     case 4:
0057         val = mfpmr(PMRN_PMC4);
0058         break;
0059     case 5:
0060         val = mfpmr(PMRN_PMC5);
0061         break;
0062     default:
0063         printk(KERN_ERR "oops trying to read PMC%d\n", idx);
0064         val = 0;
0065     }
0066     return val;
0067 }
0068 
0069 /*
0070  * Write one PMC.
0071  */
0072 static void write_pmc(int idx, unsigned long val)
0073 {
0074     switch (idx) {
0075     case 0:
0076         mtpmr(PMRN_PMC0, val);
0077         break;
0078     case 1:
0079         mtpmr(PMRN_PMC1, val);
0080         break;
0081     case 2:
0082         mtpmr(PMRN_PMC2, val);
0083         break;
0084     case 3:
0085         mtpmr(PMRN_PMC3, val);
0086         break;
0087     case 4:
0088         mtpmr(PMRN_PMC4, val);
0089         break;
0090     case 5:
0091         mtpmr(PMRN_PMC5, val);
0092         break;
0093     default:
0094         printk(KERN_ERR "oops trying to write PMC%d\n", idx);
0095     }
0096 
0097     isync();
0098 }
0099 
0100 /*
0101  * Write one local control A register
0102  */
0103 static void write_pmlca(int idx, unsigned long val)
0104 {
0105     switch (idx) {
0106     case 0:
0107         mtpmr(PMRN_PMLCA0, val);
0108         break;
0109     case 1:
0110         mtpmr(PMRN_PMLCA1, val);
0111         break;
0112     case 2:
0113         mtpmr(PMRN_PMLCA2, val);
0114         break;
0115     case 3:
0116         mtpmr(PMRN_PMLCA3, val);
0117         break;
0118     case 4:
0119         mtpmr(PMRN_PMLCA4, val);
0120         break;
0121     case 5:
0122         mtpmr(PMRN_PMLCA5, val);
0123         break;
0124     default:
0125         printk(KERN_ERR "oops trying to write PMLCA%d\n", idx);
0126     }
0127 
0128     isync();
0129 }
0130 
0131 /*
0132  * Write one local control B register
0133  */
0134 static void write_pmlcb(int idx, unsigned long val)
0135 {
0136     switch (idx) {
0137     case 0:
0138         mtpmr(PMRN_PMLCB0, val);
0139         break;
0140     case 1:
0141         mtpmr(PMRN_PMLCB1, val);
0142         break;
0143     case 2:
0144         mtpmr(PMRN_PMLCB2, val);
0145         break;
0146     case 3:
0147         mtpmr(PMRN_PMLCB3, val);
0148         break;
0149     case 4:
0150         mtpmr(PMRN_PMLCB4, val);
0151         break;
0152     case 5:
0153         mtpmr(PMRN_PMLCB5, val);
0154         break;
0155     default:
0156         printk(KERN_ERR "oops trying to write PMLCB%d\n", idx);
0157     }
0158 
0159     isync();
0160 }
0161 
0162 static void fsl_emb_pmu_read(struct perf_event *event)
0163 {
0164     s64 val, delta, prev;
0165 
0166     if (event->hw.state & PERF_HES_STOPPED)
0167         return;
0168 
0169     /*
0170      * Performance monitor interrupts come even when interrupts
0171      * are soft-disabled, as long as interrupts are hard-enabled.
0172      * Therefore we treat them like NMIs.
0173      */
0174     do {
0175         prev = local64_read(&event->hw.prev_count);
0176         barrier();
0177         val = read_pmc(event->hw.idx);
0178     } while (local64_cmpxchg(&event->hw.prev_count, prev, val) != prev);
0179 
0180     /* The counters are only 32 bits wide */
0181     delta = (val - prev) & 0xfffffffful;
0182     local64_add(delta, &event->count);
0183     local64_sub(delta, &event->hw.period_left);
0184 }
0185 
0186 /*
0187  * Disable all events to prevent PMU interrupts and to allow
0188  * events to be added or removed.
0189  */
0190 static void fsl_emb_pmu_disable(struct pmu *pmu)
0191 {
0192     struct cpu_hw_events *cpuhw;
0193     unsigned long flags;
0194 
0195     local_irq_save(flags);
0196     cpuhw = this_cpu_ptr(&cpu_hw_events);
0197 
0198     if (!cpuhw->disabled) {
0199         cpuhw->disabled = 1;
0200 
0201         /*
0202          * Check if we ever enabled the PMU on this cpu.
0203          */
0204         if (!cpuhw->pmcs_enabled) {
0205             ppc_enable_pmcs();
0206             cpuhw->pmcs_enabled = 1;
0207         }
0208 
0209         if (atomic_read(&num_events)) {
0210             /*
0211              * Set the 'freeze all counters' bit, and disable
0212              * interrupts.  The barrier is to make sure the
0213              * mtpmr has been executed and the PMU has frozen
0214              * the events before we return.
0215              */
0216 
0217             mtpmr(PMRN_PMGC0, PMGC0_FAC);
0218             isync();
0219         }
0220     }
0221     local_irq_restore(flags);
0222 }
0223 
0224 /*
0225  * Re-enable all events if disable == 0.
0226  * If we were previously disabled and events were added, then
0227  * put the new config on the PMU.
0228  */
0229 static void fsl_emb_pmu_enable(struct pmu *pmu)
0230 {
0231     struct cpu_hw_events *cpuhw;
0232     unsigned long flags;
0233 
0234     local_irq_save(flags);
0235     cpuhw = this_cpu_ptr(&cpu_hw_events);
0236     if (!cpuhw->disabled)
0237         goto out;
0238 
0239     cpuhw->disabled = 0;
0240     ppc_set_pmu_inuse(cpuhw->n_events != 0);
0241 
0242     if (cpuhw->n_events > 0) {
0243         mtpmr(PMRN_PMGC0, PMGC0_PMIE | PMGC0_FCECE);
0244         isync();
0245     }
0246 
0247  out:
0248     local_irq_restore(flags);
0249 }
0250 
0251 static int collect_events(struct perf_event *group, int max_count,
0252               struct perf_event *ctrs[])
0253 {
0254     int n = 0;
0255     struct perf_event *event;
0256 
0257     if (!is_software_event(group)) {
0258         if (n >= max_count)
0259             return -1;
0260         ctrs[n] = group;
0261         n++;
0262     }
0263     for_each_sibling_event(event, group) {
0264         if (!is_software_event(event) &&
0265             event->state != PERF_EVENT_STATE_OFF) {
0266             if (n >= max_count)
0267                 return -1;
0268             ctrs[n] = event;
0269             n++;
0270         }
0271     }
0272     return n;
0273 }
0274 
0275 /* context locked on entry */
0276 static int fsl_emb_pmu_add(struct perf_event *event, int flags)
0277 {
0278     struct cpu_hw_events *cpuhw;
0279     int ret = -EAGAIN;
0280     int num_counters = ppmu->n_counter;
0281     u64 val;
0282     int i;
0283 
0284     perf_pmu_disable(event->pmu);
0285     cpuhw = &get_cpu_var(cpu_hw_events);
0286 
0287     if (event->hw.config & FSL_EMB_EVENT_RESTRICTED)
0288         num_counters = ppmu->n_restricted;
0289 
0290     /*
0291      * Allocate counters from top-down, so that restricted-capable
0292      * counters are kept free as long as possible.
0293      */
0294     for (i = num_counters - 1; i >= 0; i--) {
0295         if (cpuhw->event[i])
0296             continue;
0297 
0298         break;
0299     }
0300 
0301     if (i < 0)
0302         goto out;
0303 
0304     event->hw.idx = i;
0305     cpuhw->event[i] = event;
0306     ++cpuhw->n_events;
0307 
0308     val = 0;
0309     if (event->hw.sample_period) {
0310         s64 left = local64_read(&event->hw.period_left);
0311         if (left < 0x80000000L)
0312             val = 0x80000000L - left;
0313     }
0314     local64_set(&event->hw.prev_count, val);
0315 
0316     if (unlikely(!(flags & PERF_EF_START))) {
0317         event->hw.state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
0318         val = 0;
0319     } else {
0320         event->hw.state &= ~(PERF_HES_STOPPED | PERF_HES_UPTODATE);
0321     }
0322 
0323     write_pmc(i, val);
0324     perf_event_update_userpage(event);
0325 
0326     write_pmlcb(i, event->hw.config >> 32);
0327     write_pmlca(i, event->hw.config_base);
0328 
0329     ret = 0;
0330  out:
0331     put_cpu_var(cpu_hw_events);
0332     perf_pmu_enable(event->pmu);
0333     return ret;
0334 }
0335 
0336 /* context locked on entry */
0337 static void fsl_emb_pmu_del(struct perf_event *event, int flags)
0338 {
0339     struct cpu_hw_events *cpuhw;
0340     int i = event->hw.idx;
0341 
0342     perf_pmu_disable(event->pmu);
0343     if (i < 0)
0344         goto out;
0345 
0346     fsl_emb_pmu_read(event);
0347 
0348     cpuhw = &get_cpu_var(cpu_hw_events);
0349 
0350     WARN_ON(event != cpuhw->event[event->hw.idx]);
0351 
0352     write_pmlca(i, 0);
0353     write_pmlcb(i, 0);
0354     write_pmc(i, 0);
0355 
0356     cpuhw->event[i] = NULL;
0357     event->hw.idx = -1;
0358 
0359     /*
0360      * TODO: if at least one restricted event exists, and we
0361      * just freed up a non-restricted-capable counter, and
0362      * there is a restricted-capable counter occupied by
0363      * a non-restricted event, migrate that event to the
0364      * vacated counter.
0365      */
0366 
0367     cpuhw->n_events--;
0368 
0369  out:
0370     perf_pmu_enable(event->pmu);
0371     put_cpu_var(cpu_hw_events);
0372 }
0373 
0374 static void fsl_emb_pmu_start(struct perf_event *event, int ef_flags)
0375 {
0376     unsigned long flags;
0377     unsigned long val;
0378     s64 left;
0379 
0380     if (event->hw.idx < 0 || !event->hw.sample_period)
0381         return;
0382 
0383     if (!(event->hw.state & PERF_HES_STOPPED))
0384         return;
0385 
0386     if (ef_flags & PERF_EF_RELOAD)
0387         WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
0388 
0389     local_irq_save(flags);
0390     perf_pmu_disable(event->pmu);
0391 
0392     event->hw.state = 0;
0393     left = local64_read(&event->hw.period_left);
0394     val = 0;
0395     if (left < 0x80000000L)
0396         val = 0x80000000L - left;
0397     write_pmc(event->hw.idx, val);
0398 
0399     perf_event_update_userpage(event);
0400     perf_pmu_enable(event->pmu);
0401     local_irq_restore(flags);
0402 }
0403 
0404 static void fsl_emb_pmu_stop(struct perf_event *event, int ef_flags)
0405 {
0406     unsigned long flags;
0407 
0408     if (event->hw.idx < 0 || !event->hw.sample_period)
0409         return;
0410 
0411     if (event->hw.state & PERF_HES_STOPPED)
0412         return;
0413 
0414     local_irq_save(flags);
0415     perf_pmu_disable(event->pmu);
0416 
0417     fsl_emb_pmu_read(event);
0418     event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
0419     write_pmc(event->hw.idx, 0);
0420 
0421     perf_event_update_userpage(event);
0422     perf_pmu_enable(event->pmu);
0423     local_irq_restore(flags);
0424 }
0425 
0426 /*
0427  * Release the PMU if this is the last perf_event.
0428  */
0429 static void hw_perf_event_destroy(struct perf_event *event)
0430 {
0431     if (!atomic_add_unless(&num_events, -1, 1)) {
0432         mutex_lock(&pmc_reserve_mutex);
0433         if (atomic_dec_return(&num_events) == 0)
0434             release_pmc_hardware();
0435         mutex_unlock(&pmc_reserve_mutex);
0436     }
0437 }
0438 
0439 /*
0440  * Translate a generic cache event_id config to a raw event_id code.
0441  */
0442 static int hw_perf_cache_event(u64 config, u64 *eventp)
0443 {
0444     unsigned long type, op, result;
0445     int ev;
0446 
0447     if (!ppmu->cache_events)
0448         return -EINVAL;
0449 
0450     /* unpack config */
0451     type = config & 0xff;
0452     op = (config >> 8) & 0xff;
0453     result = (config >> 16) & 0xff;
0454 
0455     if (type >= PERF_COUNT_HW_CACHE_MAX ||
0456         op >= PERF_COUNT_HW_CACHE_OP_MAX ||
0457         result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
0458         return -EINVAL;
0459 
0460     ev = (*ppmu->cache_events)[type][op][result];
0461     if (ev == 0)
0462         return -EOPNOTSUPP;
0463     if (ev == -1)
0464         return -EINVAL;
0465     *eventp = ev;
0466     return 0;
0467 }
0468 
0469 static int fsl_emb_pmu_event_init(struct perf_event *event)
0470 {
0471     u64 ev;
0472     struct perf_event *events[MAX_HWEVENTS];
0473     int n;
0474     int err;
0475     int num_restricted;
0476     int i;
0477 
0478     if (ppmu->n_counter > MAX_HWEVENTS) {
0479         WARN(1, "No. of perf counters (%d) is higher than max array size(%d)\n",
0480             ppmu->n_counter, MAX_HWEVENTS);
0481         ppmu->n_counter = MAX_HWEVENTS;
0482     }
0483 
0484     switch (event->attr.type) {
0485     case PERF_TYPE_HARDWARE:
0486         ev = event->attr.config;
0487         if (ev >= ppmu->n_generic || ppmu->generic_events[ev] == 0)
0488             return -EOPNOTSUPP;
0489         ev = ppmu->generic_events[ev];
0490         break;
0491 
0492     case PERF_TYPE_HW_CACHE:
0493         err = hw_perf_cache_event(event->attr.config, &ev);
0494         if (err)
0495             return err;
0496         break;
0497 
0498     case PERF_TYPE_RAW:
0499         ev = event->attr.config;
0500         break;
0501 
0502     default:
0503         return -ENOENT;
0504     }
0505 
0506     event->hw.config = ppmu->xlate_event(ev);
0507     if (!(event->hw.config & FSL_EMB_EVENT_VALID))
0508         return -EINVAL;
0509 
0510     /*
0511      * If this is in a group, check if it can go on with all the
0512      * other hardware events in the group.  We assume the event
0513      * hasn't been linked into its leader's sibling list at this point.
0514      */
0515     n = 0;
0516     if (event->group_leader != event) {
0517         n = collect_events(event->group_leader,
0518                            ppmu->n_counter - 1, events);
0519         if (n < 0)
0520             return -EINVAL;
0521     }
0522 
0523     if (event->hw.config & FSL_EMB_EVENT_RESTRICTED) {
0524         num_restricted = 0;
0525         for (i = 0; i < n; i++) {
0526             if (events[i]->hw.config & FSL_EMB_EVENT_RESTRICTED)
0527                 num_restricted++;
0528         }
0529 
0530         if (num_restricted >= ppmu->n_restricted)
0531             return -EINVAL;
0532     }
0533 
0534     event->hw.idx = -1;
0535 
0536     event->hw.config_base = PMLCA_CE | PMLCA_FCM1 |
0537                             (u32)((ev << 16) & PMLCA_EVENT_MASK);
0538 
0539     if (event->attr.exclude_user)
0540         event->hw.config_base |= PMLCA_FCU;
0541     if (event->attr.exclude_kernel)
0542         event->hw.config_base |= PMLCA_FCS;
0543     if (event->attr.exclude_idle)
0544         return -ENOTSUPP;
0545 
0546     event->hw.last_period = event->hw.sample_period;
0547     local64_set(&event->hw.period_left, event->hw.last_period);
0548 
0549     /*
0550      * See if we need to reserve the PMU.
0551      * If no events are currently in use, then we have to take a
0552      * mutex to ensure that we don't race with another task doing
0553      * reserve_pmc_hardware or release_pmc_hardware.
0554      */
0555     err = 0;
0556     if (!atomic_inc_not_zero(&num_events)) {
0557         mutex_lock(&pmc_reserve_mutex);
0558         if (atomic_read(&num_events) == 0 &&
0559             reserve_pmc_hardware(perf_event_interrupt))
0560             err = -EBUSY;
0561         else
0562             atomic_inc(&num_events);
0563         mutex_unlock(&pmc_reserve_mutex);
0564 
0565         mtpmr(PMRN_PMGC0, PMGC0_FAC);
0566         isync();
0567     }
0568     event->destroy = hw_perf_event_destroy;
0569 
0570     return err;
0571 }
0572 
0573 static struct pmu fsl_emb_pmu = {
0574     .pmu_enable = fsl_emb_pmu_enable,
0575     .pmu_disable    = fsl_emb_pmu_disable,
0576     .event_init = fsl_emb_pmu_event_init,
0577     .add        = fsl_emb_pmu_add,
0578     .del        = fsl_emb_pmu_del,
0579     .start      = fsl_emb_pmu_start,
0580     .stop       = fsl_emb_pmu_stop,
0581     .read       = fsl_emb_pmu_read,
0582 };
0583 
0584 /*
0585  * A counter has overflowed; update its count and record
0586  * things if requested.  Note that interrupts are hard-disabled
0587  * here so there is no possibility of being interrupted.
0588  */
0589 static void record_and_restart(struct perf_event *event, unsigned long val,
0590                    struct pt_regs *regs)
0591 {
0592     u64 period = event->hw.sample_period;
0593     s64 prev, delta, left;
0594     int record = 0;
0595 
0596     if (event->hw.state & PERF_HES_STOPPED) {
0597         write_pmc(event->hw.idx, 0);
0598         return;
0599     }
0600 
0601     /* we don't have to worry about interrupts here */
0602     prev = local64_read(&event->hw.prev_count);
0603     delta = (val - prev) & 0xfffffffful;
0604     local64_add(delta, &event->count);
0605 
0606     /*
0607      * See if the total period for this event has expired,
0608      * and update for the next period.
0609      */
0610     val = 0;
0611     left = local64_read(&event->hw.period_left) - delta;
0612     if (period) {
0613         if (left <= 0) {
0614             left += period;
0615             if (left <= 0)
0616                 left = period;
0617             record = 1;
0618             event->hw.last_period = event->hw.sample_period;
0619         }
0620         if (left < 0x80000000LL)
0621             val = 0x80000000LL - left;
0622     }
0623 
0624     write_pmc(event->hw.idx, val);
0625     local64_set(&event->hw.prev_count, val);
0626     local64_set(&event->hw.period_left, left);
0627     perf_event_update_userpage(event);
0628 
0629     /*
0630      * Finally record data if requested.
0631      */
0632     if (record) {
0633         struct perf_sample_data data;
0634 
0635         perf_sample_data_init(&data, 0, event->hw.last_period);
0636 
0637         if (perf_event_overflow(event, &data, regs))
0638             fsl_emb_pmu_stop(event, 0);
0639     }
0640 }
0641 
0642 static void perf_event_interrupt(struct pt_regs *regs)
0643 {
0644     int i;
0645     struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
0646     struct perf_event *event;
0647     unsigned long val;
0648     int found = 0;
0649 
0650     for (i = 0; i < ppmu->n_counter; ++i) {
0651         event = cpuhw->event[i];
0652 
0653         val = read_pmc(i);
0654         if ((int)val < 0) {
0655             if (event) {
0656                 /* event has overflowed */
0657                 found = 1;
0658                 record_and_restart(event, val, regs);
0659             } else {
0660                 /*
0661                  * Disabled counter is negative,
0662                  * reset it just in case.
0663                  */
0664                 write_pmc(i, 0);
0665             }
0666         }
0667     }
0668 
0669     /* PMM will keep counters frozen until we return from the interrupt. */
0670     mtmsr(mfmsr() | MSR_PMM);
0671     mtpmr(PMRN_PMGC0, PMGC0_PMIE | PMGC0_FCECE);
0672     isync();
0673 }
0674 
0675 void hw_perf_event_setup(int cpu)
0676 {
0677     struct cpu_hw_events *cpuhw = &per_cpu(cpu_hw_events, cpu);
0678 
0679     memset(cpuhw, 0, sizeof(*cpuhw));
0680 }
0681 
0682 int register_fsl_emb_pmu(struct fsl_emb_pmu *pmu)
0683 {
0684     if (ppmu)
0685         return -EBUSY;      /* something's already registered */
0686 
0687     ppmu = pmu;
0688     pr_info("%s performance monitor hardware support registered\n",
0689         pmu->name);
0690 
0691     perf_pmu_register(&fsl_emb_pmu, "cpu", PERF_TYPE_RAW);
0692 
0693     return 0;
0694 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!