pmu/ebb/ebb.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Copyright 2014, Michael Ellerman, IBM Corp.
0004  */
0005
0006 #define _GNU_SOURCE /* For CPU_ZERO etc. */
0007
0008 #include <sched.h>
0009 #include <sys/wait.h>
0010 #include <setjmp.h>
0011 #include <signal.h>
0012 #include <stdio.h>
0013 #include <stdlib.h>
0014 #include <string.h>
0015 #include <sys/ioctl.h>
0016
0017 #include "trace.h"
0018 #include "ebb.h"
0019
0020
0021 void (*ebb_user_func)(void);
0022
0023 void ebb_hook(void)
0024 {
0025     if (ebb_user_func)
0026         ebb_user_func();
0027 }
0028
0029 struct ebb_state ebb_state;
0030
0031 u64 sample_period = 0x40000000ull;
0032
0033 void reset_ebb_with_clear_mask(unsigned long mmcr0_clear_mask)
0034 {
0035     u64 val;
0036
0037     /* 2) clear MMCR0[PMAO] - docs say BESCR[PMEO] should do this */
0038     /* 3) set MMCR0[PMAE]   - docs say BESCR[PME] should do this */
0039     val = mfspr(SPRN_MMCR0);
0040     mtspr(SPRN_MMCR0, (val & ~mmcr0_clear_mask) | MMCR0_PMAE);
0041
0042     /* 4) clear BESCR[PMEO] */
0043     mtspr(SPRN_BESCRR, BESCR_PMEO);
0044
0045     /* 5) set BESCR[PME] */
0046     mtspr(SPRN_BESCRS, BESCR_PME);
0047
0048     /* 6) rfebb 1 - done in our caller */
0049 }
0050
0051 void reset_ebb(void)
0052 {
0053     reset_ebb_with_clear_mask(MMCR0_PMAO | MMCR0_FC);
0054 }
0055
0056 /* Called outside of the EBB handler to check MMCR0 is sane */
0057 int ebb_check_mmcr0(void)
0058 {
0059     u64 val;
0060
0061     val = mfspr(SPRN_MMCR0);
0062     if ((val & (MMCR0_FC | MMCR0_PMAO)) == MMCR0_FC) {
0063         /* It's OK if we see FC & PMAO, but not FC by itself */
0064         printf("Outside of loop, only FC set 0x%llx\n", val);
0065         return 1;
0066     }
0067
0068     return 0;
0069 }
0070
0071 bool ebb_check_count(int pmc, u64 sample_period, int fudge)
0072 {
0073     u64 count, upper, lower;
0074
0075     count = ebb_state.stats.pmc_count[PMC_INDEX(pmc)];
0076
0077     lower = ebb_state.stats.ebb_count * (sample_period - fudge);
0078
0079     if (count < lower) {
0080         printf("PMC%d count (0x%llx) below lower limit 0x%llx (-0x%llx)\n",
0081             pmc, count, lower, lower - count);
0082         return false;
0083     }
0084
0085     upper = ebb_state.stats.ebb_count * (sample_period + fudge);
0086
0087     if (count > upper) {
0088         printf("PMC%d count (0x%llx) above upper limit 0x%llx (+0x%llx)\n",
0089             pmc, count, upper, count - upper);
0090         return false;
0091     }
0092
0093     printf("PMC%d count (0x%llx) is between 0x%llx and 0x%llx delta +0x%llx/-0x%llx\n",
0094         pmc, count, lower, upper, count - lower, upper - count);
0095
0096     return true;
0097 }
0098
0099 void standard_ebb_callee(void)
0100 {
0101     int found, i;
0102     u64 val;
0103
0104     val = mfspr(SPRN_BESCR);
0105     if (!(val & BESCR_PMEO)) {
0106         ebb_state.stats.spurious++;
0107         goto out;
0108     }
0109
0110     ebb_state.stats.ebb_count++;
0111     trace_log_counter(ebb_state.trace, ebb_state.stats.ebb_count);
0112
0113     val = mfspr(SPRN_MMCR0);
0114     trace_log_reg(ebb_state.trace, SPRN_MMCR0, val);
0115
0116     found = 0;
0117     for (i = 1; i <= 6; i++) {
0118         if (ebb_state.pmc_enable[PMC_INDEX(i)])
0119             found += count_pmc(i, sample_period);
0120     }
0121
0122     if (!found)
0123         ebb_state.stats.no_overflow++;
0124
0125 out:
0126     reset_ebb();
0127 }
0128
0129 extern void ebb_handler(void);
0130
0131 void setup_ebb_handler(void (*callee)(void))
0132 {
0133     u64 entry;
0134
0135 #if defined(_CALL_ELF) && _CALL_ELF == 2
0136     entry = (u64)ebb_handler;
0137 #else
0138     struct opd
0139     {
0140         u64 entry;
0141         u64 toc;
0142     } *opd;
0143
0144     opd = (struct opd *)ebb_handler;
0145     entry = opd->entry;
0146 #endif
0147     printf("EBB Handler is at %#llx\n", entry);
0148
0149     ebb_user_func = callee;
0150
0151     /* Ensure ebb_user_func is set before we set the handler */
0152     mb();
0153     mtspr(SPRN_EBBHR, entry);
0154
0155     /* Make sure the handler is set before we return */
0156     mb();
0157 }
0158
0159 void clear_ebb_stats(void)
0160 {
0161     memset(&ebb_state.stats, 0, sizeof(ebb_state.stats));
0162 }
0163
0164 void dump_summary_ebb_state(void)
0165 {
0166     printf("ebb_state:\n"           \
0167            "  ebb_count    = %d\n"      \
0168            "  spurious     = %d\n"      \
0169            "  negative     = %d\n"      \
0170            "  no_overflow  = %d\n"      \
0171            "  pmc[1] count = 0x%llx\n"  \
0172            "  pmc[2] count = 0x%llx\n"  \
0173            "  pmc[3] count = 0x%llx\n"  \
0174            "  pmc[4] count = 0x%llx\n"  \
0175            "  pmc[5] count = 0x%llx\n"  \
0176            "  pmc[6] count = 0x%llx\n",
0177         ebb_state.stats.ebb_count, ebb_state.stats.spurious,
0178         ebb_state.stats.negative, ebb_state.stats.no_overflow,
0179         ebb_state.stats.pmc_count[0], ebb_state.stats.pmc_count[1],
0180         ebb_state.stats.pmc_count[2], ebb_state.stats.pmc_count[3],
0181         ebb_state.stats.pmc_count[4], ebb_state.stats.pmc_count[5]);
0182 }
0183
0184 static char *decode_mmcr0(u32 value)
0185 {
0186     static char buf[16];
0187
0188     buf[0] = '\0';
0189
0190     if (value & (1 << 31))
0191         strcat(buf, "FC ");
0192     if (value & (1 << 26))
0193         strcat(buf, "PMAE ");
0194     if (value & (1 << 7))
0195         strcat(buf, "PMAO ");
0196
0197     return buf;
0198 }
0199
0200 static char *decode_bescr(u64 value)
0201 {
0202     static char buf[16];
0203
0204     buf[0] = '\0';
0205
0206     if (value & (1ull << 63))
0207         strcat(buf, "GE ");
0208     if (value & (1ull << 32))
0209         strcat(buf, "PMAE ");
0210     if (value & 1)
0211         strcat(buf, "PMAO ");
0212
0213     return buf;
0214 }
0215
0216 void dump_ebb_hw_state(void)
0217 {
0218     u64 bescr;
0219     u32 mmcr0;
0220
0221     mmcr0 = mfspr(SPRN_MMCR0);
0222     bescr = mfspr(SPRN_BESCR);
0223
0224     printf("HW state:\n"        \
0225            "MMCR0 0x%016x %s\n" \
0226            "MMCR2 0x%016lx\n"   \
0227            "EBBHR 0x%016lx\n"   \
0228            "BESCR 0x%016llx %s\n"   \
0229            "PMC1  0x%016lx\n"   \
0230            "PMC2  0x%016lx\n"   \
0231            "PMC3  0x%016lx\n"   \
0232            "PMC4  0x%016lx\n"   \
0233            "PMC5  0x%016lx\n"   \
0234            "PMC6  0x%016lx\n"   \
0235            "SIAR  0x%016lx\n",
0236            mmcr0, decode_mmcr0(mmcr0), mfspr(SPRN_MMCR2),
0237            mfspr(SPRN_EBBHR), bescr, decode_bescr(bescr),
0238            mfspr(SPRN_PMC1), mfspr(SPRN_PMC2), mfspr(SPRN_PMC3),
0239            mfspr(SPRN_PMC4), mfspr(SPRN_PMC5), mfspr(SPRN_PMC6),
0240            mfspr(SPRN_SIAR));
0241 }
0242
0243 void dump_ebb_state(void)
0244 {
0245     dump_summary_ebb_state();
0246
0247     dump_ebb_hw_state();
0248
0249     trace_buffer_print(ebb_state.trace);
0250 }
0251
0252 int count_pmc(int pmc, uint32_t sample_period)
0253 {
0254     uint32_t start_value;
0255     u64 val;
0256
0257     /* 0) Read PMC */
0258     start_value = pmc_sample_period(sample_period);
0259
0260     val = read_pmc(pmc);
0261     if (val < start_value)
0262         ebb_state.stats.negative++;
0263     else
0264         ebb_state.stats.pmc_count[PMC_INDEX(pmc)] += val - start_value;
0265
0266     trace_log_reg(ebb_state.trace, SPRN_PMC1 + pmc - 1, val);
0267
0268     /* 1) Reset PMC */
0269     write_pmc(pmc, start_value);
0270
0271     /* Report if we overflowed */
0272     return val >= COUNTER_OVERFLOW;
0273 }
0274
0275 int ebb_event_enable(struct event *e)
0276 {
0277     int rc;
0278
0279     /* Ensure any SPR writes are ordered vs us */
0280     mb();
0281
0282     rc = ioctl(e->fd, PERF_EVENT_IOC_ENABLE);
0283     if (rc)
0284         return rc;
0285
0286     rc = event_read(e);
0287
0288     /* Ditto */
0289     mb();
0290
0291     return rc;
0292 }
0293
0294 void ebb_freeze_pmcs(void)
0295 {
0296     mtspr(SPRN_MMCR0, mfspr(SPRN_MMCR0) | MMCR0_FC);
0297     mb();
0298 }
0299
0300 void ebb_unfreeze_pmcs(void)
0301 {
0302     /* Unfreeze counters */
0303     mtspr(SPRN_MMCR0, mfspr(SPRN_MMCR0) & ~MMCR0_FC);
0304     mb();
0305 }
0306
0307 void ebb_global_enable(void)
0308 {
0309     /* Enable EBBs globally and PMU EBBs */
0310     mtspr(SPRN_BESCR, 0x8000000100000000ull);
0311     mb();
0312 }
0313
0314 void ebb_global_disable(void)
0315 {
0316     /* Disable EBBs & freeze counters, events are still scheduled */
0317     mtspr(SPRN_BESCRR, BESCR_PME);
0318     mb();
0319 }
0320
0321 bool ebb_is_supported(void)
0322 {
0323 #ifdef PPC_FEATURE2_EBB
0324     /* EBB requires at least POWER8 */
0325     return have_hwcap2(PPC_FEATURE2_EBB);
0326 #else
0327     return false;
0328 #endif
0329 }
0330
0331 void event_ebb_init(struct event *e)
0332 {
0333     e->attr.config |= (1ull << 63);
0334 }
0335
0336 void event_bhrb_init(struct event *e, unsigned ifm)
0337 {
0338     e->attr.config |= (1ull << 62) | ((u64)ifm << 60);
0339 }
0340
0341 void event_leader_ebb_init(struct event *e)
0342 {
0343     event_ebb_init(e);
0344
0345     e->attr.exclusive = 1;
0346     e->attr.pinned = 1;
0347 }
0348
0349 int ebb_child(union pipe read_pipe, union pipe write_pipe)
0350 {
0351     struct event event;
0352     uint64_t val;
0353
0354     FAIL_IF(wait_for_parent(read_pipe));
0355
0356     event_init_named(&event, 0x1001e, "cycles");
0357     event_leader_ebb_init(&event);
0358
0359     event.attr.exclude_kernel = 1;
0360     event.attr.exclude_hv = 1;
0361     event.attr.exclude_idle = 1;
0362
0363     FAIL_IF(event_open(&event));
0364
0365     ebb_enable_pmc_counting(1);
0366     setup_ebb_handler(standard_ebb_callee);
0367     ebb_global_enable();
0368
0369     FAIL_IF(event_enable(&event));
0370
0371     if (event_read(&event)) {
0372         /*
0373          * Some tests expect to fail here, so don't report an error on
0374          * this line, and return a distinguisable error code. Tell the
0375          * parent an error happened.
0376          */
0377         notify_parent_of_error(write_pipe);
0378         return 2;
0379     }
0380
0381     mtspr(SPRN_PMC1, pmc_sample_period(sample_period));
0382
0383     FAIL_IF(notify_parent(write_pipe));
0384     FAIL_IF(wait_for_parent(read_pipe));
0385     FAIL_IF(notify_parent(write_pipe));
0386
0387     while (ebb_state.stats.ebb_count < 20) {
0388         FAIL_IF(core_busy_loop());
0389
0390         /* To try and hit SIGILL case */
0391         val  = mfspr(SPRN_MMCRA);
0392         val |= mfspr(SPRN_MMCR2);
0393         val |= mfspr(SPRN_MMCR0);
0394     }
0395
0396     ebb_global_disable();
0397     ebb_freeze_pmcs();
0398
0399     dump_ebb_state();
0400
0401     event_close(&event);
0402
0403     FAIL_IF(ebb_state.stats.ebb_count == 0);
0404
0405     return 0;
0406 }
0407
0408 static jmp_buf setjmp_env;
0409
0410 static void sigill_handler(int signal)
0411 {
0412     printf("Took sigill\n");
0413     longjmp(setjmp_env, 1);
0414 }
0415
0416 static struct sigaction sigill_action = {
0417     .sa_handler = sigill_handler,
0418 };
0419
0420 int catch_sigill(void (*func)(void))
0421 {
0422     if (sigaction(SIGILL, &sigill_action, NULL)) {
0423         perror("sigaction");
0424         return 1;
0425     }
0426
0427     if (setjmp(setjmp_env) == 0) {
0428         func();
0429         return 1;
0430     }
0431
0432     return 0;
0433 }
0434
0435 void write_pmc1(void)
0436 {
0437     mtspr(SPRN_PMC1, 0);
0438 }
0439
0440 void write_pmc(int pmc, u64 value)
0441 {
0442     switch (pmc) {
0443         case 1: mtspr(SPRN_PMC1, value); break;
0444         case 2: mtspr(SPRN_PMC2, value); break;
0445         case 3: mtspr(SPRN_PMC3, value); break;
0446         case 4: mtspr(SPRN_PMC4, value); break;
0447         case 5: mtspr(SPRN_PMC5, value); break;
0448         case 6: mtspr(SPRN_PMC6, value); break;
0449     }
0450 }
0451
0452 u64 read_pmc(int pmc)
0453 {
0454     switch (pmc) {
0455         case 1: return mfspr(SPRN_PMC1);
0456         case 2: return mfspr(SPRN_PMC2);
0457         case 3: return mfspr(SPRN_PMC3);
0458         case 4: return mfspr(SPRN_PMC4);
0459         case 5: return mfspr(SPRN_PMC5);
0460         case 6: return mfspr(SPRN_PMC6);
0461     }
0462
0463     return 0;
0464 }
0465
0466 static void term_handler(int signal)
0467 {
0468     dump_summary_ebb_state();
0469     dump_ebb_hw_state();
0470     abort();
0471 }
0472
0473 struct sigaction term_action = {
0474     .sa_handler = term_handler,
0475 };
0476
0477 static void __attribute__((constructor)) ebb_init(void)
0478 {
0479     clear_ebb_stats();
0480
0481     if (sigaction(SIGTERM, &term_action, NULL))
0482         perror("sigaction");
0483
0484     ebb_state.trace = trace_buffer_allocate(1 * 1024 * 1024);
0485 }