s390/kernel/perf_pai_crypto.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * Performance event support - Processor Activity Instrumentation Facility
0004  *
0005  *  Copyright IBM Corp. 2022
0006  *  Author(s): Thomas Richter <tmricht@linux.ibm.com>
0007  */
0008 #define KMSG_COMPONENT  "pai_crypto"
0009 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
0010
0011 #include <linux/kernel.h>
0012 #include <linux/kernel_stat.h>
0013 #include <linux/percpu.h>
0014 #include <linux/notifier.h>
0015 #include <linux/init.h>
0016 #include <linux/export.h>
0017 #include <linux/io.h>
0018 #include <linux/perf_event.h>
0019
0020 #include <asm/ctl_reg.h>
0021 #include <asm/pai.h>
0022 #include <asm/debug.h>
0023
0024 static debug_info_t *cfm_dbg;
0025 static unsigned int paicrypt_cnt;   /* Size of the mapped counter sets */
0026                     /* extracted with QPACI instruction */
0027
0028 DEFINE_STATIC_KEY_FALSE(pai_key);
0029
0030 struct pai_userdata {
0031     u16 num;
0032     u64 value;
0033 } __packed;
0034
0035 struct paicrypt_map {
0036     unsigned long *page;        /* Page for CPU to store counters */
0037     struct pai_userdata *save;  /* Page to store no-zero counters */
0038     unsigned int users;     /* # of PAI crypto users */
0039     unsigned int sampler;       /* # of PAI crypto samplers */
0040     unsigned int counter;       /* # of PAI crypto counters */
0041     struct perf_event *event;   /* Perf event for sampling */
0042 };
0043
0044 static DEFINE_PER_CPU(struct paicrypt_map, paicrypt_map);
0045
0046 /* Release the PMU if event is the last perf event */
0047 static DEFINE_MUTEX(pai_reserve_mutex);
0048
0049 /* Adjust usage counters and remove allocated memory when all users are
0050  * gone.
0051  */
0052 static void paicrypt_event_destroy(struct perf_event *event)
0053 {
0054     struct paicrypt_map *cpump = per_cpu_ptr(&paicrypt_map, event->cpu);
0055
0056     cpump->event = NULL;
0057     static_branch_dec(&pai_key);
0058     mutex_lock(&pai_reserve_mutex);
0059     if (event->attr.sample_period)
0060         cpump->sampler -= 1;
0061     else
0062         cpump->counter -= 1;
0063     debug_sprintf_event(cfm_dbg, 5, "%s event %#llx cpu %d"
0064                 " sampler %d counter %d\n", __func__,
0065                 event->attr.config, event->cpu, cpump->sampler,
0066                 cpump->counter);
0067     if (!cpump->counter && !cpump->sampler) {
0068         debug_sprintf_event(cfm_dbg, 4, "%s page %#lx save %p\n",
0069                     __func__, (unsigned long)cpump->page,
0070                     cpump->save);
0071         free_page((unsigned long)cpump->page);
0072         cpump->page = NULL;
0073         kvfree(cpump->save);
0074         cpump->save = NULL;
0075     }
0076     mutex_unlock(&pai_reserve_mutex);
0077 }
0078
0079 static u64 paicrypt_getctr(struct paicrypt_map *cpump, int nr, bool kernel)
0080 {
0081     if (kernel)
0082         nr += PAI_CRYPTO_MAXCTR;
0083     return cpump->page[nr];
0084 }
0085
0086 /* Read the counter values. Return value from location in CMP. For event
0087  * CRYPTO_ALL sum up all events.
0088  */
0089 static u64 paicrypt_getdata(struct perf_event *event, bool kernel)
0090 {
0091     struct paicrypt_map *cpump = this_cpu_ptr(&paicrypt_map);
0092     u64 sum = 0;
0093     int i;
0094
0095     if (event->attr.config != PAI_CRYPTO_BASE) {
0096         return paicrypt_getctr(cpump,
0097                        event->attr.config - PAI_CRYPTO_BASE,
0098                        kernel);
0099     }
0100
0101     for (i = 1; i <= paicrypt_cnt; i++) {
0102         u64 val = paicrypt_getctr(cpump, i, kernel);
0103
0104         if (!val)
0105             continue;
0106         sum += val;
0107     }
0108     return sum;
0109 }
0110
0111 static u64 paicrypt_getall(struct perf_event *event)
0112 {
0113     u64 sum = 0;
0114
0115     if (!event->attr.exclude_kernel)
0116         sum += paicrypt_getdata(event, true);
0117     if (!event->attr.exclude_user)
0118         sum += paicrypt_getdata(event, false);
0119
0120     return sum;
0121 }
0122
0123 /* Used to avoid races in checking concurrent access of counting and
0124  * sampling for crypto events
0125  *
0126  * Only one instance of event pai_crypto/CRYPTO_ALL/ for sampling is
0127  * allowed and when this event is running, no counting event is allowed.
0128  * Several counting events are allowed in parallel, but no sampling event
0129  * is allowed while one (or more) counting events are running.
0130  *
0131  * This function is called in process context and it is save to block.
0132  * When the event initialization functions fails, no other call back will
0133  * be invoked.
0134  *
0135  * Allocate the memory for the event.
0136  */
0137 static int paicrypt_busy(struct perf_event_attr *a, struct paicrypt_map *cpump)
0138 {
0139     unsigned int *use_ptr;
0140     int rc = 0;
0141
0142     mutex_lock(&pai_reserve_mutex);
0143     if (a->sample_period) {     /* Sampling requested */
0144         use_ptr = &cpump->sampler;
0145         if (cpump->counter || cpump->sampler)
0146             rc = -EBUSY;    /* ... sampling/counting active */
0147     } else {            /* Counting requested */
0148         use_ptr = &cpump->counter;
0149         if (cpump->sampler)
0150             rc = -EBUSY;    /* ... and sampling active */
0151     }
0152     if (rc)
0153         goto unlock;
0154
0155     /* Allocate memory for counter page and counter extraction.
0156      * Only the first counting event has to allocate a page.
0157      */
0158     if (cpump->page)
0159         goto unlock;
0160
0161     rc = -ENOMEM;
0162     cpump->page = (unsigned long *)get_zeroed_page(GFP_KERNEL);
0163     if (!cpump->page)
0164         goto unlock;
0165     cpump->save = kvmalloc_array(paicrypt_cnt + 1,
0166                      sizeof(struct pai_userdata), GFP_KERNEL);
0167     if (!cpump->save) {
0168         free_page((unsigned long)cpump->page);
0169         cpump->page = NULL;
0170         goto unlock;
0171     }
0172     rc = 0;
0173
0174 unlock:
0175     /* If rc is non-zero, do not increment counter/sampler. */
0176     if (!rc)
0177         *use_ptr += 1;
0178     debug_sprintf_event(cfm_dbg, 5, "%s sample_period %#llx sampler %d"
0179                 " counter %d page %#lx save %p rc %d\n", __func__,
0180                 a->sample_period, cpump->sampler, cpump->counter,
0181                 (unsigned long)cpump->page, cpump->save, rc);
0182     mutex_unlock(&pai_reserve_mutex);
0183     return rc;
0184 }
0185
0186 /* Might be called on different CPU than the one the event is intended for. */
0187 static int paicrypt_event_init(struct perf_event *event)
0188 {
0189     struct perf_event_attr *a = &event->attr;
0190     struct paicrypt_map *cpump;
0191     int rc;
0192
0193     /* PAI crypto PMU registered as PERF_TYPE_RAW, check event type */
0194     if (a->type != PERF_TYPE_RAW && event->pmu->type != a->type)
0195         return -ENOENT;
0196     /* PAI crypto event must be in valid range */
0197     if (a->config < PAI_CRYPTO_BASE ||
0198         a->config > PAI_CRYPTO_BASE + paicrypt_cnt)
0199         return -EINVAL;
0200     /* Allow only CPU wide operation, no process context for now. */
0201     if (event->hw.target || event->cpu == -1)
0202         return -ENOENT;
0203     /* Allow only CRYPTO_ALL for sampling. */
0204     if (a->sample_period && a->config != PAI_CRYPTO_BASE)
0205         return -EINVAL;
0206
0207     cpump = per_cpu_ptr(&paicrypt_map, event->cpu);
0208     rc = paicrypt_busy(a, cpump);
0209     if (rc)
0210         return rc;
0211
0212     /* Event initialization sets last_tag to 0. When later on the events
0213      * are deleted and re-added, do not reset the event count value to zero.
0214      * Events are added, deleted and re-added when 2 or more events
0215      * are active at the same time.
0216      */
0217     event->hw.last_tag = 0;
0218     cpump->event = event;
0219     event->destroy = paicrypt_event_destroy;
0220
0221     if (a->sample_period) {
0222         a->sample_period = 1;
0223         a->freq = 0;
0224         /* Register for paicrypt_sched_task() to be called */
0225         event->attach_state |= PERF_ATTACH_SCHED_CB;
0226         /* Add raw data which contain the memory mapped counters */
0227         a->sample_type |= PERF_SAMPLE_RAW;
0228         /* Turn off inheritance */
0229         a->inherit = 0;
0230     }
0231
0232     static_branch_inc(&pai_key);
0233     return 0;
0234 }
0235
0236 static void paicrypt_read(struct perf_event *event)
0237 {
0238     u64 prev, new, delta;
0239
0240     prev = local64_read(&event->hw.prev_count);
0241     new = paicrypt_getall(event);
0242     local64_set(&event->hw.prev_count, new);
0243     delta = (prev <= new) ? new - prev
0244                   : (-1ULL - prev) + new + 1;    /* overflow */
0245     local64_add(delta, &event->count);
0246 }
0247
0248 static void paicrypt_start(struct perf_event *event, int flags)
0249 {
0250     u64 sum;
0251
0252     if (!event->hw.last_tag) {
0253         event->hw.last_tag = 1;
0254         sum = paicrypt_getall(event);       /* Get current value */
0255         local64_set(&event->count, 0);
0256         local64_set(&event->hw.prev_count, sum);
0257     }
0258 }
0259
0260 static int paicrypt_add(struct perf_event *event, int flags)
0261 {
0262     struct paicrypt_map *cpump = this_cpu_ptr(&paicrypt_map);
0263     unsigned long ccd;
0264
0265     if (cpump->users++ == 0) {
0266         ccd = virt_to_phys(cpump->page) | PAI_CRYPTO_KERNEL_OFFSET;
0267         WRITE_ONCE(S390_lowcore.ccd, ccd);
0268         __ctl_set_bit(0, 50);
0269     }
0270     cpump->event = event;
0271     if (flags & PERF_EF_START && !event->attr.sample_period) {
0272         /* Only counting needs initial counter value */
0273         paicrypt_start(event, PERF_EF_RELOAD);
0274     }
0275     event->hw.state = 0;
0276     if (event->attr.sample_period)
0277         perf_sched_cb_inc(event->pmu);
0278     return 0;
0279 }
0280
0281 static void paicrypt_stop(struct perf_event *event, int flags)
0282 {
0283     paicrypt_read(event);
0284     event->hw.state = PERF_HES_STOPPED;
0285 }
0286
0287 static void paicrypt_del(struct perf_event *event, int flags)
0288 {
0289     struct paicrypt_map *cpump = this_cpu_ptr(&paicrypt_map);
0290
0291     if (event->attr.sample_period)
0292         perf_sched_cb_dec(event->pmu);
0293     if (!event->attr.sample_period)
0294         /* Only counting needs to read counter */
0295         paicrypt_stop(event, PERF_EF_UPDATE);
0296     if (cpump->users-- == 1) {
0297         __ctl_clear_bit(0, 50);
0298         WRITE_ONCE(S390_lowcore.ccd, 0);
0299     }
0300 }
0301
0302 /* Create raw data and save it in buffer. Returns number of bytes copied.
0303  * Saves only positive counter entries of the form
0304  * 2 bytes: Number of counter
0305  * 8 bytes: Value of counter
0306  */
0307 static size_t paicrypt_copy(struct pai_userdata *userdata,
0308                 struct paicrypt_map *cpump,
0309                 bool exclude_user, bool exclude_kernel)
0310 {
0311     int i, outidx = 0;
0312
0313     for (i = 1; i <= paicrypt_cnt; i++) {
0314         u64 val = 0;
0315
0316         if (!exclude_kernel)
0317             val += paicrypt_getctr(cpump, i, true);
0318         if (!exclude_user)
0319             val += paicrypt_getctr(cpump, i, false);
0320         if (val) {
0321             userdata[outidx].num = i;
0322             userdata[outidx].value = val;
0323             outidx++;
0324         }
0325     }
0326     return outidx * sizeof(struct pai_userdata);
0327 }
0328
0329 static int paicrypt_push_sample(void)
0330 {
0331     struct paicrypt_map *cpump = this_cpu_ptr(&paicrypt_map);
0332     struct perf_event *event = cpump->event;
0333     struct perf_sample_data data;
0334     struct perf_raw_record raw;
0335     struct pt_regs regs;
0336     size_t rawsize;
0337     int overflow;
0338
0339     if (!cpump->event)      /* No event active */
0340         return 0;
0341     rawsize = paicrypt_copy(cpump->save, cpump,
0342                 cpump->event->attr.exclude_user,
0343                 cpump->event->attr.exclude_kernel);
0344     if (!rawsize)           /* No incremented counters */
0345         return 0;
0346
0347     /* Setup perf sample */
0348     memset(&regs, 0, sizeof(regs));
0349     memset(&raw, 0, sizeof(raw));
0350     memset(&data, 0, sizeof(data));
0351     perf_sample_data_init(&data, 0, event->hw.last_period);
0352     if (event->attr.sample_type & PERF_SAMPLE_TID) {
0353         data.tid_entry.pid = task_tgid_nr(current);
0354         data.tid_entry.tid = task_pid_nr(current);
0355     }
0356     if (event->attr.sample_type & PERF_SAMPLE_TIME)
0357         data.time = event->clock();
0358     if (event->attr.sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER))
0359         data.id = event->id;
0360     if (event->attr.sample_type & PERF_SAMPLE_CPU) {
0361         data.cpu_entry.cpu = smp_processor_id();
0362         data.cpu_entry.reserved = 0;
0363     }
0364     if (event->attr.sample_type & PERF_SAMPLE_RAW) {
0365         raw.frag.size = rawsize;
0366         raw.frag.data = cpump->save;
0367         raw.size = raw.frag.size;
0368         data.raw = &raw;
0369     }
0370
0371     overflow = perf_event_overflow(event, &data, &regs);
0372     perf_event_update_userpage(event);
0373     /* Clear lowcore page after read */
0374     memset(cpump->page, 0, PAGE_SIZE);
0375     return overflow;
0376 }
0377
0378 /* Called on schedule-in and schedule-out. No access to event structure,
0379  * but for sampling only event CRYPTO_ALL is allowed.
0380  */
0381 static void paicrypt_sched_task(struct perf_event_context *ctx, bool sched_in)
0382 {
0383     /* We started with a clean page on event installation. So read out
0384      * results on schedule_out and if page was dirty, clear values.
0385      */
0386     if (!sched_in)
0387         paicrypt_push_sample();
0388 }
0389
0390 /* Attribute definitions for paicrypt interface. As with other CPU
0391  * Measurement Facilities, there is one attribute per mapped counter.
0392  * The number of mapped counters may vary per machine generation. Use
0393  * the QUERY PROCESSOR ACTIVITY COUNTER INFORMATION (QPACI) instruction
0394  * to determine the number of mapped counters. The instructions returns
0395  * a positive number, which is the highest number of supported counters.
0396  * All counters less than this number are also supported, there are no
0397  * holes. A returned number of zero means no support for mapped counters.
0398  *
0399  * The identification of the counter is a unique number. The chosen range
0400  * is 0x1000 + offset in mapped kernel page.
0401  * All CPU Measurement Facility counters identifiers must be unique and
0402  * the numbers from 0 to 496 are already used for the CPU Measurement
0403  * Counter facility. Numbers 0xb0000, 0xbc000 and 0xbd000 are already
0404  * used for the CPU Measurement Sampling facility.
0405  */
0406 PMU_FORMAT_ATTR(event, "config:0-63");
0407
0408 static struct attribute *paicrypt_format_attr[] = {
0409     &format_attr_event.attr,
0410     NULL,
0411 };
0412
0413 static struct attribute_group paicrypt_events_group = {
0414     .name = "events",
0415     .attrs = NULL           /* Filled in attr_event_init() */
0416 };
0417
0418 static struct attribute_group paicrypt_format_group = {
0419     .name = "format",
0420     .attrs = paicrypt_format_attr,
0421 };
0422
0423 static const struct attribute_group *paicrypt_attr_groups[] = {
0424     &paicrypt_events_group,
0425     &paicrypt_format_group,
0426     NULL,
0427 };
0428
0429 /* Performance monitoring unit for mapped counters */
0430 static struct pmu paicrypt = {
0431     .task_ctx_nr  = perf_invalid_context,
0432     .event_init   = paicrypt_event_init,
0433     .add          = paicrypt_add,
0434     .del          = paicrypt_del,
0435     .start        = paicrypt_start,
0436     .stop         = paicrypt_stop,
0437     .read         = paicrypt_read,
0438     .sched_task   = paicrypt_sched_task,
0439     .attr_groups  = paicrypt_attr_groups
0440 };
0441
0442 /* List of symbolic PAI counter names. */
0443 static const char * const paicrypt_ctrnames[] = {
0444     [0] = "CRYPTO_ALL",
0445     [1] = "KM_DEA",
0446     [2] = "KM_TDEA_128",
0447     [3] = "KM_TDEA_192",
0448     [4] = "KM_ENCRYPTED_DEA",
0449     [5] = "KM_ENCRYPTED_TDEA_128",
0450     [6] = "KM_ENCRYPTED_TDEA_192",
0451     [7] = "KM_AES_128",
0452     [8] = "KM_AES_192",
0453     [9] = "KM_AES_256",
0454     [10] = "KM_ENCRYPTED_AES_128",
0455     [11] = "KM_ENCRYPTED_AES_192",
0456     [12] = "KM_ENCRYPTED_AES_256",
0457     [13] = "KM_XTS_AES_128",
0458     [14] = "KM_XTS_AES_256",
0459     [15] = "KM_XTS_ENCRYPTED_AES_128",
0460     [16] = "KM_XTS_ENCRYPTED_AES_256",
0461     [17] = "KMC_DEA",
0462     [18] = "KMC_TDEA_128",
0463     [19] = "KMC_TDEA_192",
0464     [20] = "KMC_ENCRYPTED_DEA",
0465     [21] = "KMC_ENCRYPTED_TDEA_128",
0466     [22] = "KMC_ENCRYPTED_TDEA_192",
0467     [23] = "KMC_AES_128",
0468     [24] = "KMC_AES_192",
0469     [25] = "KMC_AES_256",
0470     [26] = "KMC_ENCRYPTED_AES_128",
0471     [27] = "KMC_ENCRYPTED_AES_192",
0472     [28] = "KMC_ENCRYPTED_AES_256",
0473     [29] = "KMC_PRNG",
0474     [30] = "KMA_GCM_AES_128",
0475     [31] = "KMA_GCM_AES_192",
0476     [32] = "KMA_GCM_AES_256",
0477     [33] = "KMA_GCM_ENCRYPTED_AES_128",
0478     [34] = "KMA_GCM_ENCRYPTED_AES_192",
0479     [35] = "KMA_GCM_ENCRYPTED_AES_256",
0480     [36] = "KMF_DEA",
0481     [37] = "KMF_TDEA_128",
0482     [38] = "KMF_TDEA_192",
0483     [39] = "KMF_ENCRYPTED_DEA",
0484     [40] = "KMF_ENCRYPTED_TDEA_128",
0485     [41] = "KMF_ENCRYPTED_TDEA_192",
0486     [42] = "KMF_AES_128",
0487     [43] = "KMF_AES_192",
0488     [44] = "KMF_AES_256",
0489     [45] = "KMF_ENCRYPTED_AES_128",
0490     [46] = "KMF_ENCRYPTED_AES_192",
0491     [47] = "KMF_ENCRYPTED_AES_256",
0492     [48] = "KMCTR_DEA",
0493     [49] = "KMCTR_TDEA_128",
0494     [50] = "KMCTR_TDEA_192",
0495     [51] = "KMCTR_ENCRYPTED_DEA",
0496     [52] = "KMCTR_ENCRYPTED_TDEA_128",
0497     [53] = "KMCTR_ENCRYPTED_TDEA_192",
0498     [54] = "KMCTR_AES_128",
0499     [55] = "KMCTR_AES_192",
0500     [56] = "KMCTR_AES_256",
0501     [57] = "KMCTR_ENCRYPTED_AES_128",
0502     [58] = "KMCTR_ENCRYPTED_AES_192",
0503     [59] = "KMCTR_ENCRYPTED_AES_256",
0504     [60] = "KMO_DEA",
0505     [61] = "KMO_TDEA_128",
0506     [62] = "KMO_TDEA_192",
0507     [63] = "KMO_ENCRYPTED_DEA",
0508     [64] = "KMO_ENCRYPTED_TDEA_128",
0509     [65] = "KMO_ENCRYPTED_TDEA_192",
0510     [66] = "KMO_AES_128",
0511     [67] = "KMO_AES_192",
0512     [68] = "KMO_AES_256",
0513     [69] = "KMO_ENCRYPTED_AES_128",
0514     [70] = "KMO_ENCRYPTED_AES_192",
0515     [71] = "KMO_ENCRYPTED_AES_256",
0516     [72] = "KIMD_SHA_1",
0517     [73] = "KIMD_SHA_256",
0518     [74] = "KIMD_SHA_512",
0519     [75] = "KIMD_SHA3_224",
0520     [76] = "KIMD_SHA3_256",
0521     [77] = "KIMD_SHA3_384",
0522     [78] = "KIMD_SHA3_512",
0523     [79] = "KIMD_SHAKE_128",
0524     [80] = "KIMD_SHAKE_256",
0525     [81] = "KIMD_GHASH",
0526     [82] = "KLMD_SHA_1",
0527     [83] = "KLMD_SHA_256",
0528     [84] = "KLMD_SHA_512",
0529     [85] = "KLMD_SHA3_224",
0530     [86] = "KLMD_SHA3_256",
0531     [87] = "KLMD_SHA3_384",
0532     [88] = "KLMD_SHA3_512",
0533     [89] = "KLMD_SHAKE_128",
0534     [90] = "KLMD_SHAKE_256",
0535     [91] = "KMAC_DEA",
0536     [92] = "KMAC_TDEA_128",
0537     [93] = "KMAC_TDEA_192",
0538     [94] = "KMAC_ENCRYPTED_DEA",
0539     [95] = "KMAC_ENCRYPTED_TDEA_128",
0540     [96] = "KMAC_ENCRYPTED_TDEA_192",
0541     [97] = "KMAC_AES_128",
0542     [98] = "KMAC_AES_192",
0543     [99] = "KMAC_AES_256",
0544     [100] = "KMAC_ENCRYPTED_AES_128",
0545     [101] = "KMAC_ENCRYPTED_AES_192",
0546     [102] = "KMAC_ENCRYPTED_AES_256",
0547     [103] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_DEA",
0548     [104] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_TDEA_128",
0549     [105] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_TDEA_192",
0550     [106] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_DEA",
0551     [107] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_TDEA_128",
0552     [108] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_TDEA_192",
0553     [109] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_AES_128",
0554     [110] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_AES_192",
0555     [111] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_AES_256",
0556     [112] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_AES_128",
0557     [113] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_AES_192",
0558     [114] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_AES_256A",
0559     [115] = "PCC_COMPUTE_XTS_PARAMETER_USING_AES_128",
0560     [116] = "PCC_COMPUTE_XTS_PARAMETER_USING_AES_256",
0561     [117] = "PCC_COMPUTE_XTS_PARAMETER_USING_ENCRYPTED_AES_128",
0562     [118] = "PCC_COMPUTE_XTS_PARAMETER_USING_ENCRYPTED_AES_256",
0563     [119] = "PCC_SCALAR_MULTIPLY_P256",
0564     [120] = "PCC_SCALAR_MULTIPLY_P384",
0565     [121] = "PCC_SCALAR_MULTIPLY_P521",
0566     [122] = "PCC_SCALAR_MULTIPLY_ED25519",
0567     [123] = "PCC_SCALAR_MULTIPLY_ED448",
0568     [124] = "PCC_SCALAR_MULTIPLY_X25519",
0569     [125] = "PCC_SCALAR_MULTIPLY_X448",
0570     [126] = "PRNO_SHA_512_DRNG",
0571     [127] = "PRNO_TRNG_QUERY_RAW_TO_CONDITIONED_RATIO",
0572     [128] = "PRNO_TRNG",
0573     [129] = "KDSA_ECDSA_VERIFY_P256",
0574     [130] = "KDSA_ECDSA_VERIFY_P384",
0575     [131] = "KDSA_ECDSA_VERIFY_P521",
0576     [132] = "KDSA_ECDSA_SIGN_P256",
0577     [133] = "KDSA_ECDSA_SIGN_P384",
0578     [134] = "KDSA_ECDSA_SIGN_P521",
0579     [135] = "KDSA_ENCRYPTED_ECDSA_SIGN_P256",
0580     [136] = "KDSA_ENCRYPTED_ECDSA_SIGN_P384",
0581     [137] = "KDSA_ENCRYPTED_ECDSA_SIGN_P521",
0582     [138] = "KDSA_EDDSA_VERIFY_ED25519",
0583     [139] = "KDSA_EDDSA_VERIFY_ED448",
0584     [140] = "KDSA_EDDSA_SIGN_ED25519",
0585     [141] = "KDSA_EDDSA_SIGN_ED448",
0586     [142] = "KDSA_ENCRYPTED_EDDSA_SIGN_ED25519",
0587     [143] = "KDSA_ENCRYPTED_EDDSA_SIGN_ED448",
0588     [144] = "PCKMO_ENCRYPT_DEA_KEY",
0589     [145] = "PCKMO_ENCRYPT_TDEA_128_KEY",
0590     [146] = "PCKMO_ENCRYPT_TDEA_192_KEY",
0591     [147] = "PCKMO_ENCRYPT_AES_128_KEY",
0592     [148] = "PCKMO_ENCRYPT_AES_192_KEY",
0593     [149] = "PCKMO_ENCRYPT_AES_256_KEY",
0594     [150] = "PCKMO_ENCRYPT_ECC_P256_KEY",
0595     [151] = "PCKMO_ENCRYPT_ECC_P384_KEY",
0596     [152] = "PCKMO_ENCRYPT_ECC_P521_KEY",
0597     [153] = "PCKMO_ENCRYPT_ECC_ED25519_KEY",
0598     [154] = "PCKMO_ENCRYPT_ECC_ED448_KEY",
0599     [155] = "IBM_RESERVED_155",
0600     [156] = "IBM_RESERVED_156",
0601 };
0602
0603 static void __init attr_event_free(struct attribute **attrs, int num)
0604 {
0605     struct perf_pmu_events_attr *pa;
0606     int i;
0607
0608     for (i = 0; i < num; i++) {
0609         struct device_attribute *dap;
0610
0611         dap = container_of(attrs[i], struct device_attribute, attr);
0612         pa = container_of(dap, struct perf_pmu_events_attr, attr);
0613         kfree(pa);
0614     }
0615     kfree(attrs);
0616 }
0617
0618 static int __init attr_event_init_one(struct attribute **attrs, int num)
0619 {
0620     struct perf_pmu_events_attr *pa;
0621
0622     pa = kzalloc(sizeof(*pa), GFP_KERNEL);
0623     if (!pa)
0624         return -ENOMEM;
0625
0626     sysfs_attr_init(&pa->attr.attr);
0627     pa->id = PAI_CRYPTO_BASE + num;
0628     pa->attr.attr.name = paicrypt_ctrnames[num];
0629     pa->attr.attr.mode = 0444;
0630     pa->attr.show = cpumf_events_sysfs_show;
0631     pa->attr.store = NULL;
0632     attrs[num] = &pa->attr.attr;
0633     return 0;
0634 }
0635
0636 /* Create PMU sysfs event attributes on the fly. */
0637 static int __init attr_event_init(void)
0638 {
0639     struct attribute **attrs;
0640     int ret, i;
0641
0642     attrs = kmalloc_array(ARRAY_SIZE(paicrypt_ctrnames) + 1, sizeof(*attrs),
0643                   GFP_KERNEL);
0644     if (!attrs)
0645         return -ENOMEM;
0646     for (i = 0; i < ARRAY_SIZE(paicrypt_ctrnames); i++) {
0647         ret = attr_event_init_one(attrs, i);
0648         if (ret) {
0649             attr_event_free(attrs, i - 1);
0650             return ret;
0651         }
0652     }
0653     attrs[i] = NULL;
0654     paicrypt_events_group.attrs = attrs;
0655     return 0;
0656 }
0657
0658 static int __init paicrypt_init(void)
0659 {
0660     struct qpaci_info_block ib;
0661     int rc;
0662
0663     if (!test_facility(196))
0664         return 0;
0665
0666     qpaci(&ib);
0667     paicrypt_cnt = ib.num_cc;
0668     if (paicrypt_cnt == 0)
0669         return 0;
0670     if (paicrypt_cnt >= PAI_CRYPTO_MAXCTR)
0671         paicrypt_cnt = PAI_CRYPTO_MAXCTR - 1;
0672
0673     rc = attr_event_init();     /* Export known PAI crypto events */
0674     if (rc) {
0675         pr_err("Creation of PMU pai_crypto /sysfs failed\n");
0676         return rc;
0677     }
0678
0679     /* Setup s390dbf facility */
0680     cfm_dbg = debug_register(KMSG_COMPONENT, 2, 256, 128);
0681     if (!cfm_dbg) {
0682         pr_err("Registration of s390dbf pai_crypto failed\n");
0683         return -ENOMEM;
0684     }
0685     debug_register_view(cfm_dbg, &debug_sprintf_view);
0686
0687     rc = perf_pmu_register(&paicrypt, "pai_crypto", -1);
0688     if (rc) {
0689         pr_err("Registering the pai_crypto PMU failed with rc=%i\n",
0690                rc);
0691         debug_unregister_view(cfm_dbg, &debug_sprintf_view);
0692         debug_unregister(cfm_dbg);
0693         return rc;
0694     }
0695     return 0;
0696 }
0697
0698 device_initcall(paicrypt_init);