kernel/trace/trace_events_user.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Copyright (c) 2021, Microsoft Corporation.
0004  *
0005  * Authors:
0006  *   Beau Belgrave <beaub@linux.microsoft.com>
0007  */
0008
0009 #include <linux/bitmap.h>
0010 #include <linux/cdev.h>
0011 #include <linux/hashtable.h>
0012 #include <linux/list.h>
0013 #include <linux/io.h>
0014 #include <linux/uio.h>
0015 #include <linux/ioctl.h>
0016 #include <linux/jhash.h>
0017 #include <linux/trace_events.h>
0018 #include <linux/tracefs.h>
0019 #include <linux/types.h>
0020 #include <linux/uaccess.h>
0021 /* Reminder to move to uapi when everything works */
0022 #ifdef CONFIG_COMPILE_TEST
0023 #include <linux/user_events.h>
0024 #else
0025 #include <uapi/linux/user_events.h>
0026 #endif
0027 #include "trace.h"
0028 #include "trace_dynevent.h"
0029
0030 #define USER_EVENTS_PREFIX_LEN (sizeof(USER_EVENTS_PREFIX)-1)
0031
0032 #define FIELD_DEPTH_TYPE 0
0033 #define FIELD_DEPTH_NAME 1
0034 #define FIELD_DEPTH_SIZE 2
0035
0036 /*
0037  * Limits how many trace_event calls user processes can create:
0038  * Must be a power of two of PAGE_SIZE.
0039  */
0040 #define MAX_PAGE_ORDER 0
0041 #define MAX_PAGES (1 << MAX_PAGE_ORDER)
0042 #define MAX_EVENTS (MAX_PAGES * PAGE_SIZE)
0043
0044 /* Limit how long of an event name plus args within the subsystem. */
0045 #define MAX_EVENT_DESC 512
0046 #define EVENT_NAME(user_event) ((user_event)->tracepoint.name)
0047 #define MAX_FIELD_ARRAY_SIZE 1024
0048 #define MAX_FIELD_ARG_NAME 256
0049
0050 static char *register_page_data;
0051
0052 static DEFINE_MUTEX(reg_mutex);
0053 static DEFINE_HASHTABLE(register_table, 4);
0054 static DECLARE_BITMAP(page_bitmap, MAX_EVENTS);
0055
0056 /*
0057  * Stores per-event properties, as users register events
0058  * within a file a user_event might be created if it does not
0059  * already exist. These are globally used and their lifetime
0060  * is tied to the refcnt member. These cannot go away until the
0061  * refcnt reaches zero.
0062  */
0063 struct user_event {
0064     struct tracepoint tracepoint;
0065     struct trace_event_call call;
0066     struct trace_event_class class;
0067     struct dyn_event devent;
0068     struct hlist_node node;
0069     struct list_head fields;
0070     struct list_head validators;
0071     atomic_t refcnt;
0072     int index;
0073     int flags;
0074     int min_size;
0075 };
0076
0077 /*
0078  * Stores per-file events references, as users register events
0079  * within a file this structure is modified and freed via RCU.
0080  * The lifetime of this struct is tied to the lifetime of the file.
0081  * These are not shared and only accessible by the file that created it.
0082  */
0083 struct user_event_refs {
0084     struct rcu_head rcu;
0085     int count;
0086     struct user_event *events[];
0087 };
0088
0089 #define VALIDATOR_ENSURE_NULL (1 << 0)
0090 #define VALIDATOR_REL (1 << 1)
0091
0092 struct user_event_validator {
0093     struct list_head link;
0094     int offset;
0095     int flags;
0096 };
0097
0098 typedef void (*user_event_func_t) (struct user_event *user, struct iov_iter *i,
0099                    void *tpdata, bool *faulted);
0100
0101 static int user_event_parse(char *name, char *args, char *flags,
0102                 struct user_event **newuser);
0103
0104 static u32 user_event_key(char *name)
0105 {
0106     return jhash(name, strlen(name), 0);
0107 }
0108
0109 static __always_inline __must_check
0110 size_t copy_nofault(void *addr, size_t bytes, struct iov_iter *i)
0111 {
0112     size_t ret;
0113
0114     pagefault_disable();
0115
0116     ret = copy_from_iter_nocache(addr, bytes, i);
0117
0118     pagefault_enable();
0119
0120     return ret;
0121 }
0122
0123 static struct list_head *user_event_get_fields(struct trace_event_call *call)
0124 {
0125     struct user_event *user = (struct user_event *)call->data;
0126
0127     return &user->fields;
0128 }
0129
0130 /*
0131  * Parses a register command for user_events
0132  * Format: event_name[:FLAG1[,FLAG2...]] [field1[;field2...]]
0133  *
0134  * Example event named 'test' with a 20 char 'msg' field with an unsigned int
0135  * 'id' field after:
0136  * test char[20] msg;unsigned int id
0137  *
0138  * NOTE: Offsets are from the user data perspective, they are not from the
0139  * trace_entry/buffer perspective. We automatically add the common properties
0140  * sizes to the offset for the user.
0141  *
0142  * Upon success user_event has its ref count increased by 1.
0143  */
0144 static int user_event_parse_cmd(char *raw_command, struct user_event **newuser)
0145 {
0146     char *name = raw_command;
0147     char *args = strpbrk(name, " ");
0148     char *flags;
0149
0150     if (args)
0151         *args++ = '\0';
0152
0153     flags = strpbrk(name, ":");
0154
0155     if (flags)
0156         *flags++ = '\0';
0157
0158     return user_event_parse(name, args, flags, newuser);
0159 }
0160
0161 static int user_field_array_size(const char *type)
0162 {
0163     const char *start = strchr(type, '[');
0164     char val[8];
0165     char *bracket;
0166     int size = 0;
0167
0168     if (start == NULL)
0169         return -EINVAL;
0170
0171     if (strscpy(val, start + 1, sizeof(val)) <= 0)
0172         return -EINVAL;
0173
0174     bracket = strchr(val, ']');
0175
0176     if (!bracket)
0177         return -EINVAL;
0178
0179     *bracket = '\0';
0180
0181     if (kstrtouint(val, 0, &size))
0182         return -EINVAL;
0183
0184     if (size > MAX_FIELD_ARRAY_SIZE)
0185         return -EINVAL;
0186
0187     return size;
0188 }
0189
0190 static int user_field_size(const char *type)
0191 {
0192     /* long is not allowed from a user, since it's ambigious in size */
0193     if (strcmp(type, "s64") == 0)
0194         return sizeof(s64);
0195     if (strcmp(type, "u64") == 0)
0196         return sizeof(u64);
0197     if (strcmp(type, "s32") == 0)
0198         return sizeof(s32);
0199     if (strcmp(type, "u32") == 0)
0200         return sizeof(u32);
0201     if (strcmp(type, "int") == 0)
0202         return sizeof(int);
0203     if (strcmp(type, "unsigned int") == 0)
0204         return sizeof(unsigned int);
0205     if (strcmp(type, "s16") == 0)
0206         return sizeof(s16);
0207     if (strcmp(type, "u16") == 0)
0208         return sizeof(u16);
0209     if (strcmp(type, "short") == 0)
0210         return sizeof(short);
0211     if (strcmp(type, "unsigned short") == 0)
0212         return sizeof(unsigned short);
0213     if (strcmp(type, "s8") == 0)
0214         return sizeof(s8);
0215     if (strcmp(type, "u8") == 0)
0216         return sizeof(u8);
0217     if (strcmp(type, "char") == 0)
0218         return sizeof(char);
0219     if (strcmp(type, "unsigned char") == 0)
0220         return sizeof(unsigned char);
0221     if (str_has_prefix(type, "char["))
0222         return user_field_array_size(type);
0223     if (str_has_prefix(type, "unsigned char["))
0224         return user_field_array_size(type);
0225     if (str_has_prefix(type, "__data_loc "))
0226         return sizeof(u32);
0227     if (str_has_prefix(type, "__rel_loc "))
0228         return sizeof(u32);
0229
0230     /* Uknown basic type, error */
0231     return -EINVAL;
0232 }
0233
0234 static void user_event_destroy_validators(struct user_event *user)
0235 {
0236     struct user_event_validator *validator, *next;
0237     struct list_head *head = &user->validators;
0238
0239     list_for_each_entry_safe(validator, next, head, link) {
0240         list_del(&validator->link);
0241         kfree(validator);
0242     }
0243 }
0244
0245 static void user_event_destroy_fields(struct user_event *user)
0246 {
0247     struct ftrace_event_field *field, *next;
0248     struct list_head *head = &user->fields;
0249
0250     list_for_each_entry_safe(field, next, head, link) {
0251         list_del(&field->link);
0252         kfree(field);
0253     }
0254 }
0255
0256 static int user_event_add_field(struct user_event *user, const char *type,
0257                 const char *name, int offset, int size,
0258                 int is_signed, int filter_type)
0259 {
0260     struct user_event_validator *validator;
0261     struct ftrace_event_field *field;
0262     int validator_flags = 0;
0263
0264     field = kmalloc(sizeof(*field), GFP_KERNEL);
0265
0266     if (!field)
0267         return -ENOMEM;
0268
0269     if (str_has_prefix(type, "__data_loc "))
0270         goto add_validator;
0271
0272     if (str_has_prefix(type, "__rel_loc ")) {
0273         validator_flags |= VALIDATOR_REL;
0274         goto add_validator;
0275     }
0276
0277     goto add_field;
0278
0279 add_validator:
0280     if (strstr(type, "char") != 0)
0281         validator_flags |= VALIDATOR_ENSURE_NULL;
0282
0283     validator = kmalloc(sizeof(*validator), GFP_KERNEL);
0284
0285     if (!validator) {
0286         kfree(field);
0287         return -ENOMEM;
0288     }
0289
0290     validator->flags = validator_flags;
0291     validator->offset = offset;
0292
0293     /* Want sequential access when validating */
0294     list_add_tail(&validator->link, &user->validators);
0295
0296 add_field:
0297     field->type = type;
0298     field->name = name;
0299     field->offset = offset;
0300     field->size = size;
0301     field->is_signed = is_signed;
0302     field->filter_type = filter_type;
0303
0304     list_add(&field->link, &user->fields);
0305
0306     /*
0307      * Min size from user writes that are required, this does not include
0308      * the size of trace_entry (common fields).
0309      */
0310     user->min_size = (offset + size) - sizeof(struct trace_entry);
0311
0312     return 0;
0313 }
0314
0315 /*
0316  * Parses the values of a field within the description
0317  * Format: type name [size]
0318  */
0319 static int user_event_parse_field(char *field, struct user_event *user,
0320                   u32 *offset)
0321 {
0322     char *part, *type, *name;
0323     u32 depth = 0, saved_offset = *offset;
0324     int len, size = -EINVAL;
0325     bool is_struct = false;
0326
0327     field = skip_spaces(field);
0328
0329     if (*field == '\0')
0330         return 0;
0331
0332     /* Handle types that have a space within */
0333     len = str_has_prefix(field, "unsigned ");
0334     if (len)
0335         goto skip_next;
0336
0337     len = str_has_prefix(field, "struct ");
0338     if (len) {
0339         is_struct = true;
0340         goto skip_next;
0341     }
0342
0343     len = str_has_prefix(field, "__data_loc unsigned ");
0344     if (len)
0345         goto skip_next;
0346
0347     len = str_has_prefix(field, "__data_loc ");
0348     if (len)
0349         goto skip_next;
0350
0351     len = str_has_prefix(field, "__rel_loc unsigned ");
0352     if (len)
0353         goto skip_next;
0354
0355     len = str_has_prefix(field, "__rel_loc ");
0356     if (len)
0357         goto skip_next;
0358
0359     goto parse;
0360 skip_next:
0361     type = field;
0362     field = strpbrk(field + len, " ");
0363
0364     if (field == NULL)
0365         return -EINVAL;
0366
0367     *field++ = '\0';
0368     depth++;
0369 parse:
0370     name = NULL;
0371
0372     while ((part = strsep(&field, " ")) != NULL) {
0373         switch (depth++) {
0374         case FIELD_DEPTH_TYPE:
0375             type = part;
0376             break;
0377         case FIELD_DEPTH_NAME:
0378             name = part;
0379             break;
0380         case FIELD_DEPTH_SIZE:
0381             if (!is_struct)
0382                 return -EINVAL;
0383
0384             if (kstrtou32(part, 10, &size))
0385                 return -EINVAL;
0386             break;
0387         default:
0388             return -EINVAL;
0389         }
0390     }
0391
0392     if (depth < FIELD_DEPTH_SIZE || !name)
0393         return -EINVAL;
0394
0395     if (depth == FIELD_DEPTH_SIZE)
0396         size = user_field_size(type);
0397
0398     if (size == 0)
0399         return -EINVAL;
0400
0401     if (size < 0)
0402         return size;
0403
0404     *offset = saved_offset + size;
0405
0406     return user_event_add_field(user, type, name, saved_offset, size,
0407                     type[0] != 'u', FILTER_OTHER);
0408 }
0409
0410 static int user_event_parse_fields(struct user_event *user, char *args)
0411 {
0412     char *field;
0413     u32 offset = sizeof(struct trace_entry);
0414     int ret = -EINVAL;
0415
0416     if (args == NULL)
0417         return 0;
0418
0419     while ((field = strsep(&args, ";")) != NULL) {
0420         ret = user_event_parse_field(field, user, &offset);
0421
0422         if (ret)
0423             break;
0424     }
0425
0426     return ret;
0427 }
0428
0429 static struct trace_event_fields user_event_fields_array[1];
0430
0431 static const char *user_field_format(const char *type)
0432 {
0433     if (strcmp(type, "s64") == 0)
0434         return "%lld";
0435     if (strcmp(type, "u64") == 0)
0436         return "%llu";
0437     if (strcmp(type, "s32") == 0)
0438         return "%d";
0439     if (strcmp(type, "u32") == 0)
0440         return "%u";
0441     if (strcmp(type, "int") == 0)
0442         return "%d";
0443     if (strcmp(type, "unsigned int") == 0)
0444         return "%u";
0445     if (strcmp(type, "s16") == 0)
0446         return "%d";
0447     if (strcmp(type, "u16") == 0)
0448         return "%u";
0449     if (strcmp(type, "short") == 0)
0450         return "%d";
0451     if (strcmp(type, "unsigned short") == 0)
0452         return "%u";
0453     if (strcmp(type, "s8") == 0)
0454         return "%d";
0455     if (strcmp(type, "u8") == 0)
0456         return "%u";
0457     if (strcmp(type, "char") == 0)
0458         return "%d";
0459     if (strcmp(type, "unsigned char") == 0)
0460         return "%u";
0461     if (strstr(type, "char[") != 0)
0462         return "%s";
0463
0464     /* Unknown, likely struct, allowed treat as 64-bit */
0465     return "%llu";
0466 }
0467
0468 static bool user_field_is_dyn_string(const char *type, const char **str_func)
0469 {
0470     if (str_has_prefix(type, "__data_loc ")) {
0471         *str_func = "__get_str";
0472         goto check;
0473     }
0474
0475     if (str_has_prefix(type, "__rel_loc ")) {
0476         *str_func = "__get_rel_str";
0477         goto check;
0478     }
0479
0480     return false;
0481 check:
0482     return strstr(type, "char") != 0;
0483 }
0484
0485 #define LEN_OR_ZERO (len ? len - pos : 0)
0486 static int user_event_set_print_fmt(struct user_event *user, char *buf, int len)
0487 {
0488     struct ftrace_event_field *field, *next;
0489     struct list_head *head = &user->fields;
0490     int pos = 0, depth = 0;
0491     const char *str_func;
0492
0493     pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
0494
0495     list_for_each_entry_safe_reverse(field, next, head, link) {
0496         if (depth != 0)
0497             pos += snprintf(buf + pos, LEN_OR_ZERO, " ");
0498
0499         pos += snprintf(buf + pos, LEN_OR_ZERO, "%s=%s",
0500                 field->name, user_field_format(field->type));
0501
0502         depth++;
0503     }
0504
0505     pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
0506
0507     list_for_each_entry_safe_reverse(field, next, head, link) {
0508         if (user_field_is_dyn_string(field->type, &str_func))
0509             pos += snprintf(buf + pos, LEN_OR_ZERO,
0510                     ", %s(%s)", str_func, field->name);
0511         else
0512             pos += snprintf(buf + pos, LEN_OR_ZERO,
0513                     ", REC->%s", field->name);
0514     }
0515
0516     return pos + 1;
0517 }
0518 #undef LEN_OR_ZERO
0519
0520 static int user_event_create_print_fmt(struct user_event *user)
0521 {
0522     char *print_fmt;
0523     int len;
0524
0525     len = user_event_set_print_fmt(user, NULL, 0);
0526
0527     print_fmt = kmalloc(len, GFP_KERNEL);
0528
0529     if (!print_fmt)
0530         return -ENOMEM;
0531
0532     user_event_set_print_fmt(user, print_fmt, len);
0533
0534     user->call.print_fmt = print_fmt;
0535
0536     return 0;
0537 }
0538
0539 static enum print_line_t user_event_print_trace(struct trace_iterator *iter,
0540                         int flags,
0541                         struct trace_event *event)
0542 {
0543     /* Unsafe to try to decode user provided print_fmt, use hex */
0544     trace_print_hex_dump_seq(&iter->seq, "", DUMP_PREFIX_OFFSET, 16,
0545                  1, iter->ent, iter->ent_size, true);
0546
0547     return trace_handle_return(&iter->seq);
0548 }
0549
0550 static struct trace_event_functions user_event_funcs = {
0551     .trace = user_event_print_trace,
0552 };
0553
0554 static int user_event_set_call_visible(struct user_event *user, bool visible)
0555 {
0556     int ret;
0557     const struct cred *old_cred;
0558     struct cred *cred;
0559
0560     cred = prepare_creds();
0561
0562     if (!cred)
0563         return -ENOMEM;
0564
0565     /*
0566      * While by default tracefs is locked down, systems can be configured
0567      * to allow user_event files to be less locked down. The extreme case
0568      * being "other" has read/write access to user_events_data/status.
0569      *
0570      * When not locked down, processes may not have permissions to
0571      * add/remove calls themselves to tracefs. We need to temporarily
0572      * switch to root file permission to allow for this scenario.
0573      */
0574     cred->fsuid = GLOBAL_ROOT_UID;
0575
0576     old_cred = override_creds(cred);
0577
0578     if (visible)
0579         ret = trace_add_event_call(&user->call);
0580     else
0581         ret = trace_remove_event_call(&user->call);
0582
0583     revert_creds(old_cred);
0584     put_cred(cred);
0585
0586     return ret;
0587 }
0588
0589 static int destroy_user_event(struct user_event *user)
0590 {
0591     int ret = 0;
0592
0593     /* Must destroy fields before call removal */
0594     user_event_destroy_fields(user);
0595
0596     ret = user_event_set_call_visible(user, false);
0597
0598     if (ret)
0599         return ret;
0600
0601     dyn_event_remove(&user->devent);
0602
0603     register_page_data[user->index] = 0;
0604     clear_bit(user->index, page_bitmap);
0605     hash_del(&user->node);
0606
0607     user_event_destroy_validators(user);
0608     kfree(user->call.print_fmt);
0609     kfree(EVENT_NAME(user));
0610     kfree(user);
0611
0612     return ret;
0613 }
0614
0615 static struct user_event *find_user_event(char *name, u32 *outkey)
0616 {
0617     struct user_event *user;
0618     u32 key = user_event_key(name);
0619
0620     *outkey = key;
0621
0622     hash_for_each_possible(register_table, user, node, key)
0623         if (!strcmp(EVENT_NAME(user), name)) {
0624             atomic_inc(&user->refcnt);
0625             return user;
0626         }
0627
0628     return NULL;
0629 }
0630
0631 static int user_event_validate(struct user_event *user, void *data, int len)
0632 {
0633     struct list_head *head = &user->validators;
0634     struct user_event_validator *validator;
0635     void *pos, *end = data + len;
0636     u32 loc, offset, size;
0637
0638     list_for_each_entry(validator, head, link) {
0639         pos = data + validator->offset;
0640
0641         /* Already done min_size check, no bounds check here */
0642         loc = *(u32 *)pos;
0643         offset = loc & 0xffff;
0644         size = loc >> 16;
0645
0646         if (likely(validator->flags & VALIDATOR_REL))
0647             pos += offset + sizeof(loc);
0648         else
0649             pos = data + offset;
0650
0651         pos += size;
0652
0653         if (unlikely(pos > end))
0654             return -EFAULT;
0655
0656         if (likely(validator->flags & VALIDATOR_ENSURE_NULL))
0657             if (unlikely(*(char *)(pos - 1) != '\0'))
0658                 return -EFAULT;
0659     }
0660
0661     return 0;
0662 }
0663
0664 /*
0665  * Writes the user supplied payload out to a trace file.
0666  */
0667 static void user_event_ftrace(struct user_event *user, struct iov_iter *i,
0668                   void *tpdata, bool *faulted)
0669 {
0670     struct trace_event_file *file;
0671     struct trace_entry *entry;
0672     struct trace_event_buffer event_buffer;
0673     size_t size = sizeof(*entry) + i->count;
0674
0675     file = (struct trace_event_file *)tpdata;
0676
0677     if (!file ||
0678         !(file->flags & EVENT_FILE_FL_ENABLED) ||
0679         trace_trigger_soft_disabled(file))
0680         return;
0681
0682     /* Allocates and fills trace_entry, + 1 of this is data payload */
0683     entry = trace_event_buffer_reserve(&event_buffer, file, size);
0684
0685     if (unlikely(!entry))
0686         return;
0687
0688     if (unlikely(!copy_nofault(entry + 1, i->count, i)))
0689         goto discard;
0690
0691     if (!list_empty(&user->validators) &&
0692         unlikely(user_event_validate(user, entry, size)))
0693         goto discard;
0694
0695     trace_event_buffer_commit(&event_buffer);
0696
0697     return;
0698 discard:
0699     *faulted = true;
0700     __trace_event_discard_commit(event_buffer.buffer,
0701                      event_buffer.event);
0702 }
0703
0704 #ifdef CONFIG_PERF_EVENTS
0705 /*
0706  * Writes the user supplied payload out to perf ring buffer.
0707  */
0708 static void user_event_perf(struct user_event *user, struct iov_iter *i,
0709                 void *tpdata, bool *faulted)
0710 {
0711     struct hlist_head *perf_head;
0712
0713     perf_head = this_cpu_ptr(user->call.perf_events);
0714
0715     if (perf_head && !hlist_empty(perf_head)) {
0716         struct trace_entry *perf_entry;
0717         struct pt_regs *regs;
0718         size_t size = sizeof(*perf_entry) + i->count;
0719         int context;
0720
0721         perf_entry = perf_trace_buf_alloc(ALIGN(size, 8),
0722                           &regs, &context);
0723
0724         if (unlikely(!perf_entry))
0725             return;
0726
0727         perf_fetch_caller_regs(regs);
0728
0729         if (unlikely(!copy_nofault(perf_entry + 1, i->count, i)))
0730             goto discard;
0731
0732         if (!list_empty(&user->validators) &&
0733             unlikely(user_event_validate(user, perf_entry, size)))
0734             goto discard;
0735
0736         perf_trace_buf_submit(perf_entry, size, context,
0737                       user->call.event.type, 1, regs,
0738                       perf_head, NULL);
0739
0740         return;
0741 discard:
0742         *faulted = true;
0743         perf_swevent_put_recursion_context(context);
0744     }
0745 }
0746 #endif
0747
0748 /*
0749  * Update the register page that is shared between user processes.
0750  */
0751 static void update_reg_page_for(struct user_event *user)
0752 {
0753     struct tracepoint *tp = &user->tracepoint;
0754     char status = 0;
0755
0756     if (atomic_read(&tp->key.enabled) > 0) {
0757         struct tracepoint_func *probe_func_ptr;
0758         user_event_func_t probe_func;
0759
0760         rcu_read_lock_sched();
0761
0762         probe_func_ptr = rcu_dereference_sched(tp->funcs);
0763
0764         if (probe_func_ptr) {
0765             do {
0766                 probe_func = probe_func_ptr->func;
0767
0768                 if (probe_func == user_event_ftrace)
0769                     status |= EVENT_STATUS_FTRACE;
0770 #ifdef CONFIG_PERF_EVENTS
0771                 else if (probe_func == user_event_perf)
0772                     status |= EVENT_STATUS_PERF;
0773 #endif
0774                 else
0775                     status |= EVENT_STATUS_OTHER;
0776             } while ((++probe_func_ptr)->func);
0777         }
0778
0779         rcu_read_unlock_sched();
0780     }
0781
0782     register_page_data[user->index] = status;
0783 }
0784
0785 /*
0786  * Register callback for our events from tracing sub-systems.
0787  */
0788 static int user_event_reg(struct trace_event_call *call,
0789               enum trace_reg type,
0790               void *data)
0791 {
0792     struct user_event *user = (struct user_event *)call->data;
0793     int ret = 0;
0794
0795     if (!user)
0796         return -ENOENT;
0797
0798     switch (type) {
0799     case TRACE_REG_REGISTER:
0800         ret = tracepoint_probe_register(call->tp,
0801                         call->class->probe,
0802                         data);
0803         if (!ret)
0804             goto inc;
0805         break;
0806
0807     case TRACE_REG_UNREGISTER:
0808         tracepoint_probe_unregister(call->tp,
0809                         call->class->probe,
0810                         data);
0811         goto dec;
0812
0813 #ifdef CONFIG_PERF_EVENTS
0814     case TRACE_REG_PERF_REGISTER:
0815         ret = tracepoint_probe_register(call->tp,
0816                         call->class->perf_probe,
0817                         data);
0818         if (!ret)
0819             goto inc;
0820         break;
0821
0822     case TRACE_REG_PERF_UNREGISTER:
0823         tracepoint_probe_unregister(call->tp,
0824                         call->class->perf_probe,
0825                         data);
0826         goto dec;
0827
0828     case TRACE_REG_PERF_OPEN:
0829     case TRACE_REG_PERF_CLOSE:
0830     case TRACE_REG_PERF_ADD:
0831     case TRACE_REG_PERF_DEL:
0832         break;
0833 #endif
0834     }
0835
0836     return ret;
0837 inc:
0838     atomic_inc(&user->refcnt);
0839     update_reg_page_for(user);
0840     return 0;
0841 dec:
0842     update_reg_page_for(user);
0843     atomic_dec(&user->refcnt);
0844     return 0;
0845 }
0846
0847 static int user_event_create(const char *raw_command)
0848 {
0849     struct user_event *user;
0850     char *name;
0851     int ret;
0852
0853     if (!str_has_prefix(raw_command, USER_EVENTS_PREFIX))
0854         return -ECANCELED;
0855
0856     raw_command += USER_EVENTS_PREFIX_LEN;
0857     raw_command = skip_spaces(raw_command);
0858
0859     name = kstrdup(raw_command, GFP_KERNEL);
0860
0861     if (!name)
0862         return -ENOMEM;
0863
0864     mutex_lock(&reg_mutex);
0865
0866     ret = user_event_parse_cmd(name, &user);
0867
0868     if (!ret)
0869         atomic_dec(&user->refcnt);
0870
0871     mutex_unlock(&reg_mutex);
0872
0873     if (ret)
0874         kfree(name);
0875
0876     return ret;
0877 }
0878
0879 static int user_event_show(struct seq_file *m, struct dyn_event *ev)
0880 {
0881     struct user_event *user = container_of(ev, struct user_event, devent);
0882     struct ftrace_event_field *field, *next;
0883     struct list_head *head;
0884     int depth = 0;
0885
0886     seq_printf(m, "%s%s", USER_EVENTS_PREFIX, EVENT_NAME(user));
0887
0888     head = trace_get_fields(&user->call);
0889
0890     list_for_each_entry_safe_reverse(field, next, head, link) {
0891         if (depth == 0)
0892             seq_puts(m, " ");
0893         else
0894             seq_puts(m, "; ");
0895
0896         seq_printf(m, "%s %s", field->type, field->name);
0897
0898         if (str_has_prefix(field->type, "struct "))
0899             seq_printf(m, " %d", field->size);
0900
0901         depth++;
0902     }
0903
0904     seq_puts(m, "\n");
0905
0906     return 0;
0907 }
0908
0909 static bool user_event_is_busy(struct dyn_event *ev)
0910 {
0911     struct user_event *user = container_of(ev, struct user_event, devent);
0912
0913     return atomic_read(&user->refcnt) != 0;
0914 }
0915
0916 static int user_event_free(struct dyn_event *ev)
0917 {
0918     struct user_event *user = container_of(ev, struct user_event, devent);
0919
0920     if (atomic_read(&user->refcnt) != 0)
0921         return -EBUSY;
0922
0923     return destroy_user_event(user);
0924 }
0925
0926 static bool user_field_match(struct ftrace_event_field *field, int argc,
0927                  const char **argv, int *iout)
0928 {
0929     char *field_name, *arg_name;
0930     int len, pos, i = *iout;
0931     bool colon = false, match = false;
0932
0933     if (i >= argc)
0934         return false;
0935
0936     len = MAX_FIELD_ARG_NAME;
0937     field_name = kmalloc(len, GFP_KERNEL);
0938     arg_name = kmalloc(len, GFP_KERNEL);
0939
0940     if (!arg_name || !field_name)
0941         goto out;
0942
0943     pos = 0;
0944
0945     for (; i < argc; ++i) {
0946         if (i != *iout)
0947             pos += snprintf(arg_name + pos, len - pos, " ");
0948
0949         pos += snprintf(arg_name + pos, len - pos, argv[i]);
0950
0951         if (strchr(argv[i], ';')) {
0952             ++i;
0953             colon = true;
0954             break;
0955         }
0956     }
0957
0958     pos = 0;
0959
0960     pos += snprintf(field_name + pos, len - pos, field->type);
0961     pos += snprintf(field_name + pos, len - pos, " ");
0962     pos += snprintf(field_name + pos, len - pos, field->name);
0963
0964     if (colon)
0965         pos += snprintf(field_name + pos, len - pos, ";");
0966
0967     *iout = i;
0968
0969     match = strcmp(arg_name, field_name) == 0;
0970 out:
0971     kfree(arg_name);
0972     kfree(field_name);
0973
0974     return match;
0975 }
0976
0977 static bool user_fields_match(struct user_event *user, int argc,
0978                   const char **argv)
0979 {
0980     struct ftrace_event_field *field, *next;
0981     struct list_head *head = &user->fields;
0982     int i = 0;
0983
0984     list_for_each_entry_safe_reverse(field, next, head, link)
0985         if (!user_field_match(field, argc, argv, &i))
0986             return false;
0987
0988     if (i != argc)
0989         return false;
0990
0991     return true;
0992 }
0993
0994 static bool user_event_match(const char *system, const char *event,
0995                  int argc, const char **argv, struct dyn_event *ev)
0996 {
0997     struct user_event *user = container_of(ev, struct user_event, devent);
0998     bool match;
0999
1000     match = strcmp(EVENT_NAME(user), event) == 0 &&
1001         (!system || strcmp(system, USER_EVENTS_SYSTEM) == 0);
1002
1003     if (match && argc > 0)
1004         match = user_fields_match(user, argc, argv);
1005
1006     return match;
1007 }
1008
1009 static struct dyn_event_operations user_event_dops = {
1010     .create = user_event_create,
1011     .show = user_event_show,
1012     .is_busy = user_event_is_busy,
1013     .free = user_event_free,
1014     .match = user_event_match,
1015 };
1016
1017 static int user_event_trace_register(struct user_event *user)
1018 {
1019     int ret;
1020
1021     ret = register_trace_event(&user->call.event);
1022
1023     if (!ret)
1024         return -ENODEV;
1025
1026     ret = user_event_set_call_visible(user, true);
1027
1028     if (ret)
1029         unregister_trace_event(&user->call.event);
1030
1031     return ret;
1032 }
1033
1034 /*
1035  * Parses the event name, arguments and flags then registers if successful.
1036  * The name buffer lifetime is owned by this method for success cases only.
1037  * Upon success the returned user_event has its ref count increased by 1.
1038  */
1039 static int user_event_parse(char *name, char *args, char *flags,
1040                 struct user_event **newuser)
1041 {
1042     int ret;
1043     int index;
1044     u32 key;
1045     struct user_event *user;
1046
1047     /* Prevent dyn_event from racing */
1048     mutex_lock(&event_mutex);
1049     user = find_user_event(name, &key);
1050     mutex_unlock(&event_mutex);
1051
1052     if (user) {
1053         *newuser = user;
1054         /*
1055          * Name is allocated by caller, free it since it already exists.
1056          * Caller only worries about failure cases for freeing.
1057          */
1058         kfree(name);
1059         return 0;
1060     }
1061
1062     index = find_first_zero_bit(page_bitmap, MAX_EVENTS);
1063
1064     if (index == MAX_EVENTS)
1065         return -EMFILE;
1066
1067     user = kzalloc(sizeof(*user), GFP_KERNEL);
1068
1069     if (!user)
1070         return -ENOMEM;
1071
1072     INIT_LIST_HEAD(&user->class.fields);
1073     INIT_LIST_HEAD(&user->fields);
1074     INIT_LIST_HEAD(&user->validators);
1075
1076     user->tracepoint.name = name;
1077
1078     ret = user_event_parse_fields(user, args);
1079
1080     if (ret)
1081         goto put_user;
1082
1083     ret = user_event_create_print_fmt(user);
1084
1085     if (ret)
1086         goto put_user;
1087
1088     user->call.data = user;
1089     user->call.class = &user->class;
1090     user->call.name = name;
1091     user->call.flags = TRACE_EVENT_FL_TRACEPOINT;
1092     user->call.tp = &user->tracepoint;
1093     user->call.event.funcs = &user_event_funcs;
1094
1095     user->class.system = USER_EVENTS_SYSTEM;
1096     user->class.fields_array = user_event_fields_array;
1097     user->class.get_fields = user_event_get_fields;
1098     user->class.reg = user_event_reg;
1099     user->class.probe = user_event_ftrace;
1100 #ifdef CONFIG_PERF_EVENTS
1101     user->class.perf_probe = user_event_perf;
1102 #endif
1103
1104     mutex_lock(&event_mutex);
1105
1106     ret = user_event_trace_register(user);
1107
1108     if (ret)
1109         goto put_user_lock;
1110
1111     user->index = index;
1112
1113     /* Ensure we track ref */
1114     atomic_inc(&user->refcnt);
1115
1116     dyn_event_init(&user->devent, &user_event_dops);
1117     dyn_event_add(&user->devent, &user->call);
1118     set_bit(user->index, page_bitmap);
1119     hash_add(register_table, &user->node, key);
1120
1121     mutex_unlock(&event_mutex);
1122
1123     *newuser = user;
1124     return 0;
1125 put_user_lock:
1126     mutex_unlock(&event_mutex);
1127 put_user:
1128     user_event_destroy_fields(user);
1129     user_event_destroy_validators(user);
1130     kfree(user);
1131     return ret;
1132 }
1133
1134 /*
1135  * Deletes a previously created event if it is no longer being used.
1136  */
1137 static int delete_user_event(char *name)
1138 {
1139     u32 key;
1140     int ret;
1141     struct user_event *user = find_user_event(name, &key);
1142
1143     if (!user)
1144         return -ENOENT;
1145
1146     /* Ensure we are the last ref */
1147     if (atomic_read(&user->refcnt) != 1) {
1148         ret = -EBUSY;
1149         goto put_ref;
1150     }
1151
1152     ret = destroy_user_event(user);
1153
1154     if (ret)
1155         goto put_ref;
1156
1157     return ret;
1158 put_ref:
1159     /* No longer have this ref */
1160     atomic_dec(&user->refcnt);
1161
1162     return ret;
1163 }
1164
1165 /*
1166  * Validates the user payload and writes via iterator.
1167  */
1168 static ssize_t user_events_write_core(struct file *file, struct iov_iter *i)
1169 {
1170     struct user_event_refs *refs;
1171     struct user_event *user = NULL;
1172     struct tracepoint *tp;
1173     ssize_t ret = i->count;
1174     int idx;
1175
1176     if (unlikely(copy_from_iter(&idx, sizeof(idx), i) != sizeof(idx)))
1177         return -EFAULT;
1178
1179     rcu_read_lock_sched();
1180
1181     refs = rcu_dereference_sched(file->private_data);
1182
1183     /*
1184      * The refs->events array is protected by RCU, and new items may be
1185      * added. But the user retrieved from indexing into the events array
1186      * shall be immutable while the file is opened.
1187      */
1188     if (likely(refs && idx < refs->count))
1189         user = refs->events[idx];
1190
1191     rcu_read_unlock_sched();
1192
1193     if (unlikely(user == NULL))
1194         return -ENOENT;
1195
1196     if (unlikely(i->count < user->min_size))
1197         return -EINVAL;
1198
1199     tp = &user->tracepoint;
1200
1201     /*
1202      * It's possible key.enabled disables after this check, however
1203      * we don't mind if a few events are included in this condition.
1204      */
1205     if (likely(atomic_read(&tp->key.enabled) > 0)) {
1206         struct tracepoint_func *probe_func_ptr;
1207         user_event_func_t probe_func;
1208         struct iov_iter copy;
1209         void *tpdata;
1210         bool faulted;
1211
1212         if (unlikely(fault_in_iov_iter_readable(i, i->count)))
1213             return -EFAULT;
1214
1215         faulted = false;
1216
1217         rcu_read_lock_sched();
1218
1219         probe_func_ptr = rcu_dereference_sched(tp->funcs);
1220
1221         if (probe_func_ptr) {
1222             do {
1223                 copy = *i;
1224                 probe_func = probe_func_ptr->func;
1225                 tpdata = probe_func_ptr->data;
1226                 probe_func(user, &copy, tpdata, &faulted);
1227             } while ((++probe_func_ptr)->func);
1228         }
1229
1230         rcu_read_unlock_sched();
1231
1232         if (unlikely(faulted))
1233             return -EFAULT;
1234     }
1235
1236     return ret;
1237 }
1238
1239 static ssize_t user_events_write(struct file *file, const char __user *ubuf,
1240                  size_t count, loff_t *ppos)
1241 {
1242     struct iovec iov;
1243     struct iov_iter i;
1244
1245     if (unlikely(*ppos != 0))
1246         return -EFAULT;
1247
1248     if (unlikely(import_single_range(READ, (char *)ubuf, count, &iov, &i)))
1249         return -EFAULT;
1250
1251     return user_events_write_core(file, &i);
1252 }
1253
1254 static ssize_t user_events_write_iter(struct kiocb *kp, struct iov_iter *i)
1255 {
1256     return user_events_write_core(kp->ki_filp, i);
1257 }
1258
1259 static int user_events_ref_add(struct file *file, struct user_event *user)
1260 {
1261     struct user_event_refs *refs, *new_refs;
1262     int i, size, count = 0;
1263
1264     refs = rcu_dereference_protected(file->private_data,
1265                      lockdep_is_held(&reg_mutex));
1266
1267     if (refs) {
1268         count = refs->count;
1269
1270         for (i = 0; i < count; ++i)
1271             if (refs->events[i] == user)
1272                 return i;
1273     }
1274
1275     size = struct_size(refs, events, count + 1);
1276
1277     new_refs = kzalloc(size, GFP_KERNEL);
1278
1279     if (!new_refs)
1280         return -ENOMEM;
1281
1282     new_refs->count = count + 1;
1283
1284     for (i = 0; i < count; ++i)
1285         new_refs->events[i] = refs->events[i];
1286
1287     new_refs->events[i] = user;
1288
1289     atomic_inc(&user->refcnt);
1290
1291     rcu_assign_pointer(file->private_data, new_refs);
1292
1293     if (refs)
1294         kfree_rcu(refs, rcu);
1295
1296     return i;
1297 }
1298
1299 static long user_reg_get(struct user_reg __user *ureg, struct user_reg *kreg)
1300 {
1301     u32 size;
1302     long ret;
1303
1304     ret = get_user(size, &ureg->size);
1305
1306     if (ret)
1307         return ret;
1308
1309     if (size > PAGE_SIZE)
1310         return -E2BIG;
1311
1312     return copy_struct_from_user(kreg, sizeof(*kreg), ureg, size);
1313 }
1314
1315 /*
1316  * Registers a user_event on behalf of a user process.
1317  */
1318 static long user_events_ioctl_reg(struct file *file, unsigned long uarg)
1319 {
1320     struct user_reg __user *ureg = (struct user_reg __user *)uarg;
1321     struct user_reg reg;
1322     struct user_event *user;
1323     char *name;
1324     long ret;
1325
1326     ret = user_reg_get(ureg, &reg);
1327
1328     if (ret)
1329         return ret;
1330
1331     name = strndup_user((const char __user *)(uintptr_t)reg.name_args,
1332                 MAX_EVENT_DESC);
1333
1334     if (IS_ERR(name)) {
1335         ret = PTR_ERR(name);
1336         return ret;
1337     }
1338
1339     ret = user_event_parse_cmd(name, &user);
1340
1341     if (ret) {
1342         kfree(name);
1343         return ret;
1344     }
1345
1346     ret = user_events_ref_add(file, user);
1347
1348     /* No longer need parse ref, ref_add either worked or not */
1349     atomic_dec(&user->refcnt);
1350
1351     /* Positive number is index and valid */
1352     if (ret < 0)
1353         return ret;
1354
1355     put_user((u32)ret, &ureg->write_index);
1356     put_user(user->index, &ureg->status_index);
1357
1358     return 0;
1359 }
1360
1361 /*
1362  * Deletes a user_event on behalf of a user process.
1363  */
1364 static long user_events_ioctl_del(struct file *file, unsigned long uarg)
1365 {
1366     void __user *ubuf = (void __user *)uarg;
1367     char *name;
1368     long ret;
1369
1370     name = strndup_user(ubuf, MAX_EVENT_DESC);
1371
1372     if (IS_ERR(name))
1373         return PTR_ERR(name);
1374
1375     /* event_mutex prevents dyn_event from racing */
1376     mutex_lock(&event_mutex);
1377     ret = delete_user_event(name);
1378     mutex_unlock(&event_mutex);
1379
1380     kfree(name);
1381
1382     return ret;
1383 }
1384
1385 /*
1386  * Handles the ioctl from user mode to register or alter operations.
1387  */
1388 static long user_events_ioctl(struct file *file, unsigned int cmd,
1389                   unsigned long uarg)
1390 {
1391     long ret = -ENOTTY;
1392
1393     switch (cmd) {
1394     case DIAG_IOCSREG:
1395         mutex_lock(&reg_mutex);
1396         ret = user_events_ioctl_reg(file, uarg);
1397         mutex_unlock(&reg_mutex);
1398         break;
1399
1400     case DIAG_IOCSDEL:
1401         mutex_lock(&reg_mutex);
1402         ret = user_events_ioctl_del(file, uarg);
1403         mutex_unlock(&reg_mutex);
1404         break;
1405     }
1406
1407     return ret;
1408 }
1409
1410 /*
1411  * Handles the final close of the file from user mode.
1412  */
1413 static int user_events_release(struct inode *node, struct file *file)
1414 {
1415     struct user_event_refs *refs;
1416     struct user_event *user;
1417     int i;
1418
1419     /*
1420      * Ensure refs cannot change under any situation by taking the
1421      * register mutex during the final freeing of the references.
1422      */
1423     mutex_lock(&reg_mutex);
1424
1425     refs = file->private_data;
1426
1427     if (!refs)
1428         goto out;
1429
1430     /*
1431      * The lifetime of refs has reached an end, it's tied to this file.
1432      * The underlying user_events are ref counted, and cannot be freed.
1433      * After this decrement, the user_events may be freed elsewhere.
1434      */
1435     for (i = 0; i < refs->count; ++i) {
1436         user = refs->events[i];
1437
1438         if (user)
1439             atomic_dec(&user->refcnt);
1440     }
1441 out:
1442     file->private_data = NULL;
1443
1444     mutex_unlock(&reg_mutex);
1445
1446     kfree(refs);
1447
1448     return 0;
1449 }
1450
1451 static const struct file_operations user_data_fops = {
1452     .write = user_events_write,
1453     .write_iter = user_events_write_iter,
1454     .unlocked_ioctl = user_events_ioctl,
1455     .release = user_events_release,
1456 };
1457
1458 /*
1459  * Maps the shared page into the user process for checking if event is enabled.
1460  */
1461 static int user_status_mmap(struct file *file, struct vm_area_struct *vma)
1462 {
1463     unsigned long size = vma->vm_end - vma->vm_start;
1464
1465     if (size != MAX_EVENTS)
1466         return -EINVAL;
1467
1468     return remap_pfn_range(vma, vma->vm_start,
1469                    virt_to_phys(register_page_data) >> PAGE_SHIFT,
1470                    size, vm_get_page_prot(VM_READ));
1471 }
1472
1473 static void *user_seq_start(struct seq_file *m, loff_t *pos)
1474 {
1475     if (*pos)
1476         return NULL;
1477
1478     return (void *)1;
1479 }
1480
1481 static void *user_seq_next(struct seq_file *m, void *p, loff_t *pos)
1482 {
1483     ++*pos;
1484     return NULL;
1485 }
1486
1487 static void user_seq_stop(struct seq_file *m, void *p)
1488 {
1489 }
1490
1491 static int user_seq_show(struct seq_file *m, void *p)
1492 {
1493     struct user_event *user;
1494     char status;
1495     int i, active = 0, busy = 0, flags;
1496
1497     mutex_lock(&reg_mutex);
1498
1499     hash_for_each(register_table, i, user, node) {
1500         status = register_page_data[user->index];
1501         flags = user->flags;
1502
1503         seq_printf(m, "%d:%s", user->index, EVENT_NAME(user));
1504
1505         if (flags != 0 || status != 0)
1506             seq_puts(m, " #");
1507
1508         if (status != 0) {
1509             seq_puts(m, " Used by");
1510             if (status & EVENT_STATUS_FTRACE)
1511                 seq_puts(m, " ftrace");
1512             if (status & EVENT_STATUS_PERF)
1513                 seq_puts(m, " perf");
1514             if (status & EVENT_STATUS_OTHER)
1515                 seq_puts(m, " other");
1516             busy++;
1517         }
1518
1519         seq_puts(m, "\n");
1520         active++;
1521     }
1522
1523     mutex_unlock(&reg_mutex);
1524
1525     seq_puts(m, "\n");
1526     seq_printf(m, "Active: %d\n", active);
1527     seq_printf(m, "Busy: %d\n", busy);
1528     seq_printf(m, "Max: %ld\n", MAX_EVENTS);
1529
1530     return 0;
1531 }
1532
1533 static const struct seq_operations user_seq_ops = {
1534     .start = user_seq_start,
1535     .next  = user_seq_next,
1536     .stop  = user_seq_stop,
1537     .show  = user_seq_show,
1538 };
1539
1540 static int user_status_open(struct inode *node, struct file *file)
1541 {
1542     return seq_open(file, &user_seq_ops);
1543 }
1544
1545 static const struct file_operations user_status_fops = {
1546     .open = user_status_open,
1547     .mmap = user_status_mmap,
1548     .read = seq_read,
1549     .llseek  = seq_lseek,
1550     .release = seq_release,
1551 };
1552
1553 /*
1554  * Creates a set of tracefs files to allow user mode interactions.
1555  */
1556 static int create_user_tracefs(void)
1557 {
1558     struct dentry *edata, *emmap;
1559
1560     edata = tracefs_create_file("user_events_data", TRACE_MODE_WRITE,
1561                     NULL, NULL, &user_data_fops);
1562
1563     if (!edata) {
1564         pr_warn("Could not create tracefs 'user_events_data' entry\n");
1565         goto err;
1566     }
1567
1568     /* mmap with MAP_SHARED requires writable fd */
1569     emmap = tracefs_create_file("user_events_status", TRACE_MODE_WRITE,
1570                     NULL, NULL, &user_status_fops);
1571
1572     if (!emmap) {
1573         tracefs_remove(edata);
1574         pr_warn("Could not create tracefs 'user_events_mmap' entry\n");
1575         goto err;
1576     }
1577
1578     return 0;
1579 err:
1580     return -ENODEV;
1581 }
1582
1583 static void set_page_reservations(bool set)
1584 {
1585     int page;
1586
1587     for (page = 0; page < MAX_PAGES; ++page) {
1588         void *addr = register_page_data + (PAGE_SIZE * page);
1589
1590         if (set)
1591             SetPageReserved(virt_to_page(addr));
1592         else
1593             ClearPageReserved(virt_to_page(addr));
1594     }
1595 }
1596
1597 static int __init trace_events_user_init(void)
1598 {
1599     struct page *pages;
1600     int ret;
1601
1602     /* Zero all bits beside 0 (which is reserved for failures) */
1603     bitmap_zero(page_bitmap, MAX_EVENTS);
1604     set_bit(0, page_bitmap);
1605
1606     pages = alloc_pages(GFP_KERNEL | __GFP_ZERO, MAX_PAGE_ORDER);
1607     if (!pages)
1608         return -ENOMEM;
1609     register_page_data = page_address(pages);
1610
1611     set_page_reservations(true);
1612
1613     ret = create_user_tracefs();
1614
1615     if (ret) {
1616         pr_warn("user_events could not register with tracefs\n");
1617         set_page_reservations(false);
1618         __free_pages(pages, MAX_PAGE_ORDER);
1619         return ret;
1620     }
1621
1622     if (dyn_event_register(&user_event_dops))
1623         pr_warn("user_events could not register with dyn_events\n");
1624
1625     return 0;
1626 }
1627
1628 fs_initcall(trace_events_user_init);