Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * Copyright (C) 2006 Jens Axboe <axboe@kernel.dk>
0004  *
0005  */
0006 
0007 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
0008 
0009 #include <linux/kernel.h>
0010 #include <linux/blkdev.h>
0011 #include <linux/blktrace_api.h>
0012 #include <linux/percpu.h>
0013 #include <linux/init.h>
0014 #include <linux/mutex.h>
0015 #include <linux/slab.h>
0016 #include <linux/debugfs.h>
0017 #include <linux/export.h>
0018 #include <linux/time.h>
0019 #include <linux/uaccess.h>
0020 #include <linux/list.h>
0021 #include <linux/blk-cgroup.h>
0022 
0023 #include "../../block/blk.h"
0024 
0025 #include <trace/events/block.h>
0026 
0027 #include "trace_output.h"
0028 
0029 #ifdef CONFIG_BLK_DEV_IO_TRACE
0030 
0031 static unsigned int blktrace_seq __read_mostly = 1;
0032 
0033 static struct trace_array *blk_tr;
0034 static bool blk_tracer_enabled __read_mostly;
0035 
0036 static LIST_HEAD(running_trace_list);
0037 static __cacheline_aligned_in_smp DEFINE_RAW_SPINLOCK(running_trace_lock);
0038 
0039 /* Select an alternative, minimalistic output than the original one */
0040 #define TRACE_BLK_OPT_CLASSIC   0x1
0041 #define TRACE_BLK_OPT_CGROUP    0x2
0042 #define TRACE_BLK_OPT_CGNAME    0x4
0043 
0044 static struct tracer_opt blk_tracer_opts[] = {
0045     /* Default disable the minimalistic output */
0046     { TRACER_OPT(blk_classic, TRACE_BLK_OPT_CLASSIC) },
0047 #ifdef CONFIG_BLK_CGROUP
0048     { TRACER_OPT(blk_cgroup, TRACE_BLK_OPT_CGROUP) },
0049     { TRACER_OPT(blk_cgname, TRACE_BLK_OPT_CGNAME) },
0050 #endif
0051     { }
0052 };
0053 
0054 static struct tracer_flags blk_tracer_flags = {
0055     .val  = 0,
0056     .opts = blk_tracer_opts,
0057 };
0058 
0059 /* Global reference count of probes */
0060 static DEFINE_MUTEX(blk_probe_mutex);
0061 static int blk_probes_ref;
0062 
0063 static void blk_register_tracepoints(void);
0064 static void blk_unregister_tracepoints(void);
0065 
0066 /*
0067  * Send out a notify message.
0068  */
0069 static void trace_note(struct blk_trace *bt, pid_t pid, int action,
0070                const void *data, size_t len, u64 cgid)
0071 {
0072     struct blk_io_trace *t;
0073     struct ring_buffer_event *event = NULL;
0074     struct trace_buffer *buffer = NULL;
0075     unsigned int trace_ctx = 0;
0076     int cpu = smp_processor_id();
0077     bool blk_tracer = blk_tracer_enabled;
0078     ssize_t cgid_len = cgid ? sizeof(cgid) : 0;
0079 
0080     if (blk_tracer) {
0081         buffer = blk_tr->array_buffer.buffer;
0082         trace_ctx = tracing_gen_ctx_flags(0);
0083         event = trace_buffer_lock_reserve(buffer, TRACE_BLK,
0084                           sizeof(*t) + len + cgid_len,
0085                           trace_ctx);
0086         if (!event)
0087             return;
0088         t = ring_buffer_event_data(event);
0089         goto record_it;
0090     }
0091 
0092     if (!bt->rchan)
0093         return;
0094 
0095     t = relay_reserve(bt->rchan, sizeof(*t) + len + cgid_len);
0096     if (t) {
0097         t->magic = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION;
0098         t->time = ktime_to_ns(ktime_get());
0099 record_it:
0100         t->device = bt->dev;
0101         t->action = action | (cgid ? __BLK_TN_CGROUP : 0);
0102         t->pid = pid;
0103         t->cpu = cpu;
0104         t->pdu_len = len + cgid_len;
0105         if (cgid_len)
0106             memcpy((void *)t + sizeof(*t), &cgid, cgid_len);
0107         memcpy((void *) t + sizeof(*t) + cgid_len, data, len);
0108 
0109         if (blk_tracer)
0110             trace_buffer_unlock_commit(blk_tr, buffer, event, trace_ctx);
0111     }
0112 }
0113 
0114 /*
0115  * Send out a notify for this process, if we haven't done so since a trace
0116  * started
0117  */
0118 static void trace_note_tsk(struct task_struct *tsk)
0119 {
0120     unsigned long flags;
0121     struct blk_trace *bt;
0122 
0123     tsk->btrace_seq = blktrace_seq;
0124     raw_spin_lock_irqsave(&running_trace_lock, flags);
0125     list_for_each_entry(bt, &running_trace_list, running_list) {
0126         trace_note(bt, tsk->pid, BLK_TN_PROCESS, tsk->comm,
0127                sizeof(tsk->comm), 0);
0128     }
0129     raw_spin_unlock_irqrestore(&running_trace_lock, flags);
0130 }
0131 
0132 static void trace_note_time(struct blk_trace *bt)
0133 {
0134     struct timespec64 now;
0135     unsigned long flags;
0136     u32 words[2];
0137 
0138     /* need to check user space to see if this breaks in y2038 or y2106 */
0139     ktime_get_real_ts64(&now);
0140     words[0] = (u32)now.tv_sec;
0141     words[1] = now.tv_nsec;
0142 
0143     local_irq_save(flags);
0144     trace_note(bt, 0, BLK_TN_TIMESTAMP, words, sizeof(words), 0);
0145     local_irq_restore(flags);
0146 }
0147 
0148 void __blk_trace_note_message(struct blk_trace *bt,
0149         struct cgroup_subsys_state *css, const char *fmt, ...)
0150 {
0151     int n;
0152     va_list args;
0153     unsigned long flags;
0154     char *buf;
0155     u64 cgid = 0;
0156 
0157     if (unlikely(bt->trace_state != Blktrace_running &&
0158              !blk_tracer_enabled))
0159         return;
0160 
0161     /*
0162      * If the BLK_TC_NOTIFY action mask isn't set, don't send any note
0163      * message to the trace.
0164      */
0165     if (!(bt->act_mask & BLK_TC_NOTIFY))
0166         return;
0167 
0168     local_irq_save(flags);
0169     buf = this_cpu_ptr(bt->msg_data);
0170     va_start(args, fmt);
0171     n = vscnprintf(buf, BLK_TN_MAX_MSG, fmt, args);
0172     va_end(args);
0173 
0174 #ifdef CONFIG_BLK_CGROUP
0175     if (css && (blk_tracer_flags.val & TRACE_BLK_OPT_CGROUP))
0176         cgid = cgroup_id(css->cgroup);
0177     else
0178         cgid = 1;
0179 #endif
0180     trace_note(bt, current->pid, BLK_TN_MESSAGE, buf, n, cgid);
0181     local_irq_restore(flags);
0182 }
0183 EXPORT_SYMBOL_GPL(__blk_trace_note_message);
0184 
0185 static int act_log_check(struct blk_trace *bt, u32 what, sector_t sector,
0186              pid_t pid)
0187 {
0188     if (((bt->act_mask << BLK_TC_SHIFT) & what) == 0)
0189         return 1;
0190     if (sector && (sector < bt->start_lba || sector > bt->end_lba))
0191         return 1;
0192     if (bt->pid && pid != bt->pid)
0193         return 1;
0194 
0195     return 0;
0196 }
0197 
0198 /*
0199  * Data direction bit lookup
0200  */
0201 static const u32 ddir_act[2] = { BLK_TC_ACT(BLK_TC_READ),
0202                  BLK_TC_ACT(BLK_TC_WRITE) };
0203 
0204 #define BLK_TC_RAHEAD       BLK_TC_AHEAD
0205 #define BLK_TC_PREFLUSH     BLK_TC_FLUSH
0206 
0207 /* The ilog2() calls fall out because they're constant */
0208 #define MASK_TC_BIT(rw, __name) ((__force u32)(rw & REQ_ ## __name) <<  \
0209       (ilog2(BLK_TC_ ## __name) + BLK_TC_SHIFT - __REQ_ ## __name))
0210 
0211 /*
0212  * The worker for the various blk_add_trace*() types. Fills out a
0213  * blk_io_trace structure and places it in a per-cpu subbuffer.
0214  */
0215 static void __blk_add_trace(struct blk_trace *bt, sector_t sector, int bytes,
0216                 const blk_opf_t opf, u32 what, int error,
0217                 int pdu_len, void *pdu_data, u64 cgid)
0218 {
0219     struct task_struct *tsk = current;
0220     struct ring_buffer_event *event = NULL;
0221     struct trace_buffer *buffer = NULL;
0222     struct blk_io_trace *t;
0223     unsigned long flags = 0;
0224     unsigned long *sequence;
0225     unsigned int trace_ctx = 0;
0226     pid_t pid;
0227     int cpu;
0228     bool blk_tracer = blk_tracer_enabled;
0229     ssize_t cgid_len = cgid ? sizeof(cgid) : 0;
0230     const enum req_op op = opf & REQ_OP_MASK;
0231 
0232     if (unlikely(bt->trace_state != Blktrace_running && !blk_tracer))
0233         return;
0234 
0235     what |= ddir_act[op_is_write(op) ? WRITE : READ];
0236     what |= MASK_TC_BIT(opf, SYNC);
0237     what |= MASK_TC_BIT(opf, RAHEAD);
0238     what |= MASK_TC_BIT(opf, META);
0239     what |= MASK_TC_BIT(opf, PREFLUSH);
0240     what |= MASK_TC_BIT(opf, FUA);
0241     if (op == REQ_OP_DISCARD || op == REQ_OP_SECURE_ERASE)
0242         what |= BLK_TC_ACT(BLK_TC_DISCARD);
0243     if (op == REQ_OP_FLUSH)
0244         what |= BLK_TC_ACT(BLK_TC_FLUSH);
0245     if (cgid)
0246         what |= __BLK_TA_CGROUP;
0247 
0248     pid = tsk->pid;
0249     if (act_log_check(bt, what, sector, pid))
0250         return;
0251     cpu = raw_smp_processor_id();
0252 
0253     if (blk_tracer) {
0254         tracing_record_cmdline(current);
0255 
0256         buffer = blk_tr->array_buffer.buffer;
0257         trace_ctx = tracing_gen_ctx_flags(0);
0258         event = trace_buffer_lock_reserve(buffer, TRACE_BLK,
0259                           sizeof(*t) + pdu_len + cgid_len,
0260                           trace_ctx);
0261         if (!event)
0262             return;
0263         t = ring_buffer_event_data(event);
0264         goto record_it;
0265     }
0266 
0267     if (unlikely(tsk->btrace_seq != blktrace_seq))
0268         trace_note_tsk(tsk);
0269 
0270     /*
0271      * A word about the locking here - we disable interrupts to reserve
0272      * some space in the relay per-cpu buffer, to prevent an irq
0273      * from coming in and stepping on our toes.
0274      */
0275     local_irq_save(flags);
0276     t = relay_reserve(bt->rchan, sizeof(*t) + pdu_len + cgid_len);
0277     if (t) {
0278         sequence = per_cpu_ptr(bt->sequence, cpu);
0279 
0280         t->magic = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION;
0281         t->sequence = ++(*sequence);
0282         t->time = ktime_to_ns(ktime_get());
0283 record_it:
0284         /*
0285          * These two are not needed in ftrace as they are in the
0286          * generic trace_entry, filled by tracing_generic_entry_update,
0287          * but for the trace_event->bin() synthesizer benefit we do it
0288          * here too.
0289          */
0290         t->cpu = cpu;
0291         t->pid = pid;
0292 
0293         t->sector = sector;
0294         t->bytes = bytes;
0295         t->action = what;
0296         t->device = bt->dev;
0297         t->error = error;
0298         t->pdu_len = pdu_len + cgid_len;
0299 
0300         if (cgid_len)
0301             memcpy((void *)t + sizeof(*t), &cgid, cgid_len);
0302         if (pdu_len)
0303             memcpy((void *)t + sizeof(*t) + cgid_len, pdu_data, pdu_len);
0304 
0305         if (blk_tracer) {
0306             trace_buffer_unlock_commit(blk_tr, buffer, event, trace_ctx);
0307             return;
0308         }
0309     }
0310 
0311     local_irq_restore(flags);
0312 }
0313 
0314 static void blk_trace_free(struct request_queue *q, struct blk_trace *bt)
0315 {
0316     relay_close(bt->rchan);
0317 
0318     /*
0319      * If 'bt->dir' is not set, then both 'dropped' and 'msg' are created
0320      * under 'q->debugfs_dir', thus lookup and remove them.
0321      */
0322     if (!bt->dir) {
0323         debugfs_remove(debugfs_lookup("dropped", q->debugfs_dir));
0324         debugfs_remove(debugfs_lookup("msg", q->debugfs_dir));
0325     } else {
0326         debugfs_remove(bt->dir);
0327     }
0328     free_percpu(bt->sequence);
0329     free_percpu(bt->msg_data);
0330     kfree(bt);
0331 }
0332 
0333 static void get_probe_ref(void)
0334 {
0335     mutex_lock(&blk_probe_mutex);
0336     if (++blk_probes_ref == 1)
0337         blk_register_tracepoints();
0338     mutex_unlock(&blk_probe_mutex);
0339 }
0340 
0341 static void put_probe_ref(void)
0342 {
0343     mutex_lock(&blk_probe_mutex);
0344     if (!--blk_probes_ref)
0345         blk_unregister_tracepoints();
0346     mutex_unlock(&blk_probe_mutex);
0347 }
0348 
0349 static void blk_trace_cleanup(struct request_queue *q, struct blk_trace *bt)
0350 {
0351     synchronize_rcu();
0352     blk_trace_free(q, bt);
0353     put_probe_ref();
0354 }
0355 
0356 static int __blk_trace_remove(struct request_queue *q)
0357 {
0358     struct blk_trace *bt;
0359 
0360     bt = rcu_replace_pointer(q->blk_trace, NULL,
0361                  lockdep_is_held(&q->debugfs_mutex));
0362     if (!bt)
0363         return -EINVAL;
0364 
0365     if (bt->trace_state != Blktrace_running)
0366         blk_trace_cleanup(q, bt);
0367 
0368     return 0;
0369 }
0370 
0371 int blk_trace_remove(struct request_queue *q)
0372 {
0373     int ret;
0374 
0375     mutex_lock(&q->debugfs_mutex);
0376     ret = __blk_trace_remove(q);
0377     mutex_unlock(&q->debugfs_mutex);
0378 
0379     return ret;
0380 }
0381 EXPORT_SYMBOL_GPL(blk_trace_remove);
0382 
0383 static ssize_t blk_dropped_read(struct file *filp, char __user *buffer,
0384                 size_t count, loff_t *ppos)
0385 {
0386     struct blk_trace *bt = filp->private_data;
0387     char buf[16];
0388 
0389     snprintf(buf, sizeof(buf), "%u\n", atomic_read(&bt->dropped));
0390 
0391     return simple_read_from_buffer(buffer, count, ppos, buf, strlen(buf));
0392 }
0393 
0394 static const struct file_operations blk_dropped_fops = {
0395     .owner =    THIS_MODULE,
0396     .open =     simple_open,
0397     .read =     blk_dropped_read,
0398     .llseek =   default_llseek,
0399 };
0400 
0401 static ssize_t blk_msg_write(struct file *filp, const char __user *buffer,
0402                 size_t count, loff_t *ppos)
0403 {
0404     char *msg;
0405     struct blk_trace *bt;
0406 
0407     if (count >= BLK_TN_MAX_MSG)
0408         return -EINVAL;
0409 
0410     msg = memdup_user_nul(buffer, count);
0411     if (IS_ERR(msg))
0412         return PTR_ERR(msg);
0413 
0414     bt = filp->private_data;
0415     __blk_trace_note_message(bt, NULL, "%s", msg);
0416     kfree(msg);
0417 
0418     return count;
0419 }
0420 
0421 static const struct file_operations blk_msg_fops = {
0422     .owner =    THIS_MODULE,
0423     .open =     simple_open,
0424     .write =    blk_msg_write,
0425     .llseek =   noop_llseek,
0426 };
0427 
0428 /*
0429  * Keep track of how many times we encountered a full subbuffer, to aid
0430  * the user space app in telling how many lost events there were.
0431  */
0432 static int blk_subbuf_start_callback(struct rchan_buf *buf, void *subbuf,
0433                      void *prev_subbuf, size_t prev_padding)
0434 {
0435     struct blk_trace *bt;
0436 
0437     if (!relay_buf_full(buf))
0438         return 1;
0439 
0440     bt = buf->chan->private_data;
0441     atomic_inc(&bt->dropped);
0442     return 0;
0443 }
0444 
0445 static int blk_remove_buf_file_callback(struct dentry *dentry)
0446 {
0447     debugfs_remove(dentry);
0448 
0449     return 0;
0450 }
0451 
0452 static struct dentry *blk_create_buf_file_callback(const char *filename,
0453                            struct dentry *parent,
0454                            umode_t mode,
0455                            struct rchan_buf *buf,
0456                            int *is_global)
0457 {
0458     return debugfs_create_file(filename, mode, parent, buf,
0459                     &relay_file_operations);
0460 }
0461 
0462 static const struct rchan_callbacks blk_relay_callbacks = {
0463     .subbuf_start       = blk_subbuf_start_callback,
0464     .create_buf_file    = blk_create_buf_file_callback,
0465     .remove_buf_file    = blk_remove_buf_file_callback,
0466 };
0467 
0468 static void blk_trace_setup_lba(struct blk_trace *bt,
0469                 struct block_device *bdev)
0470 {
0471     if (bdev) {
0472         bt->start_lba = bdev->bd_start_sect;
0473         bt->end_lba = bdev->bd_start_sect + bdev_nr_sectors(bdev);
0474     } else {
0475         bt->start_lba = 0;
0476         bt->end_lba = -1ULL;
0477     }
0478 }
0479 
0480 /*
0481  * Setup everything required to start tracing
0482  */
0483 static int do_blk_trace_setup(struct request_queue *q, char *name, dev_t dev,
0484                   struct block_device *bdev,
0485                   struct blk_user_trace_setup *buts)
0486 {
0487     struct blk_trace *bt = NULL;
0488     struct dentry *dir = NULL;
0489     int ret;
0490 
0491     lockdep_assert_held(&q->debugfs_mutex);
0492 
0493     if (!buts->buf_size || !buts->buf_nr)
0494         return -EINVAL;
0495 
0496     strncpy(buts->name, name, BLKTRACE_BDEV_SIZE);
0497     buts->name[BLKTRACE_BDEV_SIZE - 1] = '\0';
0498 
0499     /*
0500      * some device names have larger paths - convert the slashes
0501      * to underscores for this to work as expected
0502      */
0503     strreplace(buts->name, '/', '_');
0504 
0505     /*
0506      * bdev can be NULL, as with scsi-generic, this is a helpful as
0507      * we can be.
0508      */
0509     if (rcu_dereference_protected(q->blk_trace,
0510                       lockdep_is_held(&q->debugfs_mutex))) {
0511         pr_warn("Concurrent blktraces are not allowed on %s\n",
0512             buts->name);
0513         return -EBUSY;
0514     }
0515 
0516     bt = kzalloc(sizeof(*bt), GFP_KERNEL);
0517     if (!bt)
0518         return -ENOMEM;
0519 
0520     ret = -ENOMEM;
0521     bt->sequence = alloc_percpu(unsigned long);
0522     if (!bt->sequence)
0523         goto err;
0524 
0525     bt->msg_data = __alloc_percpu(BLK_TN_MAX_MSG, __alignof__(char));
0526     if (!bt->msg_data)
0527         goto err;
0528 
0529     /*
0530      * When tracing the whole disk reuse the existing debugfs directory
0531      * created by the block layer on init. For partitions block devices,
0532      * and scsi-generic block devices we create a temporary new debugfs
0533      * directory that will be removed once the trace ends.
0534      */
0535     if (bdev && !bdev_is_partition(bdev))
0536         dir = q->debugfs_dir;
0537     else
0538         bt->dir = dir = debugfs_create_dir(buts->name, blk_debugfs_root);
0539 
0540     /*
0541      * As blktrace relies on debugfs for its interface the debugfs directory
0542      * is required, contrary to the usual mantra of not checking for debugfs
0543      * files or directories.
0544      */
0545     if (IS_ERR_OR_NULL(dir)) {
0546         pr_warn("debugfs_dir not present for %s so skipping\n",
0547             buts->name);
0548         ret = -ENOENT;
0549         goto err;
0550     }
0551 
0552     bt->dev = dev;
0553     atomic_set(&bt->dropped, 0);
0554     INIT_LIST_HEAD(&bt->running_list);
0555 
0556     ret = -EIO;
0557     debugfs_create_file("dropped", 0444, dir, bt, &blk_dropped_fops);
0558     debugfs_create_file("msg", 0222, dir, bt, &blk_msg_fops);
0559 
0560     bt->rchan = relay_open("trace", dir, buts->buf_size,
0561                 buts->buf_nr, &blk_relay_callbacks, bt);
0562     if (!bt->rchan)
0563         goto err;
0564 
0565     bt->act_mask = buts->act_mask;
0566     if (!bt->act_mask)
0567         bt->act_mask = (u16) -1;
0568 
0569     blk_trace_setup_lba(bt, bdev);
0570 
0571     /* overwrite with user settings */
0572     if (buts->start_lba)
0573         bt->start_lba = buts->start_lba;
0574     if (buts->end_lba)
0575         bt->end_lba = buts->end_lba;
0576 
0577     bt->pid = buts->pid;
0578     bt->trace_state = Blktrace_setup;
0579 
0580     rcu_assign_pointer(q->blk_trace, bt);
0581     get_probe_ref();
0582 
0583     ret = 0;
0584 err:
0585     if (ret)
0586         blk_trace_free(q, bt);
0587     return ret;
0588 }
0589 
0590 static int __blk_trace_setup(struct request_queue *q, char *name, dev_t dev,
0591                  struct block_device *bdev, char __user *arg)
0592 {
0593     struct blk_user_trace_setup buts;
0594     int ret;
0595 
0596     ret = copy_from_user(&buts, arg, sizeof(buts));
0597     if (ret)
0598         return -EFAULT;
0599 
0600     ret = do_blk_trace_setup(q, name, dev, bdev, &buts);
0601     if (ret)
0602         return ret;
0603 
0604     if (copy_to_user(arg, &buts, sizeof(buts))) {
0605         __blk_trace_remove(q);
0606         return -EFAULT;
0607     }
0608     return 0;
0609 }
0610 
0611 int blk_trace_setup(struct request_queue *q, char *name, dev_t dev,
0612             struct block_device *bdev,
0613             char __user *arg)
0614 {
0615     int ret;
0616 
0617     mutex_lock(&q->debugfs_mutex);
0618     ret = __blk_trace_setup(q, name, dev, bdev, arg);
0619     mutex_unlock(&q->debugfs_mutex);
0620 
0621     return ret;
0622 }
0623 EXPORT_SYMBOL_GPL(blk_trace_setup);
0624 
0625 #if defined(CONFIG_COMPAT) && defined(CONFIG_X86_64)
0626 static int compat_blk_trace_setup(struct request_queue *q, char *name,
0627                   dev_t dev, struct block_device *bdev,
0628                   char __user *arg)
0629 {
0630     struct blk_user_trace_setup buts;
0631     struct compat_blk_user_trace_setup cbuts;
0632     int ret;
0633 
0634     if (copy_from_user(&cbuts, arg, sizeof(cbuts)))
0635         return -EFAULT;
0636 
0637     buts = (struct blk_user_trace_setup) {
0638         .act_mask = cbuts.act_mask,
0639         .buf_size = cbuts.buf_size,
0640         .buf_nr = cbuts.buf_nr,
0641         .start_lba = cbuts.start_lba,
0642         .end_lba = cbuts.end_lba,
0643         .pid = cbuts.pid,
0644     };
0645 
0646     ret = do_blk_trace_setup(q, name, dev, bdev, &buts);
0647     if (ret)
0648         return ret;
0649 
0650     if (copy_to_user(arg, &buts.name, ARRAY_SIZE(buts.name))) {
0651         __blk_trace_remove(q);
0652         return -EFAULT;
0653     }
0654 
0655     return 0;
0656 }
0657 #endif
0658 
0659 static int __blk_trace_startstop(struct request_queue *q, int start)
0660 {
0661     int ret;
0662     struct blk_trace *bt;
0663 
0664     bt = rcu_dereference_protected(q->blk_trace,
0665                        lockdep_is_held(&q->debugfs_mutex));
0666     if (bt == NULL)
0667         return -EINVAL;
0668 
0669     /*
0670      * For starting a trace, we can transition from a setup or stopped
0671      * trace. For stopping a trace, the state must be running
0672      */
0673     ret = -EINVAL;
0674     if (start) {
0675         if (bt->trace_state == Blktrace_setup ||
0676             bt->trace_state == Blktrace_stopped) {
0677             blktrace_seq++;
0678             smp_mb();
0679             bt->trace_state = Blktrace_running;
0680             raw_spin_lock_irq(&running_trace_lock);
0681             list_add(&bt->running_list, &running_trace_list);
0682             raw_spin_unlock_irq(&running_trace_lock);
0683 
0684             trace_note_time(bt);
0685             ret = 0;
0686         }
0687     } else {
0688         if (bt->trace_state == Blktrace_running) {
0689             bt->trace_state = Blktrace_stopped;
0690             raw_spin_lock_irq(&running_trace_lock);
0691             list_del_init(&bt->running_list);
0692             raw_spin_unlock_irq(&running_trace_lock);
0693             relay_flush(bt->rchan);
0694             ret = 0;
0695         }
0696     }
0697 
0698     return ret;
0699 }
0700 
0701 int blk_trace_startstop(struct request_queue *q, int start)
0702 {
0703     int ret;
0704 
0705     mutex_lock(&q->debugfs_mutex);
0706     ret = __blk_trace_startstop(q, start);
0707     mutex_unlock(&q->debugfs_mutex);
0708 
0709     return ret;
0710 }
0711 EXPORT_SYMBOL_GPL(blk_trace_startstop);
0712 
0713 /*
0714  * When reading or writing the blktrace sysfs files, the references to the
0715  * opened sysfs or device files should prevent the underlying block device
0716  * from being removed. So no further delete protection is really needed.
0717  */
0718 
0719 /**
0720  * blk_trace_ioctl: - handle the ioctls associated with tracing
0721  * @bdev:   the block device
0722  * @cmd:    the ioctl cmd
0723  * @arg:    the argument data, if any
0724  *
0725  **/
0726 int blk_trace_ioctl(struct block_device *bdev, unsigned cmd, char __user *arg)
0727 {
0728     struct request_queue *q;
0729     int ret, start = 0;
0730     char b[BDEVNAME_SIZE];
0731 
0732     q = bdev_get_queue(bdev);
0733     if (!q)
0734         return -ENXIO;
0735 
0736     mutex_lock(&q->debugfs_mutex);
0737 
0738     switch (cmd) {
0739     case BLKTRACESETUP:
0740         snprintf(b, sizeof(b), "%pg", bdev);
0741         ret = __blk_trace_setup(q, b, bdev->bd_dev, bdev, arg);
0742         break;
0743 #if defined(CONFIG_COMPAT) && defined(CONFIG_X86_64)
0744     case BLKTRACESETUP32:
0745         snprintf(b, sizeof(b), "%pg", bdev);
0746         ret = compat_blk_trace_setup(q, b, bdev->bd_dev, bdev, arg);
0747         break;
0748 #endif
0749     case BLKTRACESTART:
0750         start = 1;
0751         fallthrough;
0752     case BLKTRACESTOP:
0753         ret = __blk_trace_startstop(q, start);
0754         break;
0755     case BLKTRACETEARDOWN:
0756         ret = __blk_trace_remove(q);
0757         break;
0758     default:
0759         ret = -ENOTTY;
0760         break;
0761     }
0762 
0763     mutex_unlock(&q->debugfs_mutex);
0764     return ret;
0765 }
0766 
0767 /**
0768  * blk_trace_shutdown: - stop and cleanup trace structures
0769  * @q:    the request queue associated with the device
0770  *
0771  **/
0772 void blk_trace_shutdown(struct request_queue *q)
0773 {
0774     if (rcu_dereference_protected(q->blk_trace,
0775                       lockdep_is_held(&q->debugfs_mutex))) {
0776         __blk_trace_startstop(q, 0);
0777         __blk_trace_remove(q);
0778     }
0779 }
0780 
0781 #ifdef CONFIG_BLK_CGROUP
0782 static u64 blk_trace_bio_get_cgid(struct request_queue *q, struct bio *bio)
0783 {
0784     struct cgroup_subsys_state *blkcg_css;
0785     struct blk_trace *bt;
0786 
0787     /* We don't use the 'bt' value here except as an optimization... */
0788     bt = rcu_dereference_protected(q->blk_trace, 1);
0789     if (!bt || !(blk_tracer_flags.val & TRACE_BLK_OPT_CGROUP))
0790         return 0;
0791 
0792     blkcg_css = bio_blkcg_css(bio);
0793     if (!blkcg_css)
0794         return 0;
0795     return cgroup_id(blkcg_css->cgroup);
0796 }
0797 #else
0798 static u64 blk_trace_bio_get_cgid(struct request_queue *q, struct bio *bio)
0799 {
0800     return 0;
0801 }
0802 #endif
0803 
0804 static u64
0805 blk_trace_request_get_cgid(struct request *rq)
0806 {
0807     if (!rq->bio)
0808         return 0;
0809     /* Use the first bio */
0810     return blk_trace_bio_get_cgid(rq->q, rq->bio);
0811 }
0812 
0813 /*
0814  * blktrace probes
0815  */
0816 
0817 /**
0818  * blk_add_trace_rq - Add a trace for a request oriented action
0819  * @rq:     the source request
0820  * @error:  return status to log
0821  * @nr_bytes:   number of completed bytes
0822  * @what:   the action
0823  * @cgid:   the cgroup info
0824  *
0825  * Description:
0826  *     Records an action against a request. Will log the bio offset + size.
0827  *
0828  **/
0829 static void blk_add_trace_rq(struct request *rq, blk_status_t error,
0830                  unsigned int nr_bytes, u32 what, u64 cgid)
0831 {
0832     struct blk_trace *bt;
0833 
0834     rcu_read_lock();
0835     bt = rcu_dereference(rq->q->blk_trace);
0836     if (likely(!bt)) {
0837         rcu_read_unlock();
0838         return;
0839     }
0840 
0841     if (blk_rq_is_passthrough(rq))
0842         what |= BLK_TC_ACT(BLK_TC_PC);
0843     else
0844         what |= BLK_TC_ACT(BLK_TC_FS);
0845 
0846     __blk_add_trace(bt, blk_rq_trace_sector(rq), nr_bytes, rq->cmd_flags,
0847             what, blk_status_to_errno(error), 0, NULL, cgid);
0848     rcu_read_unlock();
0849 }
0850 
0851 static void blk_add_trace_rq_insert(void *ignore, struct request *rq)
0852 {
0853     blk_add_trace_rq(rq, 0, blk_rq_bytes(rq), BLK_TA_INSERT,
0854              blk_trace_request_get_cgid(rq));
0855 }
0856 
0857 static void blk_add_trace_rq_issue(void *ignore, struct request *rq)
0858 {
0859     blk_add_trace_rq(rq, 0, blk_rq_bytes(rq), BLK_TA_ISSUE,
0860              blk_trace_request_get_cgid(rq));
0861 }
0862 
0863 static void blk_add_trace_rq_merge(void *ignore, struct request *rq)
0864 {
0865     blk_add_trace_rq(rq, 0, blk_rq_bytes(rq), BLK_TA_BACKMERGE,
0866              blk_trace_request_get_cgid(rq));
0867 }
0868 
0869 static void blk_add_trace_rq_requeue(void *ignore, struct request *rq)
0870 {
0871     blk_add_trace_rq(rq, 0, blk_rq_bytes(rq), BLK_TA_REQUEUE,
0872              blk_trace_request_get_cgid(rq));
0873 }
0874 
0875 static void blk_add_trace_rq_complete(void *ignore, struct request *rq,
0876             blk_status_t error, unsigned int nr_bytes)
0877 {
0878     blk_add_trace_rq(rq, error, nr_bytes, BLK_TA_COMPLETE,
0879              blk_trace_request_get_cgid(rq));
0880 }
0881 
0882 /**
0883  * blk_add_trace_bio - Add a trace for a bio oriented action
0884  * @q:      queue the io is for
0885  * @bio:    the source bio
0886  * @what:   the action
0887  * @error:  error, if any
0888  *
0889  * Description:
0890  *     Records an action against a bio. Will log the bio offset + size.
0891  *
0892  **/
0893 static void blk_add_trace_bio(struct request_queue *q, struct bio *bio,
0894                   u32 what, int error)
0895 {
0896     struct blk_trace *bt;
0897 
0898     rcu_read_lock();
0899     bt = rcu_dereference(q->blk_trace);
0900     if (likely(!bt)) {
0901         rcu_read_unlock();
0902         return;
0903     }
0904 
0905     __blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size,
0906             bio->bi_opf, what, error, 0, NULL,
0907             blk_trace_bio_get_cgid(q, bio));
0908     rcu_read_unlock();
0909 }
0910 
0911 static void blk_add_trace_bio_bounce(void *ignore, struct bio *bio)
0912 {
0913     blk_add_trace_bio(bio->bi_bdev->bd_disk->queue, bio, BLK_TA_BOUNCE, 0);
0914 }
0915 
0916 static void blk_add_trace_bio_complete(void *ignore,
0917                        struct request_queue *q, struct bio *bio)
0918 {
0919     blk_add_trace_bio(q, bio, BLK_TA_COMPLETE,
0920               blk_status_to_errno(bio->bi_status));
0921 }
0922 
0923 static void blk_add_trace_bio_backmerge(void *ignore, struct bio *bio)
0924 {
0925     blk_add_trace_bio(bio->bi_bdev->bd_disk->queue, bio, BLK_TA_BACKMERGE,
0926             0);
0927 }
0928 
0929 static void blk_add_trace_bio_frontmerge(void *ignore, struct bio *bio)
0930 {
0931     blk_add_trace_bio(bio->bi_bdev->bd_disk->queue, bio, BLK_TA_FRONTMERGE,
0932             0);
0933 }
0934 
0935 static void blk_add_trace_bio_queue(void *ignore, struct bio *bio)
0936 {
0937     blk_add_trace_bio(bio->bi_bdev->bd_disk->queue, bio, BLK_TA_QUEUE, 0);
0938 }
0939 
0940 static void blk_add_trace_getrq(void *ignore, struct bio *bio)
0941 {
0942     blk_add_trace_bio(bio->bi_bdev->bd_disk->queue, bio, BLK_TA_GETRQ, 0);
0943 }
0944 
0945 static void blk_add_trace_plug(void *ignore, struct request_queue *q)
0946 {
0947     struct blk_trace *bt;
0948 
0949     rcu_read_lock();
0950     bt = rcu_dereference(q->blk_trace);
0951     if (bt)
0952         __blk_add_trace(bt, 0, 0, 0, BLK_TA_PLUG, 0, 0, NULL, 0);
0953     rcu_read_unlock();
0954 }
0955 
0956 static void blk_add_trace_unplug(void *ignore, struct request_queue *q,
0957                     unsigned int depth, bool explicit)
0958 {
0959     struct blk_trace *bt;
0960 
0961     rcu_read_lock();
0962     bt = rcu_dereference(q->blk_trace);
0963     if (bt) {
0964         __be64 rpdu = cpu_to_be64(depth);
0965         u32 what;
0966 
0967         if (explicit)
0968             what = BLK_TA_UNPLUG_IO;
0969         else
0970             what = BLK_TA_UNPLUG_TIMER;
0971 
0972         __blk_add_trace(bt, 0, 0, 0, what, 0, sizeof(rpdu), &rpdu, 0);
0973     }
0974     rcu_read_unlock();
0975 }
0976 
0977 static void blk_add_trace_split(void *ignore, struct bio *bio, unsigned int pdu)
0978 {
0979     struct request_queue *q = bio->bi_bdev->bd_disk->queue;
0980     struct blk_trace *bt;
0981 
0982     rcu_read_lock();
0983     bt = rcu_dereference(q->blk_trace);
0984     if (bt) {
0985         __be64 rpdu = cpu_to_be64(pdu);
0986 
0987         __blk_add_trace(bt, bio->bi_iter.bi_sector,
0988                 bio->bi_iter.bi_size, bio->bi_opf, BLK_TA_SPLIT,
0989                 blk_status_to_errno(bio->bi_status),
0990                 sizeof(rpdu), &rpdu,
0991                 blk_trace_bio_get_cgid(q, bio));
0992     }
0993     rcu_read_unlock();
0994 }
0995 
0996 /**
0997  * blk_add_trace_bio_remap - Add a trace for a bio-remap operation
0998  * @ignore: trace callback data parameter (not used)
0999  * @bio:    the source bio
1000  * @dev:    source device
1001  * @from:   source sector
1002  *
1003  * Called after a bio is remapped to a different device and/or sector.
1004  **/
1005 static void blk_add_trace_bio_remap(void *ignore, struct bio *bio, dev_t dev,
1006                     sector_t from)
1007 {
1008     struct request_queue *q = bio->bi_bdev->bd_disk->queue;
1009     struct blk_trace *bt;
1010     struct blk_io_trace_remap r;
1011 
1012     rcu_read_lock();
1013     bt = rcu_dereference(q->blk_trace);
1014     if (likely(!bt)) {
1015         rcu_read_unlock();
1016         return;
1017     }
1018 
1019     r.device_from = cpu_to_be32(dev);
1020     r.device_to   = cpu_to_be32(bio_dev(bio));
1021     r.sector_from = cpu_to_be64(from);
1022 
1023     __blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size,
1024             bio->bi_opf, BLK_TA_REMAP,
1025             blk_status_to_errno(bio->bi_status),
1026             sizeof(r), &r, blk_trace_bio_get_cgid(q, bio));
1027     rcu_read_unlock();
1028 }
1029 
1030 /**
1031  * blk_add_trace_rq_remap - Add a trace for a request-remap operation
1032  * @ignore: trace callback data parameter (not used)
1033  * @rq:     the source request
1034  * @dev:    target device
1035  * @from:   source sector
1036  *
1037  * Description:
1038  *     Device mapper remaps request to other devices.
1039  *     Add a trace for that action.
1040  *
1041  **/
1042 static void blk_add_trace_rq_remap(void *ignore, struct request *rq, dev_t dev,
1043                    sector_t from)
1044 {
1045     struct blk_trace *bt;
1046     struct blk_io_trace_remap r;
1047 
1048     rcu_read_lock();
1049     bt = rcu_dereference(rq->q->blk_trace);
1050     if (likely(!bt)) {
1051         rcu_read_unlock();
1052         return;
1053     }
1054 
1055     r.device_from = cpu_to_be32(dev);
1056     r.device_to   = cpu_to_be32(disk_devt(rq->q->disk));
1057     r.sector_from = cpu_to_be64(from);
1058 
1059     __blk_add_trace(bt, blk_rq_pos(rq), blk_rq_bytes(rq),
1060             rq->cmd_flags, BLK_TA_REMAP, 0,
1061             sizeof(r), &r, blk_trace_request_get_cgid(rq));
1062     rcu_read_unlock();
1063 }
1064 
1065 /**
1066  * blk_add_driver_data - Add binary message with driver-specific data
1067  * @rq:     io request
1068  * @data:   driver-specific data
1069  * @len:    length of driver-specific data
1070  *
1071  * Description:
1072  *     Some drivers might want to write driver-specific data per request.
1073  *
1074  **/
1075 void blk_add_driver_data(struct request *rq, void *data, size_t len)
1076 {
1077     struct blk_trace *bt;
1078 
1079     rcu_read_lock();
1080     bt = rcu_dereference(rq->q->blk_trace);
1081     if (likely(!bt)) {
1082         rcu_read_unlock();
1083         return;
1084     }
1085 
1086     __blk_add_trace(bt, blk_rq_trace_sector(rq), blk_rq_bytes(rq), 0,
1087                 BLK_TA_DRV_DATA, 0, len, data,
1088                 blk_trace_request_get_cgid(rq));
1089     rcu_read_unlock();
1090 }
1091 EXPORT_SYMBOL_GPL(blk_add_driver_data);
1092 
1093 static void blk_register_tracepoints(void)
1094 {
1095     int ret;
1096 
1097     ret = register_trace_block_rq_insert(blk_add_trace_rq_insert, NULL);
1098     WARN_ON(ret);
1099     ret = register_trace_block_rq_issue(blk_add_trace_rq_issue, NULL);
1100     WARN_ON(ret);
1101     ret = register_trace_block_rq_merge(blk_add_trace_rq_merge, NULL);
1102     WARN_ON(ret);
1103     ret = register_trace_block_rq_requeue(blk_add_trace_rq_requeue, NULL);
1104     WARN_ON(ret);
1105     ret = register_trace_block_rq_complete(blk_add_trace_rq_complete, NULL);
1106     WARN_ON(ret);
1107     ret = register_trace_block_bio_bounce(blk_add_trace_bio_bounce, NULL);
1108     WARN_ON(ret);
1109     ret = register_trace_block_bio_complete(blk_add_trace_bio_complete, NULL);
1110     WARN_ON(ret);
1111     ret = register_trace_block_bio_backmerge(blk_add_trace_bio_backmerge, NULL);
1112     WARN_ON(ret);
1113     ret = register_trace_block_bio_frontmerge(blk_add_trace_bio_frontmerge, NULL);
1114     WARN_ON(ret);
1115     ret = register_trace_block_bio_queue(blk_add_trace_bio_queue, NULL);
1116     WARN_ON(ret);
1117     ret = register_trace_block_getrq(blk_add_trace_getrq, NULL);
1118     WARN_ON(ret);
1119     ret = register_trace_block_plug(blk_add_trace_plug, NULL);
1120     WARN_ON(ret);
1121     ret = register_trace_block_unplug(blk_add_trace_unplug, NULL);
1122     WARN_ON(ret);
1123     ret = register_trace_block_split(blk_add_trace_split, NULL);
1124     WARN_ON(ret);
1125     ret = register_trace_block_bio_remap(blk_add_trace_bio_remap, NULL);
1126     WARN_ON(ret);
1127     ret = register_trace_block_rq_remap(blk_add_trace_rq_remap, NULL);
1128     WARN_ON(ret);
1129 }
1130 
1131 static void blk_unregister_tracepoints(void)
1132 {
1133     unregister_trace_block_rq_remap(blk_add_trace_rq_remap, NULL);
1134     unregister_trace_block_bio_remap(blk_add_trace_bio_remap, NULL);
1135     unregister_trace_block_split(blk_add_trace_split, NULL);
1136     unregister_trace_block_unplug(blk_add_trace_unplug, NULL);
1137     unregister_trace_block_plug(blk_add_trace_plug, NULL);
1138     unregister_trace_block_getrq(blk_add_trace_getrq, NULL);
1139     unregister_trace_block_bio_queue(blk_add_trace_bio_queue, NULL);
1140     unregister_trace_block_bio_frontmerge(blk_add_trace_bio_frontmerge, NULL);
1141     unregister_trace_block_bio_backmerge(blk_add_trace_bio_backmerge, NULL);
1142     unregister_trace_block_bio_complete(blk_add_trace_bio_complete, NULL);
1143     unregister_trace_block_bio_bounce(blk_add_trace_bio_bounce, NULL);
1144     unregister_trace_block_rq_complete(blk_add_trace_rq_complete, NULL);
1145     unregister_trace_block_rq_requeue(blk_add_trace_rq_requeue, NULL);
1146     unregister_trace_block_rq_merge(blk_add_trace_rq_merge, NULL);
1147     unregister_trace_block_rq_issue(blk_add_trace_rq_issue, NULL);
1148     unregister_trace_block_rq_insert(blk_add_trace_rq_insert, NULL);
1149 
1150     tracepoint_synchronize_unregister();
1151 }
1152 
1153 /*
1154  * struct blk_io_tracer formatting routines
1155  */
1156 
1157 static void fill_rwbs(char *rwbs, const struct blk_io_trace *t)
1158 {
1159     int i = 0;
1160     int tc = t->action >> BLK_TC_SHIFT;
1161 
1162     if ((t->action & ~__BLK_TN_CGROUP) == BLK_TN_MESSAGE) {
1163         rwbs[i++] = 'N';
1164         goto out;
1165     }
1166 
1167     if (tc & BLK_TC_FLUSH)
1168         rwbs[i++] = 'F';
1169 
1170     if (tc & BLK_TC_DISCARD)
1171         rwbs[i++] = 'D';
1172     else if (tc & BLK_TC_WRITE)
1173         rwbs[i++] = 'W';
1174     else if (t->bytes)
1175         rwbs[i++] = 'R';
1176     else
1177         rwbs[i++] = 'N';
1178 
1179     if (tc & BLK_TC_FUA)
1180         rwbs[i++] = 'F';
1181     if (tc & BLK_TC_AHEAD)
1182         rwbs[i++] = 'A';
1183     if (tc & BLK_TC_SYNC)
1184         rwbs[i++] = 'S';
1185     if (tc & BLK_TC_META)
1186         rwbs[i++] = 'M';
1187 out:
1188     rwbs[i] = '\0';
1189 }
1190 
1191 static inline
1192 const struct blk_io_trace *te_blk_io_trace(const struct trace_entry *ent)
1193 {
1194     return (const struct blk_io_trace *)ent;
1195 }
1196 
1197 static inline const void *pdu_start(const struct trace_entry *ent, bool has_cg)
1198 {
1199     return (void *)(te_blk_io_trace(ent) + 1) + (has_cg ? sizeof(u64) : 0);
1200 }
1201 
1202 static inline u64 t_cgid(const struct trace_entry *ent)
1203 {
1204     return *(u64 *)(te_blk_io_trace(ent) + 1);
1205 }
1206 
1207 static inline int pdu_real_len(const struct trace_entry *ent, bool has_cg)
1208 {
1209     return te_blk_io_trace(ent)->pdu_len - (has_cg ? sizeof(u64) : 0);
1210 }
1211 
1212 static inline u32 t_action(const struct trace_entry *ent)
1213 {
1214     return te_blk_io_trace(ent)->action;
1215 }
1216 
1217 static inline u32 t_bytes(const struct trace_entry *ent)
1218 {
1219     return te_blk_io_trace(ent)->bytes;
1220 }
1221 
1222 static inline u32 t_sec(const struct trace_entry *ent)
1223 {
1224     return te_blk_io_trace(ent)->bytes >> 9;
1225 }
1226 
1227 static inline unsigned long long t_sector(const struct trace_entry *ent)
1228 {
1229     return te_blk_io_trace(ent)->sector;
1230 }
1231 
1232 static inline __u16 t_error(const struct trace_entry *ent)
1233 {
1234     return te_blk_io_trace(ent)->error;
1235 }
1236 
1237 static __u64 get_pdu_int(const struct trace_entry *ent, bool has_cg)
1238 {
1239     const __be64 *val = pdu_start(ent, has_cg);
1240     return be64_to_cpu(*val);
1241 }
1242 
1243 typedef void (blk_log_action_t) (struct trace_iterator *iter, const char *act,
1244     bool has_cg);
1245 
1246 static void blk_log_action_classic(struct trace_iterator *iter, const char *act,
1247     bool has_cg)
1248 {
1249     char rwbs[RWBS_LEN];
1250     unsigned long long ts  = iter->ts;
1251     unsigned long nsec_rem = do_div(ts, NSEC_PER_SEC);
1252     unsigned secs          = (unsigned long)ts;
1253     const struct blk_io_trace *t = te_blk_io_trace(iter->ent);
1254 
1255     fill_rwbs(rwbs, t);
1256 
1257     trace_seq_printf(&iter->seq,
1258              "%3d,%-3d %2d %5d.%09lu %5u %2s %3s ",
1259              MAJOR(t->device), MINOR(t->device), iter->cpu,
1260              secs, nsec_rem, iter->ent->pid, act, rwbs);
1261 }
1262 
1263 static void blk_log_action(struct trace_iterator *iter, const char *act,
1264     bool has_cg)
1265 {
1266     char rwbs[RWBS_LEN];
1267     const struct blk_io_trace *t = te_blk_io_trace(iter->ent);
1268 
1269     fill_rwbs(rwbs, t);
1270     if (has_cg) {
1271         u64 id = t_cgid(iter->ent);
1272 
1273         if (blk_tracer_flags.val & TRACE_BLK_OPT_CGNAME) {
1274             char blkcg_name_buf[NAME_MAX + 1] = "<...>";
1275 
1276             cgroup_path_from_kernfs_id(id, blkcg_name_buf,
1277                 sizeof(blkcg_name_buf));
1278             trace_seq_printf(&iter->seq, "%3d,%-3d %s %2s %3s ",
1279                  MAJOR(t->device), MINOR(t->device),
1280                  blkcg_name_buf, act, rwbs);
1281         } else {
1282             /*
1283              * The cgid portion used to be "INO,GEN".  Userland
1284              * builds a FILEID_INO32_GEN fid out of them and
1285              * opens the cgroup using open_by_handle_at(2).
1286              * While 32bit ino setups are still the same, 64bit
1287              * ones now use the 64bit ino as the whole ID and
1288              * no longer use generation.
1289              *
1290              * Regardless of the content, always output
1291              * "LOW32,HIGH32" so that FILEID_INO32_GEN fid can
1292              * be mapped back to @id on both 64 and 32bit ino
1293              * setups.  See __kernfs_fh_to_dentry().
1294              */
1295             trace_seq_printf(&iter->seq,
1296                  "%3d,%-3d %llx,%-llx %2s %3s ",
1297                  MAJOR(t->device), MINOR(t->device),
1298                  id & U32_MAX, id >> 32, act, rwbs);
1299         }
1300     } else
1301         trace_seq_printf(&iter->seq, "%3d,%-3d %2s %3s ",
1302                  MAJOR(t->device), MINOR(t->device), act, rwbs);
1303 }
1304 
1305 static void blk_log_dump_pdu(struct trace_seq *s,
1306     const struct trace_entry *ent, bool has_cg)
1307 {
1308     const unsigned char *pdu_buf;
1309     int pdu_len;
1310     int i, end;
1311 
1312     pdu_buf = pdu_start(ent, has_cg);
1313     pdu_len = pdu_real_len(ent, has_cg);
1314 
1315     if (!pdu_len)
1316         return;
1317 
1318     /* find the last zero that needs to be printed */
1319     for (end = pdu_len - 1; end >= 0; end--)
1320         if (pdu_buf[end])
1321             break;
1322     end++;
1323 
1324     trace_seq_putc(s, '(');
1325 
1326     for (i = 0; i < pdu_len; i++) {
1327 
1328         trace_seq_printf(s, "%s%02x",
1329                  i == 0 ? "" : " ", pdu_buf[i]);
1330 
1331         /*
1332          * stop when the rest is just zeros and indicate so
1333          * with a ".." appended
1334          */
1335         if (i == end && end != pdu_len - 1) {
1336             trace_seq_puts(s, " ..) ");
1337             return;
1338         }
1339     }
1340 
1341     trace_seq_puts(s, ") ");
1342 }
1343 
1344 static void blk_log_generic(struct trace_seq *s, const struct trace_entry *ent, bool has_cg)
1345 {
1346     char cmd[TASK_COMM_LEN];
1347 
1348     trace_find_cmdline(ent->pid, cmd);
1349 
1350     if (t_action(ent) & BLK_TC_ACT(BLK_TC_PC)) {
1351         trace_seq_printf(s, "%u ", t_bytes(ent));
1352         blk_log_dump_pdu(s, ent, has_cg);
1353         trace_seq_printf(s, "[%s]\n", cmd);
1354     } else {
1355         if (t_sec(ent))
1356             trace_seq_printf(s, "%llu + %u [%s]\n",
1357                         t_sector(ent), t_sec(ent), cmd);
1358         else
1359             trace_seq_printf(s, "[%s]\n", cmd);
1360     }
1361 }
1362 
1363 static void blk_log_with_error(struct trace_seq *s,
1364                   const struct trace_entry *ent, bool has_cg)
1365 {
1366     if (t_action(ent) & BLK_TC_ACT(BLK_TC_PC)) {
1367         blk_log_dump_pdu(s, ent, has_cg);
1368         trace_seq_printf(s, "[%d]\n", t_error(ent));
1369     } else {
1370         if (t_sec(ent))
1371             trace_seq_printf(s, "%llu + %u [%d]\n",
1372                      t_sector(ent),
1373                      t_sec(ent), t_error(ent));
1374         else
1375             trace_seq_printf(s, "%llu [%d]\n",
1376                      t_sector(ent), t_error(ent));
1377     }
1378 }
1379 
1380 static void blk_log_remap(struct trace_seq *s, const struct trace_entry *ent, bool has_cg)
1381 {
1382     const struct blk_io_trace_remap *__r = pdu_start(ent, has_cg);
1383 
1384     trace_seq_printf(s, "%llu + %u <- (%d,%d) %llu\n",
1385              t_sector(ent), t_sec(ent),
1386              MAJOR(be32_to_cpu(__r->device_from)),
1387              MINOR(be32_to_cpu(__r->device_from)),
1388              be64_to_cpu(__r->sector_from));
1389 }
1390 
1391 static void blk_log_plug(struct trace_seq *s, const struct trace_entry *ent, bool has_cg)
1392 {
1393     char cmd[TASK_COMM_LEN];
1394 
1395     trace_find_cmdline(ent->pid, cmd);
1396 
1397     trace_seq_printf(s, "[%s]\n", cmd);
1398 }
1399 
1400 static void blk_log_unplug(struct trace_seq *s, const struct trace_entry *ent, bool has_cg)
1401 {
1402     char cmd[TASK_COMM_LEN];
1403 
1404     trace_find_cmdline(ent->pid, cmd);
1405 
1406     trace_seq_printf(s, "[%s] %llu\n", cmd, get_pdu_int(ent, has_cg));
1407 }
1408 
1409 static void blk_log_split(struct trace_seq *s, const struct trace_entry *ent, bool has_cg)
1410 {
1411     char cmd[TASK_COMM_LEN];
1412 
1413     trace_find_cmdline(ent->pid, cmd);
1414 
1415     trace_seq_printf(s, "%llu / %llu [%s]\n", t_sector(ent),
1416              get_pdu_int(ent, has_cg), cmd);
1417 }
1418 
1419 static void blk_log_msg(struct trace_seq *s, const struct trace_entry *ent,
1420             bool has_cg)
1421 {
1422 
1423     trace_seq_putmem(s, pdu_start(ent, has_cg),
1424         pdu_real_len(ent, has_cg));
1425     trace_seq_putc(s, '\n');
1426 }
1427 
1428 /*
1429  * struct tracer operations
1430  */
1431 
1432 static void blk_tracer_print_header(struct seq_file *m)
1433 {
1434     if (!(blk_tracer_flags.val & TRACE_BLK_OPT_CLASSIC))
1435         return;
1436     seq_puts(m, "# DEV   CPU TIMESTAMP     PID ACT FLG\n"
1437             "#  |     |     |           |   |   |\n");
1438 }
1439 
1440 static void blk_tracer_start(struct trace_array *tr)
1441 {
1442     blk_tracer_enabled = true;
1443 }
1444 
1445 static int blk_tracer_init(struct trace_array *tr)
1446 {
1447     blk_tr = tr;
1448     blk_tracer_start(tr);
1449     return 0;
1450 }
1451 
1452 static void blk_tracer_stop(struct trace_array *tr)
1453 {
1454     blk_tracer_enabled = false;
1455 }
1456 
1457 static void blk_tracer_reset(struct trace_array *tr)
1458 {
1459     blk_tracer_stop(tr);
1460 }
1461 
1462 static const struct {
1463     const char *act[2];
1464     void       (*print)(struct trace_seq *s, const struct trace_entry *ent,
1465                 bool has_cg);
1466 } what2act[] = {
1467     [__BLK_TA_QUEUE]    = {{  "Q", "queue" },      blk_log_generic },
1468     [__BLK_TA_BACKMERGE]    = {{  "M", "backmerge" },  blk_log_generic },
1469     [__BLK_TA_FRONTMERGE]   = {{  "F", "frontmerge" }, blk_log_generic },
1470     [__BLK_TA_GETRQ]    = {{  "G", "getrq" },      blk_log_generic },
1471     [__BLK_TA_SLEEPRQ]  = {{  "S", "sleeprq" },    blk_log_generic },
1472     [__BLK_TA_REQUEUE]  = {{  "R", "requeue" },    blk_log_with_error },
1473     [__BLK_TA_ISSUE]    = {{  "D", "issue" },      blk_log_generic },
1474     [__BLK_TA_COMPLETE] = {{  "C", "complete" },   blk_log_with_error },
1475     [__BLK_TA_PLUG]     = {{  "P", "plug" },       blk_log_plug },
1476     [__BLK_TA_UNPLUG_IO]    = {{  "U", "unplug_io" },  blk_log_unplug },
1477     [__BLK_TA_UNPLUG_TIMER] = {{ "UT", "unplug_timer" }, blk_log_unplug },
1478     [__BLK_TA_INSERT]   = {{  "I", "insert" },     blk_log_generic },
1479     [__BLK_TA_SPLIT]    = {{  "X", "split" },      blk_log_split },
1480     [__BLK_TA_BOUNCE]   = {{  "B", "bounce" },     blk_log_generic },
1481     [__BLK_TA_REMAP]    = {{  "A", "remap" },      blk_log_remap },
1482 };
1483 
1484 static enum print_line_t print_one_line(struct trace_iterator *iter,
1485                     bool classic)
1486 {
1487     struct trace_array *tr = iter->tr;
1488     struct trace_seq *s = &iter->seq;
1489     const struct blk_io_trace *t;
1490     u16 what;
1491     bool long_act;
1492     blk_log_action_t *log_action;
1493     bool has_cg;
1494 
1495     t      = te_blk_io_trace(iter->ent);
1496     what       = (t->action & ((1 << BLK_TC_SHIFT) - 1)) & ~__BLK_TA_CGROUP;
1497     long_act   = !!(tr->trace_flags & TRACE_ITER_VERBOSE);
1498     log_action = classic ? &blk_log_action_classic : &blk_log_action;
1499     has_cg     = t->action & __BLK_TA_CGROUP;
1500 
1501     if ((t->action & ~__BLK_TN_CGROUP) == BLK_TN_MESSAGE) {
1502         log_action(iter, long_act ? "message" : "m", has_cg);
1503         blk_log_msg(s, iter->ent, has_cg);
1504         return trace_handle_return(s);
1505     }
1506 
1507     if (unlikely(what == 0 || what >= ARRAY_SIZE(what2act)))
1508         trace_seq_printf(s, "Unknown action %x\n", what);
1509     else {
1510         log_action(iter, what2act[what].act[long_act], has_cg);
1511         what2act[what].print(s, iter->ent, has_cg);
1512     }
1513 
1514     return trace_handle_return(s);
1515 }
1516 
1517 static enum print_line_t blk_trace_event_print(struct trace_iterator *iter,
1518                            int flags, struct trace_event *event)
1519 {
1520     return print_one_line(iter, false);
1521 }
1522 
1523 static void blk_trace_synthesize_old_trace(struct trace_iterator *iter)
1524 {
1525     struct trace_seq *s = &iter->seq;
1526     struct blk_io_trace *t = (struct blk_io_trace *)iter->ent;
1527     const int offset = offsetof(struct blk_io_trace, sector);
1528     struct blk_io_trace old = {
1529         .magic    = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION,
1530         .time     = iter->ts,
1531     };
1532 
1533     trace_seq_putmem(s, &old, offset);
1534     trace_seq_putmem(s, &t->sector,
1535              sizeof(old) - offset + t->pdu_len);
1536 }
1537 
1538 static enum print_line_t
1539 blk_trace_event_print_binary(struct trace_iterator *iter, int flags,
1540                  struct trace_event *event)
1541 {
1542     blk_trace_synthesize_old_trace(iter);
1543 
1544     return trace_handle_return(&iter->seq);
1545 }
1546 
1547 static enum print_line_t blk_tracer_print_line(struct trace_iterator *iter)
1548 {
1549     if (!(blk_tracer_flags.val & TRACE_BLK_OPT_CLASSIC))
1550         return TRACE_TYPE_UNHANDLED;
1551 
1552     return print_one_line(iter, true);
1553 }
1554 
1555 static int
1556 blk_tracer_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
1557 {
1558     /* don't output context-info for blk_classic output */
1559     if (bit == TRACE_BLK_OPT_CLASSIC) {
1560         if (set)
1561             tr->trace_flags &= ~TRACE_ITER_CONTEXT_INFO;
1562         else
1563             tr->trace_flags |= TRACE_ITER_CONTEXT_INFO;
1564     }
1565     return 0;
1566 }
1567 
1568 static struct tracer blk_tracer __read_mostly = {
1569     .name       = "blk",
1570     .init       = blk_tracer_init,
1571     .reset      = blk_tracer_reset,
1572     .start      = blk_tracer_start,
1573     .stop       = blk_tracer_stop,
1574     .print_header   = blk_tracer_print_header,
1575     .print_line = blk_tracer_print_line,
1576     .flags      = &blk_tracer_flags,
1577     .set_flag   = blk_tracer_set_flag,
1578 };
1579 
1580 static struct trace_event_functions trace_blk_event_funcs = {
1581     .trace      = blk_trace_event_print,
1582     .binary     = blk_trace_event_print_binary,
1583 };
1584 
1585 static struct trace_event trace_blk_event = {
1586     .type       = TRACE_BLK,
1587     .funcs      = &trace_blk_event_funcs,
1588 };
1589 
1590 static int __init init_blk_tracer(void)
1591 {
1592     if (!register_trace_event(&trace_blk_event)) {
1593         pr_warn("Warning: could not register block events\n");
1594         return 1;
1595     }
1596 
1597     if (register_tracer(&blk_tracer) != 0) {
1598         pr_warn("Warning: could not register the block tracer\n");
1599         unregister_trace_event(&trace_blk_event);
1600         return 1;
1601     }
1602 
1603     return 0;
1604 }
1605 
1606 device_initcall(init_blk_tracer);
1607 
1608 static int blk_trace_remove_queue(struct request_queue *q)
1609 {
1610     struct blk_trace *bt;
1611 
1612     bt = rcu_replace_pointer(q->blk_trace, NULL,
1613                  lockdep_is_held(&q->debugfs_mutex));
1614     if (bt == NULL)
1615         return -EINVAL;
1616 
1617     if (bt->trace_state == Blktrace_running) {
1618         bt->trace_state = Blktrace_stopped;
1619         raw_spin_lock_irq(&running_trace_lock);
1620         list_del_init(&bt->running_list);
1621         raw_spin_unlock_irq(&running_trace_lock);
1622         relay_flush(bt->rchan);
1623     }
1624 
1625     put_probe_ref();
1626     synchronize_rcu();
1627     blk_trace_free(q, bt);
1628     return 0;
1629 }
1630 
1631 /*
1632  * Setup everything required to start tracing
1633  */
1634 static int blk_trace_setup_queue(struct request_queue *q,
1635                  struct block_device *bdev)
1636 {
1637     struct blk_trace *bt = NULL;
1638     int ret = -ENOMEM;
1639 
1640     bt = kzalloc(sizeof(*bt), GFP_KERNEL);
1641     if (!bt)
1642         return -ENOMEM;
1643 
1644     bt->msg_data = __alloc_percpu(BLK_TN_MAX_MSG, __alignof__(char));
1645     if (!bt->msg_data)
1646         goto free_bt;
1647 
1648     bt->dev = bdev->bd_dev;
1649     bt->act_mask = (u16)-1;
1650 
1651     blk_trace_setup_lba(bt, bdev);
1652 
1653     rcu_assign_pointer(q->blk_trace, bt);
1654     get_probe_ref();
1655     return 0;
1656 
1657 free_bt:
1658     blk_trace_free(q, bt);
1659     return ret;
1660 }
1661 
1662 /*
1663  * sysfs interface to enable and configure tracing
1664  */
1665 
1666 static ssize_t sysfs_blk_trace_attr_show(struct device *dev,
1667                      struct device_attribute *attr,
1668                      char *buf);
1669 static ssize_t sysfs_blk_trace_attr_store(struct device *dev,
1670                       struct device_attribute *attr,
1671                       const char *buf, size_t count);
1672 #define BLK_TRACE_DEVICE_ATTR(_name) \
1673     DEVICE_ATTR(_name, S_IRUGO | S_IWUSR, \
1674             sysfs_blk_trace_attr_show, \
1675             sysfs_blk_trace_attr_store)
1676 
1677 static BLK_TRACE_DEVICE_ATTR(enable);
1678 static BLK_TRACE_DEVICE_ATTR(act_mask);
1679 static BLK_TRACE_DEVICE_ATTR(pid);
1680 static BLK_TRACE_DEVICE_ATTR(start_lba);
1681 static BLK_TRACE_DEVICE_ATTR(end_lba);
1682 
1683 static struct attribute *blk_trace_attrs[] = {
1684     &dev_attr_enable.attr,
1685     &dev_attr_act_mask.attr,
1686     &dev_attr_pid.attr,
1687     &dev_attr_start_lba.attr,
1688     &dev_attr_end_lba.attr,
1689     NULL
1690 };
1691 
1692 struct attribute_group blk_trace_attr_group = {
1693     .name  = "trace",
1694     .attrs = blk_trace_attrs,
1695 };
1696 
1697 static const struct {
1698     int mask;
1699     const char *str;
1700 } mask_maps[] = {
1701     { BLK_TC_READ,      "read"      },
1702     { BLK_TC_WRITE,     "write"     },
1703     { BLK_TC_FLUSH,     "flush"     },
1704     { BLK_TC_SYNC,      "sync"      },
1705     { BLK_TC_QUEUE,     "queue"     },
1706     { BLK_TC_REQUEUE,   "requeue"   },
1707     { BLK_TC_ISSUE,     "issue"     },
1708     { BLK_TC_COMPLETE,  "complete"  },
1709     { BLK_TC_FS,        "fs"        },
1710     { BLK_TC_PC,        "pc"        },
1711     { BLK_TC_NOTIFY,    "notify"    },
1712     { BLK_TC_AHEAD,     "ahead"     },
1713     { BLK_TC_META,      "meta"      },
1714     { BLK_TC_DISCARD,   "discard"   },
1715     { BLK_TC_DRV_DATA,  "drv_data"  },
1716     { BLK_TC_FUA,       "fua"       },
1717 };
1718 
1719 static int blk_trace_str2mask(const char *str)
1720 {
1721     int i;
1722     int mask = 0;
1723     char *buf, *s, *token;
1724 
1725     buf = kstrdup(str, GFP_KERNEL);
1726     if (buf == NULL)
1727         return -ENOMEM;
1728     s = strstrip(buf);
1729 
1730     while (1) {
1731         token = strsep(&s, ",");
1732         if (token == NULL)
1733             break;
1734 
1735         if (*token == '\0')
1736             continue;
1737 
1738         for (i = 0; i < ARRAY_SIZE(mask_maps); i++) {
1739             if (strcasecmp(token, mask_maps[i].str) == 0) {
1740                 mask |= mask_maps[i].mask;
1741                 break;
1742             }
1743         }
1744         if (i == ARRAY_SIZE(mask_maps)) {
1745             mask = -EINVAL;
1746             break;
1747         }
1748     }
1749     kfree(buf);
1750 
1751     return mask;
1752 }
1753 
1754 static ssize_t blk_trace_mask2str(char *buf, int mask)
1755 {
1756     int i;
1757     char *p = buf;
1758 
1759     for (i = 0; i < ARRAY_SIZE(mask_maps); i++) {
1760         if (mask & mask_maps[i].mask) {
1761             p += sprintf(p, "%s%s",
1762                     (p == buf) ? "" : ",", mask_maps[i].str);
1763         }
1764     }
1765     *p++ = '\n';
1766 
1767     return p - buf;
1768 }
1769 
1770 static ssize_t sysfs_blk_trace_attr_show(struct device *dev,
1771                      struct device_attribute *attr,
1772                      char *buf)
1773 {
1774     struct block_device *bdev = dev_to_bdev(dev);
1775     struct request_queue *q = bdev_get_queue(bdev);
1776     struct blk_trace *bt;
1777     ssize_t ret = -ENXIO;
1778 
1779     mutex_lock(&q->debugfs_mutex);
1780 
1781     bt = rcu_dereference_protected(q->blk_trace,
1782                        lockdep_is_held(&q->debugfs_mutex));
1783     if (attr == &dev_attr_enable) {
1784         ret = sprintf(buf, "%u\n", !!bt);
1785         goto out_unlock_bdev;
1786     }
1787 
1788     if (bt == NULL)
1789         ret = sprintf(buf, "disabled\n");
1790     else if (attr == &dev_attr_act_mask)
1791         ret = blk_trace_mask2str(buf, bt->act_mask);
1792     else if (attr == &dev_attr_pid)
1793         ret = sprintf(buf, "%u\n", bt->pid);
1794     else if (attr == &dev_attr_start_lba)
1795         ret = sprintf(buf, "%llu\n", bt->start_lba);
1796     else if (attr == &dev_attr_end_lba)
1797         ret = sprintf(buf, "%llu\n", bt->end_lba);
1798 
1799 out_unlock_bdev:
1800     mutex_unlock(&q->debugfs_mutex);
1801     return ret;
1802 }
1803 
1804 static ssize_t sysfs_blk_trace_attr_store(struct device *dev,
1805                       struct device_attribute *attr,
1806                       const char *buf, size_t count)
1807 {
1808     struct block_device *bdev = dev_to_bdev(dev);
1809     struct request_queue *q = bdev_get_queue(bdev);
1810     struct blk_trace *bt;
1811     u64 value;
1812     ssize_t ret = -EINVAL;
1813 
1814     if (count == 0)
1815         goto out;
1816 
1817     if (attr == &dev_attr_act_mask) {
1818         if (kstrtoull(buf, 0, &value)) {
1819             /* Assume it is a list of trace category names */
1820             ret = blk_trace_str2mask(buf);
1821             if (ret < 0)
1822                 goto out;
1823             value = ret;
1824         }
1825     } else {
1826         if (kstrtoull(buf, 0, &value))
1827             goto out;
1828     }
1829 
1830     mutex_lock(&q->debugfs_mutex);
1831 
1832     bt = rcu_dereference_protected(q->blk_trace,
1833                        lockdep_is_held(&q->debugfs_mutex));
1834     if (attr == &dev_attr_enable) {
1835         if (!!value == !!bt) {
1836             ret = 0;
1837             goto out_unlock_bdev;
1838         }
1839         if (value)
1840             ret = blk_trace_setup_queue(q, bdev);
1841         else
1842             ret = blk_trace_remove_queue(q);
1843         goto out_unlock_bdev;
1844     }
1845 
1846     ret = 0;
1847     if (bt == NULL) {
1848         ret = blk_trace_setup_queue(q, bdev);
1849         bt = rcu_dereference_protected(q->blk_trace,
1850                 lockdep_is_held(&q->debugfs_mutex));
1851     }
1852 
1853     if (ret == 0) {
1854         if (attr == &dev_attr_act_mask)
1855             bt->act_mask = value;
1856         else if (attr == &dev_attr_pid)
1857             bt->pid = value;
1858         else if (attr == &dev_attr_start_lba)
1859             bt->start_lba = value;
1860         else if (attr == &dev_attr_end_lba)
1861             bt->end_lba = value;
1862     }
1863 
1864 out_unlock_bdev:
1865     mutex_unlock(&q->debugfs_mutex);
1866 out:
1867     return ret ? ret : count;
1868 }
1869 #endif /* CONFIG_BLK_DEV_IO_TRACE */
1870 
1871 #ifdef CONFIG_EVENT_TRACING
1872 
1873 /**
1874  * blk_fill_rwbs - Fill the buffer rwbs by mapping op to character string.
1875  * @rwbs:   buffer to be filled
1876  * @opf:    request operation type (REQ_OP_XXX) and flags for the tracepoint
1877  *
1878  * Description:
1879  *     Maps each request operation and flag to a single character and fills the
1880  *     buffer provided by the caller with resulting string.
1881  *
1882  **/
1883 void blk_fill_rwbs(char *rwbs, blk_opf_t opf)
1884 {
1885     int i = 0;
1886 
1887     if (opf & REQ_PREFLUSH)
1888         rwbs[i++] = 'F';
1889 
1890     switch (opf & REQ_OP_MASK) {
1891     case REQ_OP_WRITE:
1892         rwbs[i++] = 'W';
1893         break;
1894     case REQ_OP_DISCARD:
1895         rwbs[i++] = 'D';
1896         break;
1897     case REQ_OP_SECURE_ERASE:
1898         rwbs[i++] = 'D';
1899         rwbs[i++] = 'E';
1900         break;
1901     case REQ_OP_FLUSH:
1902         rwbs[i++] = 'F';
1903         break;
1904     case REQ_OP_READ:
1905         rwbs[i++] = 'R';
1906         break;
1907     default:
1908         rwbs[i++] = 'N';
1909     }
1910 
1911     if (opf & REQ_FUA)
1912         rwbs[i++] = 'F';
1913     if (opf & REQ_RAHEAD)
1914         rwbs[i++] = 'A';
1915     if (opf & REQ_SYNC)
1916         rwbs[i++] = 'S';
1917     if (opf & REQ_META)
1918         rwbs[i++] = 'M';
1919 
1920     rwbs[i] = '\0';
1921 }
1922 EXPORT_SYMBOL_GPL(blk_fill_rwbs);
1923 
1924 #endif /* CONFIG_EVENT_TRACING */
1925
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!