Back to home page

LXR

 
 

    


0001 #include <linux/kernel.h>
0002 #include <linux/module.h>
0003 #include <linux/backing-dev.h>
0004 #include <linux/bio.h>
0005 #include <linux/blkdev.h>
0006 #include <linux/mm.h>
0007 #include <linux/init.h>
0008 #include <linux/slab.h>
0009 #include <linux/workqueue.h>
0010 #include <linux/smp.h>
0011 
0012 #include <linux/blk-mq.h>
0013 #include "blk-mq.h"
0014 #include "blk-mq-tag.h"
0015 
0016 static void blk_mq_sysfs_release(struct kobject *kobj)
0017 {
0018 }
0019 
0020 struct blk_mq_ctx_sysfs_entry {
0021     struct attribute attr;
0022     ssize_t (*show)(struct blk_mq_ctx *, char *);
0023     ssize_t (*store)(struct blk_mq_ctx *, const char *, size_t);
0024 };
0025 
0026 struct blk_mq_hw_ctx_sysfs_entry {
0027     struct attribute attr;
0028     ssize_t (*show)(struct blk_mq_hw_ctx *, char *);
0029     ssize_t (*store)(struct blk_mq_hw_ctx *, const char *, size_t);
0030 };
0031 
0032 static ssize_t blk_mq_sysfs_show(struct kobject *kobj, struct attribute *attr,
0033                  char *page)
0034 {
0035     struct blk_mq_ctx_sysfs_entry *entry;
0036     struct blk_mq_ctx *ctx;
0037     struct request_queue *q;
0038     ssize_t res;
0039 
0040     entry = container_of(attr, struct blk_mq_ctx_sysfs_entry, attr);
0041     ctx = container_of(kobj, struct blk_mq_ctx, kobj);
0042     q = ctx->queue;
0043 
0044     if (!entry->show)
0045         return -EIO;
0046 
0047     res = -ENOENT;
0048     mutex_lock(&q->sysfs_lock);
0049     if (!blk_queue_dying(q))
0050         res = entry->show(ctx, page);
0051     mutex_unlock(&q->sysfs_lock);
0052     return res;
0053 }
0054 
0055 static ssize_t blk_mq_sysfs_store(struct kobject *kobj, struct attribute *attr,
0056                   const char *page, size_t length)
0057 {
0058     struct blk_mq_ctx_sysfs_entry *entry;
0059     struct blk_mq_ctx *ctx;
0060     struct request_queue *q;
0061     ssize_t res;
0062 
0063     entry = container_of(attr, struct blk_mq_ctx_sysfs_entry, attr);
0064     ctx = container_of(kobj, struct blk_mq_ctx, kobj);
0065     q = ctx->queue;
0066 
0067     if (!entry->store)
0068         return -EIO;
0069 
0070     res = -ENOENT;
0071     mutex_lock(&q->sysfs_lock);
0072     if (!blk_queue_dying(q))
0073         res = entry->store(ctx, page, length);
0074     mutex_unlock(&q->sysfs_lock);
0075     return res;
0076 }
0077 
0078 static ssize_t blk_mq_hw_sysfs_show(struct kobject *kobj,
0079                     struct attribute *attr, char *page)
0080 {
0081     struct blk_mq_hw_ctx_sysfs_entry *entry;
0082     struct blk_mq_hw_ctx *hctx;
0083     struct request_queue *q;
0084     ssize_t res;
0085 
0086     entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr);
0087     hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj);
0088     q = hctx->queue;
0089 
0090     if (!entry->show)
0091         return -EIO;
0092 
0093     res = -ENOENT;
0094     mutex_lock(&q->sysfs_lock);
0095     if (!blk_queue_dying(q))
0096         res = entry->show(hctx, page);
0097     mutex_unlock(&q->sysfs_lock);
0098     return res;
0099 }
0100 
0101 static ssize_t blk_mq_hw_sysfs_store(struct kobject *kobj,
0102                      struct attribute *attr, const char *page,
0103                      size_t length)
0104 {
0105     struct blk_mq_hw_ctx_sysfs_entry *entry;
0106     struct blk_mq_hw_ctx *hctx;
0107     struct request_queue *q;
0108     ssize_t res;
0109 
0110     entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr);
0111     hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj);
0112     q = hctx->queue;
0113 
0114     if (!entry->store)
0115         return -EIO;
0116 
0117     res = -ENOENT;
0118     mutex_lock(&q->sysfs_lock);
0119     if (!blk_queue_dying(q))
0120         res = entry->store(hctx, page, length);
0121     mutex_unlock(&q->sysfs_lock);
0122     return res;
0123 }
0124 
0125 static ssize_t blk_mq_sysfs_dispatched_show(struct blk_mq_ctx *ctx, char *page)
0126 {
0127     return sprintf(page, "%lu %lu\n", ctx->rq_dispatched[1],
0128                 ctx->rq_dispatched[0]);
0129 }
0130 
0131 static ssize_t blk_mq_sysfs_merged_show(struct blk_mq_ctx *ctx, char *page)
0132 {
0133     return sprintf(page, "%lu\n", ctx->rq_merged);
0134 }
0135 
0136 static ssize_t blk_mq_sysfs_completed_show(struct blk_mq_ctx *ctx, char *page)
0137 {
0138     return sprintf(page, "%lu %lu\n", ctx->rq_completed[1],
0139                 ctx->rq_completed[0]);
0140 }
0141 
0142 static ssize_t sysfs_list_show(char *page, struct list_head *list, char *msg)
0143 {
0144     struct request *rq;
0145     int len = snprintf(page, PAGE_SIZE - 1, "%s:\n", msg);
0146 
0147     list_for_each_entry(rq, list, queuelist) {
0148         const int rq_len = 2 * sizeof(rq) + 2;
0149 
0150         /* if the output will be truncated */
0151         if (PAGE_SIZE - 1 < len + rq_len) {
0152             /* backspacing if it can't hold '\t...\n' */
0153             if (PAGE_SIZE - 1 < len + 5)
0154                 len -= rq_len;
0155             len += snprintf(page + len, PAGE_SIZE - 1 - len,
0156                     "\t...\n");
0157             break;
0158         }
0159         len += snprintf(page + len, PAGE_SIZE - 1 - len,
0160                 "\t%p\n", rq);
0161     }
0162 
0163     return len;
0164 }
0165 
0166 static ssize_t blk_mq_sysfs_rq_list_show(struct blk_mq_ctx *ctx, char *page)
0167 {
0168     ssize_t ret;
0169 
0170     spin_lock(&ctx->lock);
0171     ret = sysfs_list_show(page, &ctx->rq_list, "CTX pending");
0172     spin_unlock(&ctx->lock);
0173 
0174     return ret;
0175 }
0176 
0177 static ssize_t blk_mq_hw_sysfs_poll_show(struct blk_mq_hw_ctx *hctx, char *page)
0178 {
0179     return sprintf(page, "considered=%lu, invoked=%lu, success=%lu\n",
0180                hctx->poll_considered, hctx->poll_invoked,
0181                hctx->poll_success);
0182 }
0183 
0184 static ssize_t blk_mq_hw_sysfs_poll_store(struct blk_mq_hw_ctx *hctx,
0185                       const char *page, size_t size)
0186 {
0187     hctx->poll_considered = hctx->poll_invoked = hctx->poll_success = 0;
0188 
0189     return size;
0190 }
0191 
0192 static ssize_t blk_mq_hw_sysfs_queued_show(struct blk_mq_hw_ctx *hctx,
0193                        char *page)
0194 {
0195     return sprintf(page, "%lu\n", hctx->queued);
0196 }
0197 
0198 static ssize_t blk_mq_hw_sysfs_run_show(struct blk_mq_hw_ctx *hctx, char *page)
0199 {
0200     return sprintf(page, "%lu\n", hctx->run);
0201 }
0202 
0203 static ssize_t blk_mq_hw_sysfs_dispatched_show(struct blk_mq_hw_ctx *hctx,
0204                            char *page)
0205 {
0206     char *start_page = page;
0207     int i;
0208 
0209     page += sprintf(page, "%8u\t%lu\n", 0U, hctx->dispatched[0]);
0210 
0211     for (i = 1; i < BLK_MQ_MAX_DISPATCH_ORDER - 1; i++) {
0212         unsigned int d = 1U << (i - 1);
0213 
0214         page += sprintf(page, "%8u\t%lu\n", d, hctx->dispatched[i]);
0215     }
0216 
0217     page += sprintf(page, "%8u+\t%lu\n", 1U << (i - 1),
0218                         hctx->dispatched[i]);
0219     return page - start_page;
0220 }
0221 
0222 static ssize_t blk_mq_hw_sysfs_rq_list_show(struct blk_mq_hw_ctx *hctx,
0223                         char *page)
0224 {
0225     ssize_t ret;
0226 
0227     spin_lock(&hctx->lock);
0228     ret = sysfs_list_show(page, &hctx->dispatch, "HCTX pending");
0229     spin_unlock(&hctx->lock);
0230 
0231     return ret;
0232 }
0233 
0234 static ssize_t blk_mq_hw_sysfs_tags_show(struct blk_mq_hw_ctx *hctx, char *page)
0235 {
0236     return blk_mq_tag_sysfs_show(hctx->tags, page);
0237 }
0238 
0239 static ssize_t blk_mq_hw_sysfs_active_show(struct blk_mq_hw_ctx *hctx, char *page)
0240 {
0241     return sprintf(page, "%u\n", atomic_read(&hctx->nr_active));
0242 }
0243 
0244 static ssize_t blk_mq_hw_sysfs_cpus_show(struct blk_mq_hw_ctx *hctx, char *page)
0245 {
0246     unsigned int i, first = 1;
0247     ssize_t ret = 0;
0248 
0249     for_each_cpu(i, hctx->cpumask) {
0250         if (first)
0251             ret += sprintf(ret + page, "%u", i);
0252         else
0253             ret += sprintf(ret + page, ", %u", i);
0254 
0255         first = 0;
0256     }
0257 
0258     ret += sprintf(ret + page, "\n");
0259     return ret;
0260 }
0261 
0262 static void blk_mq_stat_clear(struct blk_mq_hw_ctx *hctx)
0263 {
0264     struct blk_mq_ctx *ctx;
0265     unsigned int i;
0266 
0267     hctx_for_each_ctx(hctx, ctx, i) {
0268         blk_stat_init(&ctx->stat[BLK_STAT_READ]);
0269         blk_stat_init(&ctx->stat[BLK_STAT_WRITE]);
0270     }
0271 }
0272 
0273 static ssize_t blk_mq_hw_sysfs_stat_store(struct blk_mq_hw_ctx *hctx,
0274                       const char *page, size_t count)
0275 {
0276     blk_mq_stat_clear(hctx);
0277     return count;
0278 }
0279 
0280 static ssize_t print_stat(char *page, struct blk_rq_stat *stat, const char *pre)
0281 {
0282     return sprintf(page, "%s samples=%llu, mean=%lld, min=%lld, max=%lld\n",
0283             pre, (long long) stat->nr_samples,
0284             (long long) stat->mean, (long long) stat->min,
0285             (long long) stat->max);
0286 }
0287 
0288 static ssize_t blk_mq_hw_sysfs_stat_show(struct blk_mq_hw_ctx *hctx, char *page)
0289 {
0290     struct blk_rq_stat stat[2];
0291     ssize_t ret;
0292 
0293     blk_stat_init(&stat[BLK_STAT_READ]);
0294     blk_stat_init(&stat[BLK_STAT_WRITE]);
0295 
0296     blk_hctx_stat_get(hctx, stat);
0297 
0298     ret = print_stat(page, &stat[BLK_STAT_READ], "read :");
0299     ret += print_stat(page + ret, &stat[BLK_STAT_WRITE], "write:");
0300     return ret;
0301 }
0302 
0303 static struct blk_mq_ctx_sysfs_entry blk_mq_sysfs_dispatched = {
0304     .attr = {.name = "dispatched", .mode = S_IRUGO },
0305     .show = blk_mq_sysfs_dispatched_show,
0306 };
0307 static struct blk_mq_ctx_sysfs_entry blk_mq_sysfs_merged = {
0308     .attr = {.name = "merged", .mode = S_IRUGO },
0309     .show = blk_mq_sysfs_merged_show,
0310 };
0311 static struct blk_mq_ctx_sysfs_entry blk_mq_sysfs_completed = {
0312     .attr = {.name = "completed", .mode = S_IRUGO },
0313     .show = blk_mq_sysfs_completed_show,
0314 };
0315 static struct blk_mq_ctx_sysfs_entry blk_mq_sysfs_rq_list = {
0316     .attr = {.name = "rq_list", .mode = S_IRUGO },
0317     .show = blk_mq_sysfs_rq_list_show,
0318 };
0319 
0320 static struct attribute *default_ctx_attrs[] = {
0321     &blk_mq_sysfs_dispatched.attr,
0322     &blk_mq_sysfs_merged.attr,
0323     &blk_mq_sysfs_completed.attr,
0324     &blk_mq_sysfs_rq_list.attr,
0325     NULL,
0326 };
0327 
0328 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_queued = {
0329     .attr = {.name = "queued", .mode = S_IRUGO },
0330     .show = blk_mq_hw_sysfs_queued_show,
0331 };
0332 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_run = {
0333     .attr = {.name = "run", .mode = S_IRUGO },
0334     .show = blk_mq_hw_sysfs_run_show,
0335 };
0336 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_dispatched = {
0337     .attr = {.name = "dispatched", .mode = S_IRUGO },
0338     .show = blk_mq_hw_sysfs_dispatched_show,
0339 };
0340 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_active = {
0341     .attr = {.name = "active", .mode = S_IRUGO },
0342     .show = blk_mq_hw_sysfs_active_show,
0343 };
0344 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_pending = {
0345     .attr = {.name = "pending", .mode = S_IRUGO },
0346     .show = blk_mq_hw_sysfs_rq_list_show,
0347 };
0348 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_tags = {
0349     .attr = {.name = "tags", .mode = S_IRUGO },
0350     .show = blk_mq_hw_sysfs_tags_show,
0351 };
0352 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_cpus = {
0353     .attr = {.name = "cpu_list", .mode = S_IRUGO },
0354     .show = blk_mq_hw_sysfs_cpus_show,
0355 };
0356 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_poll = {
0357     .attr = {.name = "io_poll", .mode = S_IWUSR | S_IRUGO },
0358     .show = blk_mq_hw_sysfs_poll_show,
0359     .store = blk_mq_hw_sysfs_poll_store,
0360 };
0361 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_stat = {
0362     .attr = {.name = "stats", .mode = S_IRUGO | S_IWUSR },
0363     .show = blk_mq_hw_sysfs_stat_show,
0364     .store = blk_mq_hw_sysfs_stat_store,
0365 };
0366 
0367 static struct attribute *default_hw_ctx_attrs[] = {
0368     &blk_mq_hw_sysfs_queued.attr,
0369     &blk_mq_hw_sysfs_run.attr,
0370     &blk_mq_hw_sysfs_dispatched.attr,
0371     &blk_mq_hw_sysfs_pending.attr,
0372     &blk_mq_hw_sysfs_tags.attr,
0373     &blk_mq_hw_sysfs_cpus.attr,
0374     &blk_mq_hw_sysfs_active.attr,
0375     &blk_mq_hw_sysfs_poll.attr,
0376     &blk_mq_hw_sysfs_stat.attr,
0377     NULL,
0378 };
0379 
0380 static const struct sysfs_ops blk_mq_sysfs_ops = {
0381     .show   = blk_mq_sysfs_show,
0382     .store  = blk_mq_sysfs_store,
0383 };
0384 
0385 static const struct sysfs_ops blk_mq_hw_sysfs_ops = {
0386     .show   = blk_mq_hw_sysfs_show,
0387     .store  = blk_mq_hw_sysfs_store,
0388 };
0389 
0390 static struct kobj_type blk_mq_ktype = {
0391     .sysfs_ops  = &blk_mq_sysfs_ops,
0392     .release    = blk_mq_sysfs_release,
0393 };
0394 
0395 static struct kobj_type blk_mq_ctx_ktype = {
0396     .sysfs_ops  = &blk_mq_sysfs_ops,
0397     .default_attrs  = default_ctx_attrs,
0398     .release    = blk_mq_sysfs_release,
0399 };
0400 
0401 static struct kobj_type blk_mq_hw_ktype = {
0402     .sysfs_ops  = &blk_mq_hw_sysfs_ops,
0403     .default_attrs  = default_hw_ctx_attrs,
0404     .release    = blk_mq_sysfs_release,
0405 };
0406 
0407 static void blk_mq_unregister_hctx(struct blk_mq_hw_ctx *hctx)
0408 {
0409     struct blk_mq_ctx *ctx;
0410     int i;
0411 
0412     if (!hctx->nr_ctx)
0413         return;
0414 
0415     hctx_for_each_ctx(hctx, ctx, i)
0416         kobject_del(&ctx->kobj);
0417 
0418     kobject_del(&hctx->kobj);
0419 }
0420 
0421 static int blk_mq_register_hctx(struct blk_mq_hw_ctx *hctx)
0422 {
0423     struct request_queue *q = hctx->queue;
0424     struct blk_mq_ctx *ctx;
0425     int i, ret;
0426 
0427     if (!hctx->nr_ctx)
0428         return 0;
0429 
0430     ret = kobject_add(&hctx->kobj, &q->mq_kobj, "%u", hctx->queue_num);
0431     if (ret)
0432         return ret;
0433 
0434     hctx_for_each_ctx(hctx, ctx, i) {
0435         ret = kobject_add(&ctx->kobj, &hctx->kobj, "cpu%u", ctx->cpu);
0436         if (ret)
0437             break;
0438     }
0439 
0440     return ret;
0441 }
0442 
0443 static void __blk_mq_unregister_dev(struct device *dev, struct request_queue *q)
0444 {
0445     struct blk_mq_hw_ctx *hctx;
0446     struct blk_mq_ctx *ctx;
0447     int i, j;
0448 
0449     queue_for_each_hw_ctx(q, hctx, i) {
0450         blk_mq_unregister_hctx(hctx);
0451 
0452         hctx_for_each_ctx(hctx, ctx, j)
0453             kobject_put(&ctx->kobj);
0454 
0455         kobject_put(&hctx->kobj);
0456     }
0457 
0458     kobject_uevent(&q->mq_kobj, KOBJ_REMOVE);
0459     kobject_del(&q->mq_kobj);
0460     kobject_put(&q->mq_kobj);
0461 
0462     kobject_put(&dev->kobj);
0463 
0464     q->mq_sysfs_init_done = false;
0465 }
0466 
0467 void blk_mq_unregister_dev(struct device *dev, struct request_queue *q)
0468 {
0469     blk_mq_disable_hotplug();
0470     __blk_mq_unregister_dev(dev, q);
0471     blk_mq_enable_hotplug();
0472 }
0473 
0474 void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx)
0475 {
0476     kobject_init(&hctx->kobj, &blk_mq_hw_ktype);
0477 }
0478 
0479 static void blk_mq_sysfs_init(struct request_queue *q)
0480 {
0481     struct blk_mq_ctx *ctx;
0482     int cpu;
0483 
0484     kobject_init(&q->mq_kobj, &blk_mq_ktype);
0485 
0486     for_each_possible_cpu(cpu) {
0487         ctx = per_cpu_ptr(q->queue_ctx, cpu);
0488         kobject_init(&ctx->kobj, &blk_mq_ctx_ktype);
0489     }
0490 }
0491 
0492 int blk_mq_register_dev(struct device *dev, struct request_queue *q)
0493 {
0494     struct blk_mq_hw_ctx *hctx;
0495     int ret, i;
0496 
0497     blk_mq_disable_hotplug();
0498 
0499     blk_mq_sysfs_init(q);
0500 
0501     ret = kobject_add(&q->mq_kobj, kobject_get(&dev->kobj), "%s", "mq");
0502     if (ret < 0)
0503         goto out;
0504 
0505     kobject_uevent(&q->mq_kobj, KOBJ_ADD);
0506 
0507     queue_for_each_hw_ctx(q, hctx, i) {
0508         ret = blk_mq_register_hctx(hctx);
0509         if (ret)
0510             break;
0511     }
0512 
0513     if (ret)
0514         __blk_mq_unregister_dev(dev, q);
0515     else
0516         q->mq_sysfs_init_done = true;
0517 out:
0518     blk_mq_enable_hotplug();
0519 
0520     return ret;
0521 }
0522 EXPORT_SYMBOL_GPL(blk_mq_register_dev);
0523 
0524 void blk_mq_sysfs_unregister(struct request_queue *q)
0525 {
0526     struct blk_mq_hw_ctx *hctx;
0527     int i;
0528 
0529     if (!q->mq_sysfs_init_done)
0530         return;
0531 
0532     queue_for_each_hw_ctx(q, hctx, i)
0533         blk_mq_unregister_hctx(hctx);
0534 }
0535 
0536 int blk_mq_sysfs_register(struct request_queue *q)
0537 {
0538     struct blk_mq_hw_ctx *hctx;
0539     int i, ret = 0;
0540 
0541     if (!q->mq_sysfs_init_done)
0542         return ret;
0543 
0544     queue_for_each_hw_ctx(q, hctx, i) {
0545         ret = blk_mq_register_hctx(hctx);
0546         if (ret)
0547             break;
0548     }
0549 
0550     return ret;
0551 }