Back to home page

LXR

 
 

    


0001 /*
0002  * Block stat tracking code
0003  *
0004  * Copyright (C) 2016 Jens Axboe
0005  */
0006 #include <linux/kernel.h>
0007 #include <linux/blk-mq.h>
0008 
0009 #include "blk-stat.h"
0010 #include "blk-mq.h"
0011 
0012 static void blk_stat_flush_batch(struct blk_rq_stat *stat)
0013 {
0014     const s32 nr_batch = READ_ONCE(stat->nr_batch);
0015     const s32 nr_samples = READ_ONCE(stat->nr_samples);
0016 
0017     if (!nr_batch)
0018         return;
0019     if (!nr_samples)
0020         stat->mean = div64_s64(stat->batch, nr_batch);
0021     else {
0022         stat->mean = div64_s64((stat->mean * nr_samples) +
0023                     stat->batch,
0024                     nr_batch + nr_samples);
0025     }
0026 
0027     stat->nr_samples += nr_batch;
0028     stat->nr_batch = stat->batch = 0;
0029 }
0030 
0031 static void blk_stat_sum(struct blk_rq_stat *dst, struct blk_rq_stat *src)
0032 {
0033     if (!src->nr_samples)
0034         return;
0035 
0036     blk_stat_flush_batch(src);
0037 
0038     dst->min = min(dst->min, src->min);
0039     dst->max = max(dst->max, src->max);
0040 
0041     if (!dst->nr_samples)
0042         dst->mean = src->mean;
0043     else {
0044         dst->mean = div64_s64((src->mean * src->nr_samples) +
0045                     (dst->mean * dst->nr_samples),
0046                     dst->nr_samples + src->nr_samples);
0047     }
0048     dst->nr_samples += src->nr_samples;
0049 }
0050 
0051 static void blk_mq_stat_get(struct request_queue *q, struct blk_rq_stat *dst)
0052 {
0053     struct blk_mq_hw_ctx *hctx;
0054     struct blk_mq_ctx *ctx;
0055     uint64_t latest = 0;
0056     int i, j, nr;
0057 
0058     blk_stat_init(&dst[BLK_STAT_READ]);
0059     blk_stat_init(&dst[BLK_STAT_WRITE]);
0060 
0061     nr = 0;
0062     do {
0063         uint64_t newest = 0;
0064 
0065         queue_for_each_hw_ctx(q, hctx, i) {
0066             hctx_for_each_ctx(hctx, ctx, j) {
0067                 blk_stat_flush_batch(&ctx->stat[BLK_STAT_READ]);
0068                 blk_stat_flush_batch(&ctx->stat[BLK_STAT_WRITE]);
0069 
0070                 if (!ctx->stat[BLK_STAT_READ].nr_samples &&
0071                     !ctx->stat[BLK_STAT_WRITE].nr_samples)
0072                     continue;
0073                 if (ctx->stat[BLK_STAT_READ].time > newest)
0074                     newest = ctx->stat[BLK_STAT_READ].time;
0075                 if (ctx->stat[BLK_STAT_WRITE].time > newest)
0076                     newest = ctx->stat[BLK_STAT_WRITE].time;
0077             }
0078         }
0079 
0080         /*
0081          * No samples
0082          */
0083         if (!newest)
0084             break;
0085 
0086         if (newest > latest)
0087             latest = newest;
0088 
0089         queue_for_each_hw_ctx(q, hctx, i) {
0090             hctx_for_each_ctx(hctx, ctx, j) {
0091                 if (ctx->stat[BLK_STAT_READ].time == newest) {
0092                     blk_stat_sum(&dst[BLK_STAT_READ],
0093                              &ctx->stat[BLK_STAT_READ]);
0094                     nr++;
0095                 }
0096                 if (ctx->stat[BLK_STAT_WRITE].time == newest) {
0097                     blk_stat_sum(&dst[BLK_STAT_WRITE],
0098                              &ctx->stat[BLK_STAT_WRITE]);
0099                     nr++;
0100                 }
0101             }
0102         }
0103         /*
0104          * If we race on finding an entry, just loop back again.
0105          * Should be very rare.
0106          */
0107     } while (!nr);
0108 
0109     dst[BLK_STAT_READ].time = dst[BLK_STAT_WRITE].time = latest;
0110 }
0111 
0112 void blk_queue_stat_get(struct request_queue *q, struct blk_rq_stat *dst)
0113 {
0114     if (q->mq_ops)
0115         blk_mq_stat_get(q, dst);
0116     else {
0117         blk_stat_flush_batch(&q->rq_stats[BLK_STAT_READ]);
0118         blk_stat_flush_batch(&q->rq_stats[BLK_STAT_WRITE]);
0119         memcpy(&dst[BLK_STAT_READ], &q->rq_stats[BLK_STAT_READ],
0120                 sizeof(struct blk_rq_stat));
0121         memcpy(&dst[BLK_STAT_WRITE], &q->rq_stats[BLK_STAT_WRITE],
0122                 sizeof(struct blk_rq_stat));
0123     }
0124 }
0125 
0126 void blk_hctx_stat_get(struct blk_mq_hw_ctx *hctx, struct blk_rq_stat *dst)
0127 {
0128     struct blk_mq_ctx *ctx;
0129     unsigned int i, nr;
0130 
0131     nr = 0;
0132     do {
0133         uint64_t newest = 0;
0134 
0135         hctx_for_each_ctx(hctx, ctx, i) {
0136             blk_stat_flush_batch(&ctx->stat[BLK_STAT_READ]);
0137             blk_stat_flush_batch(&ctx->stat[BLK_STAT_WRITE]);
0138 
0139             if (!ctx->stat[BLK_STAT_READ].nr_samples &&
0140                 !ctx->stat[BLK_STAT_WRITE].nr_samples)
0141                 continue;
0142 
0143             if (ctx->stat[BLK_STAT_READ].time > newest)
0144                 newest = ctx->stat[BLK_STAT_READ].time;
0145             if (ctx->stat[BLK_STAT_WRITE].time > newest)
0146                 newest = ctx->stat[BLK_STAT_WRITE].time;
0147         }
0148 
0149         if (!newest)
0150             break;
0151 
0152         hctx_for_each_ctx(hctx, ctx, i) {
0153             if (ctx->stat[BLK_STAT_READ].time == newest) {
0154                 blk_stat_sum(&dst[BLK_STAT_READ],
0155                         &ctx->stat[BLK_STAT_READ]);
0156                 nr++;
0157             }
0158             if (ctx->stat[BLK_STAT_WRITE].time == newest) {
0159                 blk_stat_sum(&dst[BLK_STAT_WRITE],
0160                         &ctx->stat[BLK_STAT_WRITE]);
0161                 nr++;
0162             }
0163         }
0164         /*
0165          * If we race on finding an entry, just loop back again.
0166          * Should be very rare, as the window is only updated
0167          * occasionally
0168          */
0169     } while (!nr);
0170 }
0171 
0172 static void __blk_stat_init(struct blk_rq_stat *stat, s64 time_now)
0173 {
0174     stat->min = -1ULL;
0175     stat->max = stat->nr_samples = stat->mean = 0;
0176     stat->batch = stat->nr_batch = 0;
0177     stat->time = time_now & BLK_STAT_NSEC_MASK;
0178 }
0179 
0180 void blk_stat_init(struct blk_rq_stat *stat)
0181 {
0182     __blk_stat_init(stat, ktime_to_ns(ktime_get()));
0183 }
0184 
0185 static bool __blk_stat_is_current(struct blk_rq_stat *stat, s64 now)
0186 {
0187     return (now & BLK_STAT_NSEC_MASK) == (stat->time & BLK_STAT_NSEC_MASK);
0188 }
0189 
0190 bool blk_stat_is_current(struct blk_rq_stat *stat)
0191 {
0192     return __blk_stat_is_current(stat, ktime_to_ns(ktime_get()));
0193 }
0194 
0195 void blk_stat_add(struct blk_rq_stat *stat, struct request *rq)
0196 {
0197     s64 now, value;
0198 
0199     now = __blk_stat_time(ktime_to_ns(ktime_get()));
0200     if (now < blk_stat_time(&rq->issue_stat))
0201         return;
0202 
0203     if (!__blk_stat_is_current(stat, now))
0204         __blk_stat_init(stat, now);
0205 
0206     value = now - blk_stat_time(&rq->issue_stat);
0207     if (value > stat->max)
0208         stat->max = value;
0209     if (value < stat->min)
0210         stat->min = value;
0211 
0212     if (stat->batch + value < stat->batch ||
0213         stat->nr_batch + 1 == BLK_RQ_STAT_BATCH)
0214         blk_stat_flush_batch(stat);
0215 
0216     stat->batch += value;
0217     stat->nr_batch++;
0218 }
0219 
0220 void blk_stat_clear(struct request_queue *q)
0221 {
0222     if (q->mq_ops) {
0223         struct blk_mq_hw_ctx *hctx;
0224         struct blk_mq_ctx *ctx;
0225         int i, j;
0226 
0227         queue_for_each_hw_ctx(q, hctx, i) {
0228             hctx_for_each_ctx(hctx, ctx, j) {
0229                 blk_stat_init(&ctx->stat[BLK_STAT_READ]);
0230                 blk_stat_init(&ctx->stat[BLK_STAT_WRITE]);
0231             }
0232         }
0233     } else {
0234         blk_stat_init(&q->rq_stats[BLK_STAT_READ]);
0235         blk_stat_init(&q->rq_stats[BLK_STAT_WRITE]);
0236     }
0237 }
0238 
0239 void blk_stat_set_issue_time(struct blk_issue_stat *stat)
0240 {
0241     stat->time = (stat->time & BLK_STAT_MASK) |
0242             (ktime_to_ns(ktime_get()) & BLK_STAT_TIME_MASK);
0243 }
0244 
0245 /*
0246  * Enable stat tracking, return whether it was enabled
0247  */
0248 bool blk_stat_enable(struct request_queue *q)
0249 {
0250     if (!test_bit(QUEUE_FLAG_STATS, &q->queue_flags)) {
0251         set_bit(QUEUE_FLAG_STATS, &q->queue_flags);
0252         return false;
0253     }
0254 
0255     return true;
0256 }