the-tree/block/blk-throttle.h

0001 #ifndef BLK_THROTTLE_H
0002 #define BLK_THROTTLE_H
0003
0004 #include "blk-cgroup-rwstat.h"
0005
0006 /*
0007  * To implement hierarchical throttling, throtl_grps form a tree and bios
0008  * are dispatched upwards level by level until they reach the top and get
0009  * issued.  When dispatching bios from the children and local group at each
0010  * level, if the bios are dispatched into a single bio_list, there's a risk
0011  * of a local or child group which can queue many bios at once filling up
0012  * the list starving others.
0013  *
0014  * To avoid such starvation, dispatched bios are queued separately
0015  * according to where they came from.  When they are again dispatched to
0016  * the parent, they're popped in round-robin order so that no single source
0017  * hogs the dispatch window.
0018  *
0019  * throtl_qnode is used to keep the queued bios separated by their sources.
0020  * Bios are queued to throtl_qnode which in turn is queued to
0021  * throtl_service_queue and then dispatched in round-robin order.
0022  *
0023  * It's also used to track the reference counts on blkg's.  A qnode always
0024  * belongs to a throtl_grp and gets queued on itself or the parent, so
0025  * incrementing the reference of the associated throtl_grp when a qnode is
0026  * queued and decrementing when dequeued is enough to keep the whole blkg
0027  * tree pinned while bios are in flight.
0028  */
0029 struct throtl_qnode {
0030     struct list_head    node;       /* service_queue->queued[] */
0031     struct bio_list     bios;       /* queued bios */
0032     struct throtl_grp   *tg;        /* tg this qnode belongs to */
0033 };
0034
0035 struct throtl_service_queue {
0036     struct throtl_service_queue *parent_sq; /* the parent service_queue */
0037
0038     /*
0039      * Bios queued directly to this service_queue or dispatched from
0040      * children throtl_grp's.
0041      */
0042     struct list_head    queued[2];  /* throtl_qnode [READ/WRITE] */
0043     unsigned int        nr_queued[2];   /* number of queued bios */
0044
0045     /*
0046      * RB tree of active children throtl_grp's, which are sorted by
0047      * their ->disptime.
0048      */
0049     struct rb_root_cached   pending_tree;   /* RB tree of active tgs */
0050     unsigned int        nr_pending; /* # queued in the tree */
0051     unsigned long       first_pending_disptime; /* disptime of the first tg */
0052     struct timer_list   pending_timer;  /* fires on first_pending_disptime */
0053 };
0054
0055 enum tg_state_flags {
0056     THROTL_TG_PENDING   = 1 << 0,   /* on parent's pending tree */
0057     THROTL_TG_WAS_EMPTY = 1 << 1,   /* bio_lists[] became non-empty */
0058     THROTL_TG_HAS_IOPS_LIMIT = 1 << 2,  /* tg has iops limit */
0059     THROTL_TG_CANCELING = 1 << 3,   /* starts to cancel bio */
0060 };
0061
0062 enum {
0063     LIMIT_LOW,
0064     LIMIT_MAX,
0065     LIMIT_CNT,
0066 };
0067
0068 struct throtl_grp {
0069     /* must be the first member */
0070     struct blkg_policy_data pd;
0071
0072     /* active throtl group service_queue member */
0073     struct rb_node rb_node;
0074
0075     /* throtl_data this group belongs to */
0076     struct throtl_data *td;
0077
0078     /* this group's service queue */
0079     struct throtl_service_queue service_queue;
0080
0081     /*
0082      * qnode_on_self is used when bios are directly queued to this
0083      * throtl_grp so that local bios compete fairly with bios
0084      * dispatched from children.  qnode_on_parent is used when bios are
0085      * dispatched from this throtl_grp into its parent and will compete
0086      * with the sibling qnode_on_parents and the parent's
0087      * qnode_on_self.
0088      */
0089     struct throtl_qnode qnode_on_self[2];
0090     struct throtl_qnode qnode_on_parent[2];
0091
0092     /*
0093      * Dispatch time in jiffies. This is the estimated time when group
0094      * will unthrottle and is ready to dispatch more bio. It is used as
0095      * key to sort active groups in service tree.
0096      */
0097     unsigned long disptime;
0098
0099     unsigned int flags;
0100
0101     /* are there any throtl rules between this group and td? */
0102     bool has_rules[2];
0103
0104     /* internally used bytes per second rate limits */
0105     uint64_t bps[2][LIMIT_CNT];
0106     /* user configured bps limits */
0107     uint64_t bps_conf[2][LIMIT_CNT];
0108
0109     /* internally used IOPS limits */
0110     unsigned int iops[2][LIMIT_CNT];
0111     /* user configured IOPS limits */
0112     unsigned int iops_conf[2][LIMIT_CNT];
0113
0114     /* Number of bytes dispatched in current slice */
0115     uint64_t bytes_disp[2];
0116     /* Number of bio's dispatched in current slice */
0117     unsigned int io_disp[2];
0118
0119     unsigned long last_low_overflow_time[2];
0120
0121     uint64_t last_bytes_disp[2];
0122     unsigned int last_io_disp[2];
0123
0124     unsigned long last_check_time;
0125
0126     unsigned long latency_target; /* us */
0127     unsigned long latency_target_conf; /* us */
0128     /* When did we start a new slice */
0129     unsigned long slice_start[2];
0130     unsigned long slice_end[2];
0131
0132     unsigned long last_finish_time; /* ns / 1024 */
0133     unsigned long checked_last_finish_time; /* ns / 1024 */
0134     unsigned long avg_idletime; /* ns / 1024 */
0135     unsigned long idletime_threshold; /* us */
0136     unsigned long idletime_threshold_conf; /* us */
0137
0138     unsigned int bio_cnt; /* total bios */
0139     unsigned int bad_bio_cnt; /* bios exceeding latency threshold */
0140     unsigned long bio_cnt_reset_time;
0141
0142     struct blkg_rwstat stat_bytes;
0143     struct blkg_rwstat stat_ios;
0144 };
0145
0146 extern struct blkcg_policy blkcg_policy_throtl;
0147
0148 static inline struct throtl_grp *pd_to_tg(struct blkg_policy_data *pd)
0149 {
0150     return pd ? container_of(pd, struct throtl_grp, pd) : NULL;
0151 }
0152
0153 static inline struct throtl_grp *blkg_to_tg(struct blkcg_gq *blkg)
0154 {
0155     return pd_to_tg(blkg_to_pd(blkg, &blkcg_policy_throtl));
0156 }
0157
0158 /*
0159  * Internal throttling interface
0160  */
0161 #ifndef CONFIG_BLK_DEV_THROTTLING
0162 static inline int blk_throtl_init(struct request_queue *q) { return 0; }
0163 static inline void blk_throtl_exit(struct request_queue *q) { }
0164 static inline void blk_throtl_register_queue(struct request_queue *q) { }
0165 static inline bool blk_throtl_bio(struct bio *bio) { return false; }
0166 static inline void blk_throtl_cancel_bios(struct request_queue *q) { }
0167 #else /* CONFIG_BLK_DEV_THROTTLING */
0168 int blk_throtl_init(struct request_queue *q);
0169 void blk_throtl_exit(struct request_queue *q);
0170 void blk_throtl_register_queue(struct request_queue *q);
0171 bool __blk_throtl_bio(struct bio *bio);
0172 void blk_throtl_cancel_bios(struct request_queue *q);
0173 static inline bool blk_throtl_bio(struct bio *bio)
0174 {
0175     struct throtl_grp *tg = blkg_to_tg(bio->bi_blkg);
0176
0177     /* no need to throttle bps any more if the bio has been throttled */
0178     if (bio_flagged(bio, BIO_THROTTLED) &&
0179         !(tg->flags & THROTL_TG_HAS_IOPS_LIMIT))
0180         return false;
0181
0182     if (!tg->has_rules[bio_data_dir(bio)])
0183         return false;
0184
0185     return __blk_throtl_bio(bio);
0186 }
0187 #endif /* CONFIG_BLK_DEV_THROTTLING */
0188
0189 #endif