md/bcache/movinggc.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * Moving/copying garbage collector
0004  *
0005  * Copyright 2012 Google, Inc.
0006  */
0007
0008 #include "bcache.h"
0009 #include "btree.h"
0010 #include "debug.h"
0011 #include "request.h"
0012
0013 #include <trace/events/bcache.h>
0014
0015 struct moving_io {
0016     struct closure      cl;
0017     struct keybuf_key   *w;
0018     struct data_insert_op   op;
0019     struct bbio     bio;
0020 };
0021
0022 static bool moving_pred(struct keybuf *buf, struct bkey *k)
0023 {
0024     struct cache_set *c = container_of(buf, struct cache_set,
0025                        moving_gc_keys);
0026     unsigned int i;
0027
0028     for (i = 0; i < KEY_PTRS(k); i++)
0029         if (ptr_available(c, k, i) &&
0030             GC_MOVE(PTR_BUCKET(c, k, i)))
0031             return true;
0032
0033     return false;
0034 }
0035
0036 /* Moving GC - IO loop */
0037
0038 static void moving_io_destructor(struct closure *cl)
0039 {
0040     struct moving_io *io = container_of(cl, struct moving_io, cl);
0041
0042     kfree(io);
0043 }
0044
0045 static void write_moving_finish(struct closure *cl)
0046 {
0047     struct moving_io *io = container_of(cl, struct moving_io, cl);
0048     struct bio *bio = &io->bio.bio;
0049
0050     bio_free_pages(bio);
0051
0052     if (io->op.replace_collision)
0053         trace_bcache_gc_copy_collision(&io->w->key);
0054
0055     bch_keybuf_del(&io->op.c->moving_gc_keys, io->w);
0056
0057     up(&io->op.c->moving_in_flight);
0058
0059     closure_return_with_destructor(cl, moving_io_destructor);
0060 }
0061
0062 static void read_moving_endio(struct bio *bio)
0063 {
0064     struct bbio *b = container_of(bio, struct bbio, bio);
0065     struct moving_io *io = container_of(bio->bi_private,
0066                         struct moving_io, cl);
0067
0068     if (bio->bi_status)
0069         io->op.status = bio->bi_status;
0070     else if (!KEY_DIRTY(&b->key) &&
0071          ptr_stale(io->op.c, &b->key, 0)) {
0072         io->op.status = BLK_STS_IOERR;
0073     }
0074
0075     bch_bbio_endio(io->op.c, bio, bio->bi_status, "reading data to move");
0076 }
0077
0078 static void moving_init(struct moving_io *io)
0079 {
0080     struct bio *bio = &io->bio.bio;
0081
0082     bio_init(bio, NULL, bio->bi_inline_vecs,
0083          DIV_ROUND_UP(KEY_SIZE(&io->w->key), PAGE_SECTORS), 0);
0084     bio_get(bio);
0085     bio_set_prio(bio, IOPRIO_PRIO_VALUE(IOPRIO_CLASS_IDLE, 0));
0086
0087     bio->bi_iter.bi_size    = KEY_SIZE(&io->w->key) << 9;
0088     bio->bi_private     = &io->cl;
0089     bch_bio_map(bio, NULL);
0090 }
0091
0092 static void write_moving(struct closure *cl)
0093 {
0094     struct moving_io *io = container_of(cl, struct moving_io, cl);
0095     struct data_insert_op *op = &io->op;
0096
0097     if (!op->status) {
0098         moving_init(io);
0099
0100         io->bio.bio.bi_iter.bi_sector = KEY_START(&io->w->key);
0101         op->write_prio      = 1;
0102         op->bio         = &io->bio.bio;
0103
0104         op->writeback       = KEY_DIRTY(&io->w->key);
0105         op->csum        = KEY_CSUM(&io->w->key);
0106
0107         bkey_copy(&op->replace_key, &io->w->key);
0108         op->replace     = true;
0109
0110         closure_call(&op->cl, bch_data_insert, NULL, cl);
0111     }
0112
0113     continue_at(cl, write_moving_finish, op->wq);
0114 }
0115
0116 static void read_moving_submit(struct closure *cl)
0117 {
0118     struct moving_io *io = container_of(cl, struct moving_io, cl);
0119     struct bio *bio = &io->bio.bio;
0120
0121     bch_submit_bbio(bio, io->op.c, &io->w->key, 0);
0122
0123     continue_at(cl, write_moving, io->op.wq);
0124 }
0125
0126 static void read_moving(struct cache_set *c)
0127 {
0128     struct keybuf_key *w;
0129     struct moving_io *io;
0130     struct bio *bio;
0131     struct closure cl;
0132
0133     closure_init_stack(&cl);
0134
0135     /* XXX: if we error, background writeback could stall indefinitely */
0136
0137     while (!test_bit(CACHE_SET_STOPPING, &c->flags)) {
0138         w = bch_keybuf_next_rescan(c, &c->moving_gc_keys,
0139                        &MAX_KEY, moving_pred);
0140         if (!w)
0141             break;
0142
0143         if (ptr_stale(c, &w->key, 0)) {
0144             bch_keybuf_del(&c->moving_gc_keys, w);
0145             continue;
0146         }
0147
0148         io = kzalloc(struct_size(io, bio.bio.bi_inline_vecs,
0149                      DIV_ROUND_UP(KEY_SIZE(&w->key), PAGE_SECTORS)),
0150                  GFP_KERNEL);
0151         if (!io)
0152             goto err;
0153
0154         w->private  = io;
0155         io->w       = w;
0156         io->op.inode    = KEY_INODE(&w->key);
0157         io->op.c    = c;
0158         io->op.wq   = c->moving_gc_wq;
0159
0160         moving_init(io);
0161         bio = &io->bio.bio;
0162
0163         bio_set_op_attrs(bio, REQ_OP_READ, 0);
0164         bio->bi_end_io  = read_moving_endio;
0165
0166         if (bch_bio_alloc_pages(bio, GFP_KERNEL))
0167             goto err;
0168
0169         trace_bcache_gc_copy(&w->key);
0170
0171         down(&c->moving_in_flight);
0172         closure_call(&io->cl, read_moving_submit, NULL, &cl);
0173     }
0174
0175     if (0) {
0176 err:        if (!IS_ERR_OR_NULL(w->private))
0177             kfree(w->private);
0178
0179         bch_keybuf_del(&c->moving_gc_keys, w);
0180     }
0181
0182     closure_sync(&cl);
0183 }
0184
0185 static bool bucket_cmp(struct bucket *l, struct bucket *r)
0186 {
0187     return GC_SECTORS_USED(l) < GC_SECTORS_USED(r);
0188 }
0189
0190 static unsigned int bucket_heap_top(struct cache *ca)
0191 {
0192     struct bucket *b;
0193
0194     return (b = heap_peek(&ca->heap)) ? GC_SECTORS_USED(b) : 0;
0195 }
0196
0197 void bch_moving_gc(struct cache_set *c)
0198 {
0199     struct cache *ca = c->cache;
0200     struct bucket *b;
0201     unsigned long sectors_to_move, reserve_sectors;
0202
0203     if (!c->copy_gc_enabled)
0204         return;
0205
0206     mutex_lock(&c->bucket_lock);
0207
0208     sectors_to_move = 0;
0209     reserve_sectors = ca->sb.bucket_size *
0210                  fifo_used(&ca->free[RESERVE_MOVINGGC]);
0211
0212     ca->heap.used = 0;
0213
0214     for_each_bucket(b, ca) {
0215         if (GC_MARK(b) == GC_MARK_METADATA ||
0216             !GC_SECTORS_USED(b) ||
0217             GC_SECTORS_USED(b) == ca->sb.bucket_size ||
0218             atomic_read(&b->pin))
0219             continue;
0220
0221         if (!heap_full(&ca->heap)) {
0222             sectors_to_move += GC_SECTORS_USED(b);
0223             heap_add(&ca->heap, b, bucket_cmp);
0224         } else if (bucket_cmp(b, heap_peek(&ca->heap))) {
0225             sectors_to_move -= bucket_heap_top(ca);
0226             sectors_to_move += GC_SECTORS_USED(b);
0227
0228             ca->heap.data[0] = b;
0229             heap_sift(&ca->heap, 0, bucket_cmp);
0230         }
0231     }
0232
0233     while (sectors_to_move > reserve_sectors) {
0234         heap_pop(&ca->heap, b, bucket_cmp);
0235         sectors_to_move -= GC_SECTORS_USED(b);
0236     }
0237
0238     while (heap_pop(&ca->heap, b, bucket_cmp))
0239         SET_GC_MOVE(b, 1);
0240
0241     mutex_unlock(&c->bucket_lock);
0242
0243     c->moving_gc_keys.last_scanned = ZERO_KEY;
0244
0245     read_moving(c);
0246 }
0247
0248 void bch_moving_init_cache_set(struct cache_set *c)
0249 {
0250     bch_keybuf_init(&c->moving_gc_keys);
0251     sema_init(&c->moving_in_flight, 64);
0252 }