md/persistent-data/dm-transaction-manager.c

0001 /*
0002  * Copyright (C) 2011 Red Hat, Inc.
0003  *
0004  * This file is released under the GPL.
0005  */
0006 #include "dm-transaction-manager.h"
0007 #include "dm-space-map.h"
0008 #include "dm-space-map-disk.h"
0009 #include "dm-space-map-metadata.h"
0010 #include "dm-persistent-data-internal.h"
0011
0012 #include <linux/export.h>
0013 #include <linux/mutex.h>
0014 #include <linux/hash.h>
0015 #include <linux/slab.h>
0016 #include <linux/device-mapper.h>
0017
0018 #define DM_MSG_PREFIX "transaction manager"
0019
0020 /*----------------------------------------------------------------*/
0021
0022 #define PREFETCH_SIZE 128
0023 #define PREFETCH_BITS 7
0024 #define PREFETCH_SENTINEL ((dm_block_t) -1ULL)
0025
0026 struct prefetch_set {
0027     struct mutex lock;
0028     dm_block_t blocks[PREFETCH_SIZE];
0029 };
0030
0031 static unsigned prefetch_hash(dm_block_t b)
0032 {
0033     return hash_64(b, PREFETCH_BITS);
0034 }
0035
0036 static void prefetch_wipe(struct prefetch_set *p)
0037 {
0038     unsigned i;
0039     for (i = 0; i < PREFETCH_SIZE; i++)
0040         p->blocks[i] = PREFETCH_SENTINEL;
0041 }
0042
0043 static void prefetch_init(struct prefetch_set *p)
0044 {
0045     mutex_init(&p->lock);
0046     prefetch_wipe(p);
0047 }
0048
0049 static void prefetch_add(struct prefetch_set *p, dm_block_t b)
0050 {
0051     unsigned h = prefetch_hash(b);
0052
0053     mutex_lock(&p->lock);
0054     if (p->blocks[h] == PREFETCH_SENTINEL)
0055         p->blocks[h] = b;
0056
0057     mutex_unlock(&p->lock);
0058 }
0059
0060 static void prefetch_issue(struct prefetch_set *p, struct dm_block_manager *bm)
0061 {
0062     unsigned i;
0063
0064     mutex_lock(&p->lock);
0065
0066     for (i = 0; i < PREFETCH_SIZE; i++)
0067         if (p->blocks[i] != PREFETCH_SENTINEL) {
0068             dm_bm_prefetch(bm, p->blocks[i]);
0069             p->blocks[i] = PREFETCH_SENTINEL;
0070         }
0071
0072     mutex_unlock(&p->lock);
0073 }
0074
0075 /*----------------------------------------------------------------*/
0076
0077 struct shadow_info {
0078     struct hlist_node hlist;
0079     dm_block_t where;
0080 };
0081
0082 /*
0083  * It would be nice if we scaled with the size of transaction.
0084  */
0085 #define DM_HASH_SIZE 256
0086 #define DM_HASH_MASK (DM_HASH_SIZE - 1)
0087
0088 struct dm_transaction_manager {
0089     int is_clone;
0090     struct dm_transaction_manager *real;
0091
0092     struct dm_block_manager *bm;
0093     struct dm_space_map *sm;
0094
0095     spinlock_t lock;
0096     struct hlist_head buckets[DM_HASH_SIZE];
0097
0098     struct prefetch_set prefetches;
0099 };
0100
0101 /*----------------------------------------------------------------*/
0102
0103 static int is_shadow(struct dm_transaction_manager *tm, dm_block_t b)
0104 {
0105     int r = 0;
0106     unsigned bucket = dm_hash_block(b, DM_HASH_MASK);
0107     struct shadow_info *si;
0108
0109     spin_lock(&tm->lock);
0110     hlist_for_each_entry(si, tm->buckets + bucket, hlist)
0111         if (si->where == b) {
0112             r = 1;
0113             break;
0114         }
0115     spin_unlock(&tm->lock);
0116
0117     return r;
0118 }
0119
0120 /*
0121  * This can silently fail if there's no memory.  We're ok with this since
0122  * creating redundant shadows causes no harm.
0123  */
0124 static void insert_shadow(struct dm_transaction_manager *tm, dm_block_t b)
0125 {
0126     unsigned bucket;
0127     struct shadow_info *si;
0128
0129     si = kmalloc(sizeof(*si), GFP_NOIO);
0130     if (si) {
0131         si->where = b;
0132         bucket = dm_hash_block(b, DM_HASH_MASK);
0133         spin_lock(&tm->lock);
0134         hlist_add_head(&si->hlist, tm->buckets + bucket);
0135         spin_unlock(&tm->lock);
0136     }
0137 }
0138
0139 static void wipe_shadow_table(struct dm_transaction_manager *tm)
0140 {
0141     struct shadow_info *si;
0142     struct hlist_node *tmp;
0143     struct hlist_head *bucket;
0144     int i;
0145
0146     spin_lock(&tm->lock);
0147     for (i = 0; i < DM_HASH_SIZE; i++) {
0148         bucket = tm->buckets + i;
0149         hlist_for_each_entry_safe(si, tmp, bucket, hlist)
0150             kfree(si);
0151
0152         INIT_HLIST_HEAD(bucket);
0153     }
0154
0155     spin_unlock(&tm->lock);
0156 }
0157
0158 /*----------------------------------------------------------------*/
0159
0160 static struct dm_transaction_manager *dm_tm_create(struct dm_block_manager *bm,
0161                            struct dm_space_map *sm)
0162 {
0163     int i;
0164     struct dm_transaction_manager *tm;
0165
0166     tm = kmalloc(sizeof(*tm), GFP_KERNEL);
0167     if (!tm)
0168         return ERR_PTR(-ENOMEM);
0169
0170     tm->is_clone = 0;
0171     tm->real = NULL;
0172     tm->bm = bm;
0173     tm->sm = sm;
0174
0175     spin_lock_init(&tm->lock);
0176     for (i = 0; i < DM_HASH_SIZE; i++)
0177         INIT_HLIST_HEAD(tm->buckets + i);
0178
0179     prefetch_init(&tm->prefetches);
0180
0181     return tm;
0182 }
0183
0184 struct dm_transaction_manager *dm_tm_create_non_blocking_clone(struct dm_transaction_manager *real)
0185 {
0186     struct dm_transaction_manager *tm;
0187
0188     tm = kmalloc(sizeof(*tm), GFP_KERNEL);
0189     if (tm) {
0190         tm->is_clone = 1;
0191         tm->real = real;
0192     }
0193
0194     return tm;
0195 }
0196 EXPORT_SYMBOL_GPL(dm_tm_create_non_blocking_clone);
0197
0198 void dm_tm_destroy(struct dm_transaction_manager *tm)
0199 {
0200     if (!tm->is_clone)
0201         wipe_shadow_table(tm);
0202
0203     kfree(tm);
0204 }
0205 EXPORT_SYMBOL_GPL(dm_tm_destroy);
0206
0207 int dm_tm_pre_commit(struct dm_transaction_manager *tm)
0208 {
0209     int r;
0210
0211     if (tm->is_clone)
0212         return -EWOULDBLOCK;
0213
0214     r = dm_sm_commit(tm->sm);
0215     if (r < 0)
0216         return r;
0217
0218     return dm_bm_flush(tm->bm);
0219 }
0220 EXPORT_SYMBOL_GPL(dm_tm_pre_commit);
0221
0222 int dm_tm_commit(struct dm_transaction_manager *tm, struct dm_block *root)
0223 {
0224     if (tm->is_clone)
0225         return -EWOULDBLOCK;
0226
0227     wipe_shadow_table(tm);
0228     dm_bm_unlock(root);
0229
0230     return dm_bm_flush(tm->bm);
0231 }
0232 EXPORT_SYMBOL_GPL(dm_tm_commit);
0233
0234 int dm_tm_new_block(struct dm_transaction_manager *tm,
0235             struct dm_block_validator *v,
0236             struct dm_block **result)
0237 {
0238     int r;
0239     dm_block_t new_block;
0240
0241     if (tm->is_clone)
0242         return -EWOULDBLOCK;
0243
0244     r = dm_sm_new_block(tm->sm, &new_block);
0245     if (r < 0)
0246         return r;
0247
0248     r = dm_bm_write_lock_zero(tm->bm, new_block, v, result);
0249     if (r < 0) {
0250         dm_sm_dec_block(tm->sm, new_block);
0251         return r;
0252     }
0253
0254     /*
0255      * New blocks count as shadows in that they don't need to be
0256      * shadowed again.
0257      */
0258     insert_shadow(tm, new_block);
0259
0260     return 0;
0261 }
0262
0263 static int __shadow_block(struct dm_transaction_manager *tm, dm_block_t orig,
0264               struct dm_block_validator *v,
0265               struct dm_block **result)
0266 {
0267     int r;
0268     dm_block_t new;
0269     struct dm_block *orig_block;
0270
0271     r = dm_sm_new_block(tm->sm, &new);
0272     if (r < 0)
0273         return r;
0274
0275     r = dm_sm_dec_block(tm->sm, orig);
0276     if (r < 0)
0277         return r;
0278
0279     r = dm_bm_read_lock(tm->bm, orig, v, &orig_block);
0280     if (r < 0)
0281         return r;
0282
0283     /*
0284      * It would be tempting to use dm_bm_unlock_move here, but some
0285      * code, such as the space maps, keeps using the old data structures
0286      * secure in the knowledge they won't be changed until the next
0287      * transaction.  Using unlock_move would force a synchronous read
0288      * since the old block would no longer be in the cache.
0289      */
0290     r = dm_bm_write_lock_zero(tm->bm, new, v, result);
0291     if (r) {
0292         dm_bm_unlock(orig_block);
0293         return r;
0294     }
0295
0296     memcpy(dm_block_data(*result), dm_block_data(orig_block),
0297            dm_bm_block_size(tm->bm));
0298
0299     dm_bm_unlock(orig_block);
0300     return r;
0301 }
0302
0303 int dm_tm_shadow_block(struct dm_transaction_manager *tm, dm_block_t orig,
0304                struct dm_block_validator *v, struct dm_block **result,
0305                int *inc_children)
0306 {
0307     int r;
0308
0309     if (tm->is_clone)
0310         return -EWOULDBLOCK;
0311
0312     r = dm_sm_count_is_more_than_one(tm->sm, orig, inc_children);
0313     if (r < 0)
0314         return r;
0315
0316     if (is_shadow(tm, orig) && !*inc_children)
0317         return dm_bm_write_lock(tm->bm, orig, v, result);
0318
0319     r = __shadow_block(tm, orig, v, result);
0320     if (r < 0)
0321         return r;
0322     insert_shadow(tm, dm_block_location(*result));
0323
0324     return r;
0325 }
0326 EXPORT_SYMBOL_GPL(dm_tm_shadow_block);
0327
0328 int dm_tm_read_lock(struct dm_transaction_manager *tm, dm_block_t b,
0329             struct dm_block_validator *v,
0330             struct dm_block **blk)
0331 {
0332     if (tm->is_clone) {
0333         int r = dm_bm_read_try_lock(tm->real->bm, b, v, blk);
0334
0335         if (r == -EWOULDBLOCK)
0336             prefetch_add(&tm->real->prefetches, b);
0337
0338         return r;
0339     }
0340
0341     return dm_bm_read_lock(tm->bm, b, v, blk);
0342 }
0343 EXPORT_SYMBOL_GPL(dm_tm_read_lock);
0344
0345 void dm_tm_unlock(struct dm_transaction_manager *tm, struct dm_block *b)
0346 {
0347     dm_bm_unlock(b);
0348 }
0349 EXPORT_SYMBOL_GPL(dm_tm_unlock);
0350
0351 void dm_tm_inc(struct dm_transaction_manager *tm, dm_block_t b)
0352 {
0353     /*
0354      * The non-blocking clone doesn't support this.
0355      */
0356     BUG_ON(tm->is_clone);
0357
0358     dm_sm_inc_block(tm->sm, b);
0359 }
0360 EXPORT_SYMBOL_GPL(dm_tm_inc);
0361
0362 void dm_tm_inc_range(struct dm_transaction_manager *tm, dm_block_t b, dm_block_t e)
0363 {
0364     /*
0365      * The non-blocking clone doesn't support this.
0366      */
0367     BUG_ON(tm->is_clone);
0368
0369     dm_sm_inc_blocks(tm->sm, b, e);
0370 }
0371 EXPORT_SYMBOL_GPL(dm_tm_inc_range);
0372
0373 void dm_tm_dec(struct dm_transaction_manager *tm, dm_block_t b)
0374 {
0375     /*
0376      * The non-blocking clone doesn't support this.
0377      */
0378     BUG_ON(tm->is_clone);
0379
0380     dm_sm_dec_block(tm->sm, b);
0381 }
0382 EXPORT_SYMBOL_GPL(dm_tm_dec);
0383
0384 void dm_tm_dec_range(struct dm_transaction_manager *tm, dm_block_t b, dm_block_t e)
0385 {
0386     /*
0387      * The non-blocking clone doesn't support this.
0388      */
0389     BUG_ON(tm->is_clone);
0390
0391     dm_sm_dec_blocks(tm->sm, b, e);
0392 }
0393 EXPORT_SYMBOL_GPL(dm_tm_dec_range);
0394
0395 void dm_tm_with_runs(struct dm_transaction_manager *tm,
0396              const __le64 *value_le, unsigned count, dm_tm_run_fn fn)
0397 {
0398     uint64_t b, begin, end;
0399     bool in_run = false;
0400     unsigned i;
0401
0402     for (i = 0; i < count; i++, value_le++) {
0403         b = le64_to_cpu(*value_le);
0404
0405         if (in_run) {
0406             if (b == end)
0407                 end++;
0408             else {
0409                 fn(tm, begin, end);
0410                 begin = b;
0411                 end = b + 1;
0412             }
0413         } else {
0414             in_run = true;
0415             begin = b;
0416             end = b + 1;
0417         }
0418     }
0419
0420     if (in_run)
0421         fn(tm, begin, end);
0422 }
0423 EXPORT_SYMBOL_GPL(dm_tm_with_runs);
0424
0425 int dm_tm_ref(struct dm_transaction_manager *tm, dm_block_t b,
0426           uint32_t *result)
0427 {
0428     if (tm->is_clone)
0429         return -EWOULDBLOCK;
0430
0431     return dm_sm_get_count(tm->sm, b, result);
0432 }
0433
0434 int dm_tm_block_is_shared(struct dm_transaction_manager *tm, dm_block_t b,
0435               int *result)
0436 {
0437     if (tm->is_clone)
0438         return -EWOULDBLOCK;
0439
0440     return dm_sm_count_is_more_than_one(tm->sm, b, result);
0441 }
0442
0443 struct dm_block_manager *dm_tm_get_bm(struct dm_transaction_manager *tm)
0444 {
0445     return tm->bm;
0446 }
0447
0448 void dm_tm_issue_prefetches(struct dm_transaction_manager *tm)
0449 {
0450     prefetch_issue(&tm->prefetches, tm->bm);
0451 }
0452 EXPORT_SYMBOL_GPL(dm_tm_issue_prefetches);
0453
0454 /*----------------------------------------------------------------*/
0455
0456 static int dm_tm_create_internal(struct dm_block_manager *bm,
0457                  dm_block_t sb_location,
0458                  struct dm_transaction_manager **tm,
0459                  struct dm_space_map **sm,
0460                  int create,
0461                  void *sm_root, size_t sm_len)
0462 {
0463     int r;
0464
0465     *sm = dm_sm_metadata_init();
0466     if (IS_ERR(*sm))
0467         return PTR_ERR(*sm);
0468
0469     *tm = dm_tm_create(bm, *sm);
0470     if (IS_ERR(*tm)) {
0471         dm_sm_destroy(*sm);
0472         return PTR_ERR(*tm);
0473     }
0474
0475     if (create) {
0476         r = dm_sm_metadata_create(*sm, *tm, dm_bm_nr_blocks(bm),
0477                       sb_location);
0478         if (r) {
0479             DMERR("couldn't create metadata space map");
0480             goto bad;
0481         }
0482
0483     } else {
0484         r = dm_sm_metadata_open(*sm, *tm, sm_root, sm_len);
0485         if (r) {
0486             DMERR("couldn't open metadata space map");
0487             goto bad;
0488         }
0489     }
0490
0491     return 0;
0492
0493 bad:
0494     dm_tm_destroy(*tm);
0495     dm_sm_destroy(*sm);
0496     return r;
0497 }
0498
0499 int dm_tm_create_with_sm(struct dm_block_manager *bm, dm_block_t sb_location,
0500              struct dm_transaction_manager **tm,
0501              struct dm_space_map **sm)
0502 {
0503     return dm_tm_create_internal(bm, sb_location, tm, sm, 1, NULL, 0);
0504 }
0505 EXPORT_SYMBOL_GPL(dm_tm_create_with_sm);
0506
0507 int dm_tm_open_with_sm(struct dm_block_manager *bm, dm_block_t sb_location,
0508                void *sm_root, size_t root_len,
0509                struct dm_transaction_manager **tm,
0510                struct dm_space_map **sm)
0511 {
0512     return dm_tm_create_internal(bm, sb_location, tm, sm, 0, sm_root, root_len);
0513 }
0514 EXPORT_SYMBOL_GPL(dm_tm_open_with_sm);
0515
0516 /*----------------------------------------------------------------*/