OSCL-LXR linux-6.0.1/​drivers/​md/​dm-clone-target.c	Source navigation Diff markup Identifier search General search
	Version: linux-6.0.1 spark-3.0.0 hadoop-3.2.0-src hadoop-3.3.0-src spark-2.4.7 linux-4.15.13 hadoop-2.7.4-src Revert   Change

 // SPDX-License-Identifier: GPL-2.0-only
 /*
  * Copyright (C) 2019 Arrikto, Inc. All Rights Reserved.
  */
 
 #include <linux/mm.h>
 #include <linux/bio.h>
 #include <linux/err.h>
 #include <linux/hash.h>
 #include <linux/list.h>
 #include <linux/log2.h>
 #include <linux/init.h>
 #include <linux/slab.h>
 #include <linux/wait.h>
 #include <linux/dm-io.h>
 #include <linux/mutex.h>
 #include <linux/atomic.h>
 #include <linux/bitops.h>
 #include <linux/blkdev.h>
 #include <linux/kdev_t.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/jiffies.h>
 #include <linux/mempool.h>
 #include <linux/spinlock.h>
 #include <linux/blk_types.h>
 #include <linux/dm-kcopyd.h>
 #include <linux/workqueue.h>
 #include <linux/backing-dev.h>
 #include <linux/device-mapper.h>
 
 #include "dm.h"
 #include "dm-clone-metadata.h"
 
 #define DM_MSG_PREFIX "clone"
 
 /*
  * Minimum and maximum allowed region sizes
  */
 #define MIN_REGION_SIZE (1 << 3)  /* 4KB */
 #define MAX_REGION_SIZE (1 << 21) /* 1GB */
 
 #define MIN_HYDRATIONS 256 /* Size of hydration mempool */
 #define DEFAULT_HYDRATION_THRESHOLD 1 /* 1 region */
 #define DEFAULT_HYDRATION_BATCH_SIZE 1 /* Hydrate in batches of 1 region */
 
 #define COMMIT_PERIOD HZ /* 1 sec */
 
 /*
  * Hydration hash table size: 1 << HASH_TABLE_BITS
  */
 #define HASH_TABLE_BITS 15
 
 DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(clone_hydration_throttle,
     "A percentage of time allocated for hydrating regions");
 
 /* Slab cache for struct dm_clone_region_hydration */
 static struct kmem_cache *_hydration_cache;
 
 /* dm-clone metadata modes */
 enum clone_metadata_mode {
     CM_WRITE,       /* metadata may be changed */
     CM_READ_ONLY,       /* metadata may not be changed */
     CM_FAIL,        /* all metadata I/O fails */
 };
 
 struct hash_table_bucket;
 
 struct clone {
     struct dm_target *ti;
 
     struct dm_dev *metadata_dev;
     struct dm_dev *dest_dev;
     struct dm_dev *source_dev;
 
     unsigned long nr_regions;
     sector_t region_size;
     unsigned int region_shift;
 
     /*
      * A metadata commit and the actions taken in case it fails should run
      * as a single atomic step.
      */
     struct mutex commit_lock;
 
     struct dm_clone_metadata *cmd;
 
     /* Region hydration hash table */
     struct hash_table_bucket *ht;
 
     atomic_t ios_in_flight;
 
     wait_queue_head_t hydration_stopped;
 
     mempool_t hydration_pool;
 
     unsigned long last_commit_jiffies;
 
     /*
      * We defer incoming WRITE bios for regions that are not hydrated,
      * until after these regions have been hydrated.
      *
      * Also, we defer REQ_FUA and REQ_PREFLUSH bios, until after the
      * metadata have been committed.
      */
     spinlock_t lock;
     struct bio_list deferred_bios;
     struct bio_list deferred_discard_bios;
     struct bio_list deferred_flush_bios;
     struct bio_list deferred_flush_completions;
 
     /* Maximum number of regions being copied during background hydration. */
     unsigned int hydration_threshold;
 
     /* Number of regions to batch together during background hydration. */
     unsigned int hydration_batch_size;
 
     /* Which region to hydrate next */
     unsigned long hydration_offset;
 
     atomic_t hydrations_in_flight;
 
     /*
      * Save a copy of the table line rather than reconstructing it for the
      * status.
      */
     unsigned int nr_ctr_args;
     const char **ctr_args;
 
     struct workqueue_struct *wq;
     struct work_struct worker;
     struct delayed_work waker;
 
     struct dm_kcopyd_client *kcopyd_client;
 
     enum clone_metadata_mode mode;
     unsigned long flags;
 };
 
 /*
  * dm-clone flags
  */
 #define DM_CLONE_DISCARD_PASSDOWN 0
 #define DM_CLONE_HYDRATION_ENABLED 1
 #define DM_CLONE_HYDRATION_SUSPENDED 2
 
 /*---------------------------------------------------------------------------*/
 
 /*
  * Metadata failure handling.
  */
 static enum clone_metadata_mode get_clone_mode(struct clone *clone)
 {
     return READ_ONCE(clone->mode);
 }
 
 static const char *clone_device_name(struct clone *clone)
 {
     return dm_table_device_name(clone->ti->table);
 }
 
 static void __set_clone_mode(struct clone *clone, enum clone_metadata_mode new_mode)
 {
     static const char * const descs[] = {
         "read-write",
         "read-only",
         "fail"
     };
 
     enum clone_metadata_mode old_mode = get_clone_mode(clone);
 
     /* Never move out of fail mode */
     if (old_mode == CM_FAIL)
         new_mode = CM_FAIL;
 
     switch (new_mode) {
     case CM_FAIL:
     case CM_READ_ONLY:
         dm_clone_metadata_set_read_only(clone->cmd);
         break;
 
     case CM_WRITE:
         dm_clone_metadata_set_read_write(clone->cmd);
         break;
     }
 
     WRITE_ONCE(clone->mode, new_mode);
 
     if (new_mode != old_mode) {
         dm_table_event(clone->ti->table);
         DMINFO("%s: Switching to %s mode", clone_device_name(clone),
                descs[(int)new_mode]);
     }
 }
 
 static void __abort_transaction(struct clone *clone)
 {
     const char *dev_name = clone_device_name(clone);
 
     if (get_clone_mode(clone) >= CM_READ_ONLY)
         return;
 
     DMERR("%s: Aborting current metadata transaction", dev_name);
     if (dm_clone_metadata_abort(clone->cmd)) {
         DMERR("%s: Failed to abort metadata transaction", dev_name);
         __set_clone_mode(clone, CM_FAIL);
     }
 }
 
 static void __reload_in_core_bitset(struct clone *clone)
 {
     const char *dev_name = clone_device_name(clone);
 
     if (get_clone_mode(clone) == CM_FAIL)
         return;
 
     /* Reload the on-disk bitset */
     DMINFO("%s: Reloading on-disk bitmap", dev_name);
     if (dm_clone_reload_in_core_bitset(clone->cmd)) {
         DMERR("%s: Failed to reload on-disk bitmap", dev_name);
         __set_clone_mode(clone, CM_FAIL);
     }
 }
 
 static void __metadata_operation_failed(struct clone *clone, const char *op, int r)
 {
     DMERR("%s: Metadata operation `%s' failed: error = %d",
           clone_device_name(clone), op, r);
 
     __abort_transaction(clone);
     __set_clone_mode(clone, CM_READ_ONLY);
 
     /*
      * dm_clone_reload_in_core_bitset() may run concurrently with either
      * dm_clone_set_region_hydrated() or dm_clone_cond_set_range(), but
      * it's safe as we have already set the metadata to read-only mode.
      */
     __reload_in_core_bitset(clone);
 }
 
 /*---------------------------------------------------------------------------*/
 
 /* Wake up anyone waiting for region hydrations to stop */
 static inline void wakeup_hydration_waiters(struct clone *clone)
 {
     wake_up_all(&clone->hydration_stopped);
 }
 
 static inline void wake_worker(struct clone *clone)
 {
     queue_work(clone->wq, &clone->worker);
 }
 
 /*---------------------------------------------------------------------------*/
 
 /*
  * bio helper functions.
  */
 static inline void remap_to_source(struct clone *clone, struct bio *bio)
 {
     bio_set_dev(bio, clone->source_dev->bdev);
 }
 
 static inline void remap_to_dest(struct clone *clone, struct bio *bio)
 {
     bio_set_dev(bio, clone->dest_dev->bdev);
 }
 
 static bool bio_triggers_commit(struct clone *clone, struct bio *bio)
 {
     return op_is_flush(bio->bi_opf) &&
         dm_clone_changed_this_transaction(clone->cmd);
 }
 
 /* Get the address of the region in sectors */
 static inline sector_t region_to_sector(struct clone *clone, unsigned long region_nr)
 {
     return ((sector_t)region_nr << clone->region_shift);
 }
 
 /* Get the region number of the bio */
 static inline unsigned long bio_to_region(struct clone *clone, struct bio *bio)
 {
     return (bio->bi_iter.bi_sector >> clone->region_shift);
 }
 
 /* Get the region range covered by the bio */
 static void bio_region_range(struct clone *clone, struct bio *bio,
                  unsigned long *rs, unsigned long *nr_regions)
 {
     unsigned long end;
 
     *rs = dm_sector_div_up(bio->bi_iter.bi_sector, clone->region_size);
     end = bio_end_sector(bio) >> clone->region_shift;
 
     if (*rs >= end)
         *nr_regions = 0;
     else
         *nr_regions = end - *rs;
 }
 
 /* Check whether a bio overwrites a region */
 static inline bool is_overwrite_bio(struct clone *clone, struct bio *bio)
 {
     return (bio_data_dir(bio) == WRITE && bio_sectors(bio) == clone->region_size);
 }
 
 static void fail_bios(struct bio_list *bios, blk_status_t status)
 {
     struct bio *bio;
 
     while ((bio = bio_list_pop(bios))) {
         bio->bi_status = status;
         bio_endio(bio);
     }
 }
 
 static void submit_bios(struct bio_list *bios)
 {
     struct bio *bio;
     struct blk_plug plug;
 
     blk_start_plug(&plug);
 
     while ((bio = bio_list_pop(bios)))
         submit_bio_noacct(bio);
 
     blk_finish_plug(&plug);
 }
 
 /*
  * Submit bio to the underlying device.
  *
  * If the bio triggers a commit, delay it, until after the metadata have been
  * committed.
  *
  * NOTE: The bio remapping must be performed by the caller.
  */
 static void issue_bio(struct clone *clone, struct bio *bio)
 {
     if (!bio_triggers_commit(clone, bio)) {
         submit_bio_noacct(bio);
         return;
     }
 
     /*
      * If the metadata mode is RO or FAIL we won't be able to commit the
      * metadata, so we complete the bio with an error.
      */
     if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY)) {
         bio_io_error(bio);
         return;
     }
 
     /*
      * Batch together any bios that trigger commits and then issue a single
      * commit for them in process_deferred_flush_bios().
      */
     spin_lock_irq(&clone->lock);
     bio_list_add(&clone->deferred_flush_bios, bio);
     spin_unlock_irq(&clone->lock);
 
     wake_worker(clone);
 }
 
 /*
  * Remap bio to the destination device and submit it.
  *
  * If the bio triggers a commit, delay it, until after the metadata have been
  * committed.
  */
 static void remap_and_issue(struct clone *clone, struct bio *bio)
 {
     remap_to_dest(clone, bio);
     issue_bio(clone, bio);
 }
 
 /*
  * Issue bios that have been deferred until after their region has finished
  * hydrating.
  *
  * We delegate the bio submission to the worker thread, so this is safe to call
  * from interrupt context.
  */
 static void issue_deferred_bios(struct clone *clone, struct bio_list *bios)
 {
     struct bio *bio;
     unsigned long flags;
     struct bio_list flush_bios = BIO_EMPTY_LIST;
     struct bio_list normal_bios = BIO_EMPTY_LIST;
 
     if (bio_list_empty(bios))
         return;
 
     while ((bio = bio_list_pop(bios))) {
         if (bio_triggers_commit(clone, bio))
             bio_list_add(&flush_bios, bio);
         else
             bio_list_add(&normal_bios, bio);
     }
 
     spin_lock_irqsave(&clone->lock, flags);
     bio_list_merge(&clone->deferred_bios, &normal_bios);
     bio_list_merge(&clone->deferred_flush_bios, &flush_bios);
     spin_unlock_irqrestore(&clone->lock, flags);
 
     wake_worker(clone);
 }
 
 static void complete_overwrite_bio(struct clone *clone, struct bio *bio)
 {
     unsigned long flags;
 
     /*
      * If the bio has the REQ_FUA flag set we must commit the metadata
      * before signaling its completion.
      *
      * complete_overwrite_bio() is only called by hydration_complete(),
      * after having successfully updated the metadata. This means we don't
      * need to call dm_clone_changed_this_transaction() to check if the
      * metadata has changed and thus we can avoid taking the metadata spin
      * lock.
      */
     if (!(bio->bi_opf & REQ_FUA)) {
         bio_endio(bio);
         return;
     }
 
     /*
      * If the metadata mode is RO or FAIL we won't be able to commit the
      * metadata, so we complete the bio with an error.
      */
     if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY)) {
         bio_io_error(bio);
         return;
     }
 
     /*
      * Batch together any bios that trigger commits and then issue a single
      * commit for them in process_deferred_flush_bios().
      */
     spin_lock_irqsave(&clone->lock, flags);
     bio_list_add(&clone->deferred_flush_completions, bio);
     spin_unlock_irqrestore(&clone->lock, flags);
 
     wake_worker(clone);
 }
 
 static void trim_bio(struct bio *bio, sector_t sector, unsigned int len)
 {
     bio->bi_iter.bi_sector = sector;
     bio->bi_iter.bi_size = to_bytes(len);
 }
 
 static void complete_discard_bio(struct clone *clone, struct bio *bio, bool success)
 {
     unsigned long rs, nr_regions;
 
     /*
      * If the destination device supports discards, remap and trim the
      * discard bio and pass it down. Otherwise complete the bio
      * immediately.
      */
     if (test_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags) && success) {
         remap_to_dest(clone, bio);
         bio_region_range(clone, bio, &rs, &nr_regions);
         trim_bio(bio, region_to_sector(clone, rs),
              nr_regions << clone->region_shift);
         submit_bio_noacct(bio);
     } else
         bio_endio(bio);
 }
 
 static void process_discard_bio(struct clone *clone, struct bio *bio)
 {
     unsigned long rs, nr_regions;
 
     bio_region_range(clone, bio, &rs, &nr_regions);
     if (!nr_regions) {
         bio_endio(bio);
         return;
     }
 
     if (WARN_ON(rs >= clone->nr_regions || (rs + nr_regions) < rs ||
             (rs + nr_regions) > clone->nr_regions)) {
         DMERR("%s: Invalid range (%lu + %lu, total regions %lu) for discard (%llu + %u)",
               clone_device_name(clone), rs, nr_regions,
               clone->nr_regions,
               (unsigned long long)bio->bi_iter.bi_sector,
               bio_sectors(bio));
         bio_endio(bio);
         return;
     }
 
     /*
      * The covered regions are already hydrated so we just need to pass
      * down the discard.
      */
     if (dm_clone_is_range_hydrated(clone->cmd, rs, nr_regions)) {
         complete_discard_bio(clone, bio, true);
         return;
     }
 
     /*
      * If the metadata mode is RO or FAIL we won't be able to update the
      * metadata for the regions covered by the discard so we just ignore
      * it.
      */
     if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY)) {
         bio_endio(bio);
         return;
     }
 
     /*
      * Defer discard processing.
      */
     spin_lock_irq(&clone->lock);
     bio_list_add(&clone->deferred_discard_bios, bio);
     spin_unlock_irq(&clone->lock);
 
     wake_worker(clone);
 }
 
 /*---------------------------------------------------------------------------*/
 
 /*
  * dm-clone region hydrations.
  */
 struct dm_clone_region_hydration {
     struct clone *clone;
     unsigned long region_nr;
 
     struct bio *overwrite_bio;
     bio_end_io_t *overwrite_bio_end_io;
 
     struct bio_list deferred_bios;
 
     blk_status_t status;
 
     /* Used by hydration batching */
     struct list_head list;
 
     /* Used by hydration hash table */
     struct hlist_node h;
 };
 
 /*
  * Hydration hash table implementation.
  *
  * Ideally we would like to use list_bl, which uses bit spin locks and employs
  * the least significant bit of the list head to lock the corresponding bucket,
  * reducing the memory overhead for the locks. But, currently, list_bl and bit
  * spin locks don't support IRQ safe versions. Since we have to take the lock
  * in both process and interrupt context, we must fall back to using regular
  * spin locks; one per hash table bucket.
  */
 struct hash_table_bucket {
     struct hlist_head head;
 
     /* Spinlock protecting the bucket */
     spinlock_t lock;
 };
 
 #define bucket_lock_irqsave(bucket, flags) \
     spin_lock_irqsave(&(bucket)->lock, flags)
 
 #define bucket_unlock_irqrestore(bucket, flags) \
     spin_unlock_irqrestore(&(bucket)->lock, flags)
 
 #define bucket_lock_irq(bucket) \
     spin_lock_irq(&(bucket)->lock)
 
 #define bucket_unlock_irq(bucket) \
     spin_unlock_irq(&(bucket)->lock)
 
 static int hash_table_init(struct clone *clone)
 {
     unsigned int i, sz;
     struct hash_table_bucket *bucket;
 
     sz = 1 << HASH_TABLE_BITS;
 
     clone->ht = kvmalloc(sz * sizeof(struct hash_table_bucket), GFP_KERNEL);
     if (!clone->ht)
         return -ENOMEM;
 
     for (i = 0; i < sz; i++) {
         bucket = clone->ht + i;
 
         INIT_HLIST_HEAD(&bucket->head);
         spin_lock_init(&bucket->lock);
     }
 
     return 0;
 }
 
 static void hash_table_exit(struct clone *clone)
 {
     kvfree(clone->ht);
 }
 
 static struct hash_table_bucket *get_hash_table_bucket(struct clone *clone,
                                unsigned long region_nr)
 {
     return &clone->ht[hash_long(region_nr, HASH_TABLE_BITS)];
 }
 
 /*
  * Search hash table for a hydration with hd->region_nr == region_nr
  *
  * NOTE: Must be called with the bucket lock held
  */
 static struct dm_clone_region_hydration *__hash_find(struct hash_table_bucket *bucket,
                              unsigned long region_nr)
 {
     struct dm_clone_region_hydration *hd;
 
     hlist_for_each_entry(hd, &bucket->head, h) {
         if (hd->region_nr == region_nr)
             return hd;
     }
 
     return NULL;
 }
 
 /*
  * Insert a hydration into the hash table.
  *
  * NOTE: Must be called with the bucket lock held.
  */
 static inline void __insert_region_hydration(struct hash_table_bucket *bucket,
                          struct dm_clone_region_hydration *hd)
 {
     hlist_add_head(&hd->h, &bucket->head);
 }
 
 /*
  * This function inserts a hydration into the hash table, unless someone else
  * managed to insert a hydration for the same region first. In the latter case
  * it returns the existing hydration descriptor for this region.
  *
  * NOTE: Must be called with the hydration hash table lock held.
  */
 static struct dm_clone_region_hydration *
 __find_or_insert_region_hydration(struct hash_table_bucket *bucket,
                   struct dm_clone_region_hydration *hd)
 {
     struct dm_clone_region_hydration *hd2;
 
     hd2 = __hash_find(bucket, hd->region_nr);
     if (hd2)
         return hd2;
 
     __insert_region_hydration(bucket, hd);
 
     return hd;
 }
 
 /*---------------------------------------------------------------------------*/
 
 /* Allocate a hydration */
 static struct dm_clone_region_hydration *alloc_hydration(struct clone *clone)
 {
     struct dm_clone_region_hydration *hd;
 
     /*
      * Allocate a hydration from the hydration mempool.
      * This might block but it can't fail.
      */
     hd = mempool_alloc(&clone->hydration_pool, GFP_NOIO);
     hd->clone = clone;
 
     return hd;
 }
 
 static inline void free_hydration(struct dm_clone_region_hydration *hd)
 {
     mempool_free(hd, &hd->clone->hydration_pool);
 }
 
 /* Initialize a hydration */
 static void hydration_init(struct dm_clone_region_hydration *hd, unsigned long region_nr)
 {
     hd->region_nr = region_nr;
     hd->overwrite_bio = NULL;
     bio_list_init(&hd->deferred_bios);
     hd->status = 0;
 
     INIT_LIST_HEAD(&hd->list);
     INIT_HLIST_NODE(&hd->h);
 }
 
 /*---------------------------------------------------------------------------*/
 
 /*
  * Update dm-clone's metadata after a region has finished hydrating and remove
  * hydration from the hash table.
  */
 static int hydration_update_metadata(struct dm_clone_region_hydration *hd)
 {
     int r = 0;
     unsigned long flags;
     struct hash_table_bucket *bucket;
     struct clone *clone = hd->clone;
 
     if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY))
         r = -EPERM;
 
     /* Update the metadata */
     if (likely(!r) && hd->status == BLK_STS_OK)
         r = dm_clone_set_region_hydrated(clone->cmd, hd->region_nr);
 
     bucket = get_hash_table_bucket(clone, hd->region_nr);
 
     /* Remove hydration from hash table */
     bucket_lock_irqsave(bucket, flags);
     hlist_del(&hd->h);
     bucket_unlock_irqrestore(bucket, flags);
 
     return r;
 }
 
 /*
  * Complete a region's hydration:
  *
  *  1. Update dm-clone's metadata.
  *  2. Remove hydration from hash table.
  *  3. Complete overwrite bio.
  *  4. Issue deferred bios.
  *  5. If this was the last hydration, wake up anyone waiting for
  *     hydrations to finish.
  */
 static void hydration_complete(struct dm_clone_region_hydration *hd)
 {
     int r;
     blk_status_t status;
     struct clone *clone = hd->clone;
 
     r = hydration_update_metadata(hd);
 
     if (hd->status == BLK_STS_OK && likely(!r)) {
         if (hd->overwrite_bio)
             complete_overwrite_bio(clone, hd->overwrite_bio);
 
         issue_deferred_bios(clone, &hd->deferred_bios);
     } else {
         status = r ? BLK_STS_IOERR : hd->status;
 
         if (hd->overwrite_bio)
             bio_list_add(&hd->deferred_bios, hd->overwrite_bio);
 
         fail_bios(&hd->deferred_bios, status);
     }
 
     free_hydration(hd);
 
     if (atomic_dec_and_test(&clone->hydrations_in_flight))
         wakeup_hydration_waiters(clone);
 }
 
 static void hydration_kcopyd_callback(int read_err, unsigned long write_err, void *context)
 {
     blk_status_t status;
 
     struct dm_clone_region_hydration *tmp, *hd = context;
     struct clone *clone = hd->clone;
 
     LIST_HEAD(batched_hydrations);
 
     if (read_err || write_err) {
         DMERR_LIMIT("%s: hydration failed", clone_device_name(clone));
         status = BLK_STS_IOERR;
     } else {
         status = BLK_STS_OK;
     }
     list_splice_tail(&hd->list, &batched_hydrations);
 
     hd->status = status;
     hydration_complete(hd);
 
     /* Complete batched hydrations */
     list_for_each_entry_safe(hd, tmp, &batched_hydrations, list) {
         hd->status = status;
         hydration_complete(hd);
     }
 
     /* Continue background hydration, if there is no I/O in-flight */
     if (test_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags) &&
         !atomic_read(&clone->ios_in_flight))
         wake_worker(clone);
 }
 
 static void hydration_copy(struct dm_clone_region_hydration *hd, unsigned int nr_regions)
 {
     unsigned long region_start, region_end;
     sector_t tail_size, region_size, total_size;
     struct dm_io_region from, to;
     struct clone *clone = hd->clone;
 
     if (WARN_ON(!nr_regions))
         return;
 
     region_size = clone->region_size;
     region_start = hd->region_nr;
     region_end = region_start + nr_regions - 1;
 
     total_size = region_to_sector(clone, nr_regions - 1);
 
     if (region_end == clone->nr_regions - 1) {
         /*
          * The last region of the target might be smaller than
          * region_size.
          */
         tail_size = clone->ti->len & (region_size - 1);
         if (!tail_size)
             tail_size = region_size;
     } else {
         tail_size = region_size;
     }
 
     total_size += tail_size;
 
     from.bdev = clone->source_dev->bdev;
     from.sector = region_to_sector(clone, region_start);
     from.count = total_size;
 
     to.bdev = clone->dest_dev->bdev;
     to.sector = from.sector;
     to.count = from.count;
 
     /* Issue copy */
     atomic_add(nr_regions, &clone->hydrations_in_flight);
     dm_kcopyd_copy(clone->kcopyd_client, &from, 1, &to, 0,
                hydration_kcopyd_callback, hd);
 }
 
 static void overwrite_endio(struct bio *bio)
 {
     struct dm_clone_region_hydration *hd = bio->bi_private;
 
     bio->bi_end_io = hd->overwrite_bio_end_io;
     hd->status = bio->bi_status;
 
     hydration_complete(hd);
 }
 
 static void hydration_overwrite(struct dm_clone_region_hydration *hd, struct bio *bio)
 {
     /*
      * We don't need to save and restore bio->bi_private because device
      * mapper core generates a new bio for us to use, with clean
      * bi_private.
      */
     hd->overwrite_bio = bio;
     hd->overwrite_bio_end_io = bio->bi_end_io;
 
     bio->bi_end_io = overwrite_endio;
     bio->bi_private = hd;
 
     atomic_inc(&hd->clone->hydrations_in_flight);
     submit_bio_noacct(bio);
 }
 
 /*
  * Hydrate bio's region.
  *
  * This function starts the hydration of the bio's region and puts the bio in
  * the list of deferred bios for this region. In case, by the time this
  * function is called, the region has finished hydrating it's submitted to the
  * destination device.
  *
  * NOTE: The bio remapping must be performed by the caller.
  */
 static void hydrate_bio_region(struct clone *clone, struct bio *bio)
 {
     unsigned long region_nr;
     struct hash_table_bucket *bucket;
     struct dm_clone_region_hydration *hd, *hd2;
 
     region_nr = bio_to_region(clone, bio);
     bucket = get_hash_table_bucket(clone, region_nr);
 
     bucket_lock_irq(bucket);
 
     hd = __hash_find(bucket, region_nr);
     if (hd) {
         /* Someone else is hydrating the region */
         bio_list_add(&hd->deferred_bios, bio);
         bucket_unlock_irq(bucket);
         return;
     }
 
     if (dm_clone_is_region_hydrated(clone->cmd, region_nr)) {
         /* The region has been hydrated */
         bucket_unlock_irq(bucket);
         issue_bio(clone, bio);
         return;
     }
 
     /*
      * We must allocate a hydration descriptor and start the hydration of
      * the corresponding region.
      */
     bucket_unlock_irq(bucket);
 
     hd = alloc_hydration(clone);
     hydration_init(hd, region_nr);
 
     bucket_lock_irq(bucket);
 
     /* Check if the region has been hydrated in the meantime. */
     if (dm_clone_is_region_hydrated(clone->cmd, region_nr)) {
         bucket_unlock_irq(bucket);
         free_hydration(hd);
         issue_bio(clone, bio);
         return;
     }
 
     hd2 = __find_or_insert_region_hydration(bucket, hd);
     if (hd2 != hd) {
         /* Someone else started the region's hydration. */
         bio_list_add(&hd2->deferred_bios, bio);
         bucket_unlock_irq(bucket);
         free_hydration(hd);
         return;
     }
 
     /*
      * If the metadata mode is RO or FAIL then there is no point starting a
      * hydration, since we will not be able to update the metadata when the
      * hydration finishes.
      */
     if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY)) {
         hlist_del(&hd->h);
         bucket_unlock_irq(bucket);
         free_hydration(hd);
         bio_io_error(bio);
         return;
     }
 
     /*
      * Start region hydration.
      *
      * If a bio overwrites a region, i.e., its size is equal to the
      * region's size, then we don't need to copy the region from the source
      * to the destination device.
      */
     if (is_overwrite_bio(clone, bio)) {
         bucket_unlock_irq(bucket);
         hydration_overwrite(hd, bio);
     } else {
         bio_list_add(&hd->deferred_bios, bio);
         bucket_unlock_irq(bucket);
         hydration_copy(hd, 1);
     }
 }
 
 /*---------------------------------------------------------------------------*/
 
 /*
  * Background hydrations.
  */
 
 /*
  * Batch region hydrations.
  *
  * To better utilize device bandwidth we batch together the hydration of
  * adjacent regions. This allows us to use small region sizes, e.g., 4KB, which
  * is good for small, random write performance (because of the overwriting of
  * un-hydrated regions) and at the same time issue big copy requests to kcopyd
  * to achieve high hydration bandwidth.
  */
 struct batch_info {
     struct dm_clone_region_hydration *head;
     unsigned int nr_batched_regions;
 };
 
 static void __batch_hydration(struct batch_info *batch,
                   struct dm_clone_region_hydration *hd)
 {
     struct clone *clone = hd->clone;
     unsigned int max_batch_size = READ_ONCE(clone->hydration_batch_size);
 
     if (batch->head) {
         /* Try to extend the current batch */
         if (batch->nr_batched_regions < max_batch_size &&
             (batch->head->region_nr + batch->nr_batched_regions) == hd->region_nr) {
             list_add_tail(&hd->list, &batch->head->list);
             batch->nr_batched_regions++;
             hd = NULL;
         }
 
         /* Check if we should issue the current batch */
         if (batch->nr_batched_regions >= max_batch_size || hd) {
             hydration_copy(batch->head, batch->nr_batched_regions);
             batch->head = NULL;
             batch->nr_batched_regions = 0;
         }
     }
 
     if (!hd)
         return;
 
     /* We treat max batch sizes of zero and one equivalently */
     if (max_batch_size <= 1) {
         hydration_copy(hd, 1);
         return;
     }
 
     /* Start a new batch */
     BUG_ON(!list_empty(&hd->list));
     batch->head = hd;
     batch->nr_batched_regions = 1;
 }
 
 static unsigned long __start_next_hydration(struct clone *clone,
                         unsigned long offset,
                         struct batch_info *batch)
 {
     struct hash_table_bucket *bucket;
     struct dm_clone_region_hydration *hd;
     unsigned long nr_regions = clone->nr_regions;
 
     hd = alloc_hydration(clone);
 
     /* Try to find a region to hydrate. */
     do {
         offset = dm_clone_find_next_unhydrated_region(clone->cmd, offset);
         if (offset == nr_regions)
             break;
 
         bucket = get_hash_table_bucket(clone, offset);
         bucket_lock_irq(bucket);
 
         if (!dm_clone_is_region_hydrated(clone->cmd, offset) &&
             !__hash_find(bucket, offset)) {
             hydration_init(hd, offset);
             __insert_region_hydration(bucket, hd);
             bucket_unlock_irq(bucket);
 
             /* Batch hydration */
             __batch_hydration(batch, hd);
 
             return (offset + 1);
         }
 
         bucket_unlock_irq(bucket);
 
     } while (++offset < nr_regions);
 
     if (hd)
         free_hydration(hd);
 
     return offset;
 }
 
 /*
  * This function searches for regions that still reside in the source device
  * and starts their hydration.
  */
 static void do_hydration(struct clone *clone)
 {
     unsigned int current_volume;
     unsigned long offset, nr_regions = clone->nr_regions;
 
     struct batch_info batch = {
         .head = NULL,
         .nr_batched_regions = 0,
     };
 
     if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY))
         return;
 
     if (dm_clone_is_hydration_done(clone->cmd))
         return;
 
     /*
      * Avoid race with device suspension.
      */
     atomic_inc(&clone->hydrations_in_flight);
 
     /*
      * Make sure atomic_inc() is ordered before test_bit(), otherwise we
      * might race with clone_postsuspend() and start a region hydration
      * after the target has been suspended.
      *
      * This is paired with the smp_mb__after_atomic() in
      * clone_postsuspend().
      */
     smp_mb__after_atomic();
 
     offset = clone->hydration_offset;
     while (likely(!test_bit(DM_CLONE_HYDRATION_SUSPENDED, &clone->flags)) &&
            !atomic_read(&clone->ios_in_flight) &&
            test_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags) &&
            offset < nr_regions) {
         current_volume = atomic_read(&clone->hydrations_in_flight);
         current_volume += batch.nr_batched_regions;
 
         if (current_volume > READ_ONCE(clone->hydration_threshold))
             break;
 
         offset = __start_next_hydration(clone, offset, &batch);
     }
 
     if (batch.head)
         hydration_copy(batch.head, batch.nr_batched_regions);
 
     if (offset >= nr_regions)
         offset = 0;
 
     clone->hydration_offset = offset;
 
     if (atomic_dec_and_test(&clone->hydrations_in_flight))
         wakeup_hydration_waiters(clone);
 }
 
 /*---------------------------------------------------------------------------*/
 
 static bool need_commit_due_to_time(struct clone *clone)
 {
     return !time_in_range(jiffies, clone->last_commit_jiffies,
                   clone->last_commit_jiffies + COMMIT_PERIOD);
 }
 
 /*
  * A non-zero return indicates read-only or fail mode.
  */
 static int commit_metadata(struct clone *clone, bool *dest_dev_flushed)
 {
     int r = 0;
 
     if (dest_dev_flushed)
         *dest_dev_flushed = false;
 
     mutex_lock(&clone->commit_lock);
 
     if (!dm_clone_changed_this_transaction(clone->cmd))
         goto out;
 
     if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY)) {
         r = -EPERM;
         goto out;
     }
 
     r = dm_clone_metadata_pre_commit(clone->cmd);
     if (unlikely(r)) {
         __metadata_operation_failed(clone, "dm_clone_metadata_pre_commit", r);
         goto out;
     }
 
     r = blkdev_issue_flush(clone->dest_dev->bdev);
     if (unlikely(r)) {
         __metadata_operation_failed(clone, "flush destination device", r);
         goto out;
     }
 
     if (dest_dev_flushed)
         *dest_dev_flushed = true;
 
     r = dm_clone_metadata_commit(clone->cmd);
     if (unlikely(r)) {
         __metadata_operation_failed(clone, "dm_clone_metadata_commit", r);
         goto out;
     }
 
     if (dm_clone_is_hydration_done(clone->cmd))
         dm_table_event(clone->ti->table);
 out:
     mutex_unlock(&clone->commit_lock);
 
     return r;
 }
 
 static void process_deferred_discards(struct clone *clone)
 {
     int r = -EPERM;
     struct bio *bio;
     struct blk_plug plug;
     unsigned long rs, nr_regions;
     struct bio_list discards = BIO_EMPTY_LIST;
 
     spin_lock_irq(&clone->lock);
     bio_list_merge(&discards, &clone->deferred_discard_bios);
     bio_list_init(&clone->deferred_discard_bios);
     spin_unlock_irq(&clone->lock);
 
     if (bio_list_empty(&discards))
         return;
 
     if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY))
         goto out;
 
     /* Update the metadata */
     bio_list_for_each(bio, &discards) {
         bio_region_range(clone, bio, &rs, &nr_regions);
         /*
          * A discard request might cover regions that have been already
          * hydrated. There is no need to update the metadata for these
          * regions.
          */
         r = dm_clone_cond_set_range(clone->cmd, rs, nr_regions);
         if (unlikely(r))
             break;
     }
 out:
     blk_start_plug(&plug);
     while ((bio = bio_list_pop(&discards)))
         complete_discard_bio(clone, bio, r == 0);
     blk_finish_plug(&plug);
 }
 
 static void process_deferred_bios(struct clone *clone)
 {
     struct bio_list bios = BIO_EMPTY_LIST;
 
     spin_lock_irq(&clone->lock);
     bio_list_merge(&bios, &clone->deferred_bios);
     bio_list_init(&clone->deferred_bios);
     spin_unlock_irq(&clone->lock);
 
     if (bio_list_empty(&bios))
         return;
 
     submit_bios(&bios);
 }
 
 static void process_deferred_flush_bios(struct clone *clone)
 {
     struct bio *bio;
     bool dest_dev_flushed;
     struct bio_list bios = BIO_EMPTY_LIST;
     struct bio_list bio_completions = BIO_EMPTY_LIST;
 
     /*
      * If there are any deferred flush bios, we must commit the metadata
      * before issuing them or signaling their completion.
      */
     spin_lock_irq(&clone->lock);
     bio_list_merge(&bios, &clone->deferred_flush_bios);
     bio_list_init(&clone->deferred_flush_bios);
 
     bio_list_merge(&bio_completions, &clone->deferred_flush_completions);
     bio_list_init(&clone->deferred_flush_completions);
     spin_unlock_irq(&clone->lock);
 
     if (bio_list_empty(&bios) && bio_list_empty(&bio_completions) &&
         !(dm_clone_changed_this_transaction(clone->cmd) && need_commit_due_to_time(clone)))
         return;
 
     if (commit_metadata(clone, &dest_dev_flushed)) {
         bio_list_merge(&bios, &bio_completions);
 
         while ((bio = bio_list_pop(&bios)))
             bio_io_error(bio);
 
         return;
     }
 
     clone->last_commit_jiffies = jiffies;
 
     while ((bio = bio_list_pop(&bio_completions)))
         bio_endio(bio);
 
     while ((bio = bio_list_pop(&bios))) {
         if ((bio->bi_opf & REQ_PREFLUSH) && dest_dev_flushed) {
             /* We just flushed the destination device as part of
              * the metadata commit, so there is no reason to send
              * another flush.
              */
             bio_endio(bio);
         } else {
             submit_bio_noacct(bio);
         }
     }
 }
 
 static void do_worker(struct work_struct *work)
 {
     struct clone *clone = container_of(work, typeof(*clone), worker);
 
     process_deferred_bios(clone);
     process_deferred_discards(clone);
 
     /*
      * process_deferred_flush_bios():
      *
      *   - Commit metadata
      *
      *   - Process deferred REQ_FUA completions
      *
      *   - Process deferred REQ_PREFLUSH bios
      */
     process_deferred_flush_bios(clone);
 
     /* Background hydration */
     do_hydration(clone);
 }
 
 /*
  * Commit periodically so that not too much unwritten data builds up.
  *
  * Also, restart background hydration, if it has been stopped by in-flight I/O.
  */
 static void do_waker(struct work_struct *work)
 {
     struct clone *clone = container_of(to_delayed_work(work), struct clone, waker);
 
     wake_worker(clone);
     queue_delayed_work(clone->wq, &clone->waker, COMMIT_PERIOD);
 }
 
 /*---------------------------------------------------------------------------*/
 
 /*
  * Target methods
  */
 static int clone_map(struct dm_target *ti, struct bio *bio)
 {
     struct clone *clone = ti->private;
     unsigned long region_nr;
 
     atomic_inc(&clone->ios_in_flight);
 
     if (unlikely(get_clone_mode(clone) == CM_FAIL))
         return DM_MAPIO_KILL;
 
     /*
      * REQ_PREFLUSH bios carry no data:
      *
      * - Commit metadata, if changed
      *
      * - Pass down to destination device
      */
     if (bio->bi_opf & REQ_PREFLUSH) {
         remap_and_issue(clone, bio);
         return DM_MAPIO_SUBMITTED;
     }
 
     bio->bi_iter.bi_sector = dm_target_offset(ti, bio->bi_iter.bi_sector);
 
     /*
      * dm-clone interprets discards and performs a fast hydration of the
      * discarded regions, i.e., we skip the copy from the source device and
      * just mark the regions as hydrated.
      */
     if (bio_op(bio) == REQ_OP_DISCARD) {
         process_discard_bio(clone, bio);
         return DM_MAPIO_SUBMITTED;
     }
 
     /*
      * If the bio's region is hydrated, redirect it to the destination
      * device.
      *
      * If the region is not hydrated and the bio is a READ, redirect it to
      * the source device.
      *
      * Else, defer WRITE bio until after its region has been hydrated and
      * start the region's hydration immediately.
      */
     region_nr = bio_to_region(clone, bio);
     if (dm_clone_is_region_hydrated(clone->cmd, region_nr)) {
         remap_and_issue(clone, bio);
         return DM_MAPIO_SUBMITTED;
     } else if (bio_data_dir(bio) == READ) {
         remap_to_source(clone, bio);
         return DM_MAPIO_REMAPPED;
     }
 
     remap_to_dest(clone, bio);
     hydrate_bio_region(clone, bio);
 
     return DM_MAPIO_SUBMITTED;
 }
 
 static int clone_endio(struct dm_target *ti, struct bio *bio, blk_status_t *error)
 {
     struct clone *clone = ti->private;
 
     atomic_dec(&clone->ios_in_flight);
 
     return DM_ENDIO_DONE;
 }
 
 static void emit_flags(struct clone *clone, char *result, unsigned int maxlen,
                ssize_t *sz_ptr)
 {
     ssize_t sz = *sz_ptr;
     unsigned int count;
 
     count = !test_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags);
     count += !test_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags);
 
     DMEMIT("%u ", count);
 
     if (!test_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags))
         DMEMIT("no_hydration ");
 
     if (!test_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags))
         DMEMIT("no_discard_passdown ");
 
     *sz_ptr = sz;
 }
 
 static void emit_core_args(struct clone *clone, char *result,
                unsigned int maxlen, ssize_t *sz_ptr)
 {
     ssize_t sz = *sz_ptr;
     unsigned int count = 4;
 
     DMEMIT("%u hydration_threshold %u hydration_batch_size %u ", count,
            READ_ONCE(clone->hydration_threshold),
            READ_ONCE(clone->hydration_batch_size));
 
     *sz_ptr = sz;
 }
 
 /*
  * Status format:
  *
  * <metadata block size> <#used metadata blocks>/<#total metadata blocks>
  * <clone region size> <#hydrated regions>/<#total regions> <#hydrating regions>
  * <#features> <features>* <#core args> <core args>* <clone metadata mode>
  */
 static void clone_status(struct dm_target *ti, status_type_t type,
              unsigned int status_flags, char *result,
              unsigned int maxlen)
 {
     int r;
     unsigned int i;
     ssize_t sz = 0;
     dm_block_t nr_free_metadata_blocks = 0;
     dm_block_t nr_metadata_blocks = 0;
     char buf[BDEVNAME_SIZE];
     struct clone *clone = ti->private;
 
     switch (type) {
     case STATUSTYPE_INFO:
         if (get_clone_mode(clone) == CM_FAIL) {
             DMEMIT("Fail");
             break;
         }
 
         /* Commit to ensure statistics aren't out-of-date */
         if (!(status_flags & DM_STATUS_NOFLUSH_FLAG) && !dm_suspended(ti))
             (void) commit_metadata(clone, NULL);
 
         r = dm_clone_get_free_metadata_block_count(clone->cmd, &nr_free_metadata_blocks);
 
         if (r) {
             DMERR("%s: dm_clone_get_free_metadata_block_count returned %d",
                   clone_device_name(clone), r);
             goto error;
         }
 
         r = dm_clone_get_metadata_dev_size(clone->cmd, &nr_metadata_blocks);
 
         if (r) {
             DMERR("%s: dm_clone_get_metadata_dev_size returned %d",
                   clone_device_name(clone), r);
             goto error;
         }
 
         DMEMIT("%u %llu/%llu %llu %u/%lu %u ",
                DM_CLONE_METADATA_BLOCK_SIZE,
                (unsigned long long)(nr_metadata_blocks - nr_free_metadata_blocks),
                (unsigned long long)nr_metadata_blocks,
                (unsigned long long)clone->region_size,
                dm_clone_nr_of_hydrated_regions(clone->cmd),
                clone->nr_regions,
                atomic_read(&clone->hydrations_in_flight));
 
         emit_flags(clone, result, maxlen, &sz);
         emit_core_args(clone, result, maxlen, &sz);
 
         switch (get_clone_mode(clone)) {
         case CM_WRITE:
             DMEMIT("rw");
             break;
         case CM_READ_ONLY:
             DMEMIT("ro");
             break;
         case CM_FAIL:
             DMEMIT("Fail");
         }
 
         break;
 
     case STATUSTYPE_TABLE:
         format_dev_t(buf, clone->metadata_dev->bdev->bd_dev);
         DMEMIT("%s ", buf);
 
         format_dev_t(buf, clone->dest_dev->bdev->bd_dev);
         DMEMIT("%s ", buf);
 
         format_dev_t(buf, clone->source_dev->bdev->bd_dev);
         DMEMIT("%s", buf);
 
         for (i = 0; i < clone->nr_ctr_args; i++)
             DMEMIT(" %s", clone->ctr_args[i]);
         break;
 
     case STATUSTYPE_IMA:
         *result = '\0';
         break;
     }
 
     return;
 
 error:
     DMEMIT("Error");
 }
 
 static sector_t get_dev_size(struct dm_dev *dev)
 {
     return bdev_nr_sectors(dev->bdev);
 }
 
 /*---------------------------------------------------------------------------*/
 
 /*
  * Construct a clone device mapping:
  *
  * clone <metadata dev> <destination dev> <source dev> <region size>
  *  [<#feature args> [<feature arg>]* [<#core args> [key value]*]]
  *
  * metadata dev: Fast device holding the persistent metadata
  * destination dev: The destination device, which will become a clone of the
  *                  source device
  * source dev: The read-only source device that gets cloned
  * region size: dm-clone unit size in sectors
  *
  * #feature args: Number of feature arguments passed
  * feature args: E.g. no_hydration, no_discard_passdown
  *
  * #core arguments: An even number of core arguments
  * core arguments: Key/value pairs for tuning the core
  *         E.g. 'hydration_threshold 256'
  */
 static int parse_feature_args(struct dm_arg_set *as, struct clone *clone)
 {
     int r;
     unsigned int argc;
     const char *arg_name;
     struct dm_target *ti = clone->ti;
 
     const struct dm_arg args = {
         .min = 0,
         .max = 2,
         .error = "Invalid number of feature arguments"
     };
 
     /* No feature arguments supplied */
     if (!as->argc)
         return 0;
 
     r = dm_read_arg_group(&args, as, &argc, &ti->error);
     if (r)
         return r;
 
     while (argc) {
         arg_name = dm_shift_arg(as);
         argc--;
 
         if (!strcasecmp(arg_name, "no_hydration")) {
             __clear_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags);
         } else if (!strcasecmp(arg_name, "no_discard_passdown")) {
             __clear_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags);
         } else {
             ti->error = "Invalid feature argument";
             return -EINVAL;
         }
     }
 
     return 0;
 }
 
 static int parse_core_args(struct dm_arg_set *as, struct clone *clone)
 {
     int r;
     unsigned int argc;
     unsigned int value;
     const char *arg_name;
     struct dm_target *ti = clone->ti;
 
     const struct dm_arg args = {
         .min = 0,
         .max = 4,
         .error = "Invalid number of core arguments"
     };
 
     /* Initialize core arguments */
     clone->hydration_batch_size = DEFAULT_HYDRATION_BATCH_SIZE;
     clone->hydration_threshold = DEFAULT_HYDRATION_THRESHOLD;
 
     /* No core arguments supplied */
     if (!as->argc)
         return 0;
 
     r = dm_read_arg_group(&args, as, &argc, &ti->error);
     if (r)
         return r;
 
     if (argc & 1) {
         ti->error = "Number of core arguments must be even";
         return -EINVAL;
     }
 
     while (argc) {
         arg_name = dm_shift_arg(as);
         argc -= 2;
 
         if (!strcasecmp(arg_name, "hydration_threshold")) {
             if (kstrtouint(dm_shift_arg(as), 10, &value)) {
                 ti->error = "Invalid value for argument `hydration_threshold'";
                 return -EINVAL;
             }
             clone->hydration_threshold = value;
         } else if (!strcasecmp(arg_name, "hydration_batch_size")) {
             if (kstrtouint(dm_shift_arg(as), 10, &value)) {
                 ti->error = "Invalid value for argument `hydration_batch_size'";
                 return -EINVAL;
             }
             clone->hydration_batch_size = value;
         } else {
             ti->error = "Invalid core argument";
             return -EINVAL;
         }
     }
 
     return 0;
 }
 
 static int parse_region_size(struct clone *clone, struct dm_arg_set *as, char **error)
 {
     int r;
     unsigned int region_size;
     struct dm_arg arg;
 
     arg.min = MIN_REGION_SIZE;
     arg.max = MAX_REGION_SIZE;
     arg.error = "Invalid region size";
 
     r = dm_read_arg(&arg, as, &region_size, error);
     if (r)
         return r;
 
     /* Check region size is a power of 2 */
     if (!is_power_of_2(region_size)) {
         *error = "Region size is not a power of 2";
         return -EINVAL;
     }
 
     /* Validate the region size against the device logical block size */
     if (region_size % (bdev_logical_block_size(clone->source_dev->bdev) >> 9) ||
         region_size % (bdev_logical_block_size(clone->dest_dev->bdev) >> 9)) {
         *error = "Region size is not a multiple of device logical block size";
         return -EINVAL;
     }
 
     clone->region_size = region_size;
 
     return 0;
 }
 
 static int validate_nr_regions(unsigned long n, char **error)
 {
     /*
      * dm_bitset restricts us to 2^32 regions. test_bit & co. restrict us
      * further to 2^31 regions.
      */
     if (n > (1UL << 31)) {
         *error = "Too many regions. Consider increasing the region size";
         return -EINVAL;
     }
 
     return 0;
 }
 
 static int parse_metadata_dev(struct clone *clone, struct dm_arg_set *as, char **error)
 {
     int r;
     sector_t metadata_dev_size;
 
     r = dm_get_device(clone->ti, dm_shift_arg(as), FMODE_READ | FMODE_WRITE,
               &clone->metadata_dev);
     if (r) {
         *error = "Error opening metadata device";
         return r;
     }
 
     metadata_dev_size = get_dev_size(clone->metadata_dev);
     if (metadata_dev_size > DM_CLONE_METADATA_MAX_SECTORS_WARNING)
         DMWARN("Metadata device %pg is larger than %u sectors: excess space will not be used.",
                clone->metadata_dev->bdev, DM_CLONE_METADATA_MAX_SECTORS);
 
     return 0;
 }
 
 static int parse_dest_dev(struct clone *clone, struct dm_arg_set *as, char **error)
 {
     int r;
     sector_t dest_dev_size;
 
     r = dm_get_device(clone->ti, dm_shift_arg(as), FMODE_READ | FMODE_WRITE,
               &clone->dest_dev);
     if (r) {
         *error = "Error opening destination device";
         return r;
     }
 
     dest_dev_size = get_dev_size(clone->dest_dev);
     if (dest_dev_size < clone->ti->len) {
         dm_put_device(clone->ti, clone->dest_dev);
         *error = "Device size larger than destination device";
         return -EINVAL;
     }
 
     return 0;
 }
 
 static int parse_source_dev(struct clone *clone, struct dm_arg_set *as, char **error)
 {
     int r;
     sector_t source_dev_size;
 
     r = dm_get_device(clone->ti, dm_shift_arg(as), FMODE_READ,
               &clone->source_dev);
     if (r) {
         *error = "Error opening source device";
         return r;
     }
 
     source_dev_size = get_dev_size(clone->source_dev);
     if (source_dev_size < clone->ti->len) {
         dm_put_device(clone->ti, clone->source_dev);
         *error = "Device size larger than source device";
         return -EINVAL;
     }
 
     return 0;
 }
 
 static int copy_ctr_args(struct clone *clone, int argc, const char **argv, char **error)
 {
     unsigned int i;
     const char **copy;
 
     copy = kcalloc(argc, sizeof(*copy), GFP_KERNEL);
     if (!copy)
         goto error;
 
     for (i = 0; i < argc; i++) {
         copy[i] = kstrdup(argv[i], GFP_KERNEL);
 
         if (!copy[i]) {
             while (i--)
                 kfree(copy[i]);
             kfree(copy);
             goto error;
         }
     }
 
     clone->nr_ctr_args = argc;
     clone->ctr_args = copy;
     return 0;
 
 error:
     *error = "Failed to allocate memory for table line";
     return -ENOMEM;
 }
 
 static int clone_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 {
     int r;
     sector_t nr_regions;
     struct clone *clone;
     struct dm_arg_set as;
 
     if (argc < 4) {
         ti->error = "Invalid number of arguments";
         return -EINVAL;
     }
 
     as.argc = argc;
     as.argv = argv;
 
     clone = kzalloc(sizeof(*clone), GFP_KERNEL);
     if (!clone) {
         ti->error = "Failed to allocate clone structure";
         return -ENOMEM;
     }
 
     clone->ti = ti;
 
     /* Initialize dm-clone flags */
     __set_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags);
     __set_bit(DM_CLONE_HYDRATION_SUSPENDED, &clone->flags);
     __set_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags);
 
     r = parse_metadata_dev(clone, &as, &ti->error);
     if (r)
         goto out_with_clone;
 
     r = parse_dest_dev(clone, &as, &ti->error);
     if (r)
         goto out_with_meta_dev;
 
     r = parse_source_dev(clone, &as, &ti->error);
     if (r)
         goto out_with_dest_dev;
 
     r = parse_region_size(clone, &as, &ti->error);
     if (r)
         goto out_with_source_dev;
 
     clone->region_shift = __ffs(clone->region_size);
     nr_regions = dm_sector_div_up(ti->len, clone->region_size);
 
     /* Check for overflow */
     if (nr_regions != (unsigned long)nr_regions) {
         ti->error = "Too many regions. Consider increasing the region size";
         r = -EOVERFLOW;
         goto out_with_source_dev;
     }
 
     clone->nr_regions = nr_regions;
 
     r = validate_nr_regions(clone->nr_regions, &ti->error);
     if (r)
         goto out_with_source_dev;
 
     r = dm_set_target_max_io_len(ti, clone->region_size);
     if (r) {
         ti->error = "Failed to set max io len";
         goto out_with_source_dev;
     }
 
     r = parse_feature_args(&as, clone);
     if (r)
         goto out_with_source_dev;
 
     r = parse_core_args(&as, clone);
     if (r)
         goto out_with_source_dev;
 
     /* Load metadata */
     clone->cmd = dm_clone_metadata_open(clone->metadata_dev->bdev, ti->len,
                         clone->region_size);
     if (IS_ERR(clone->cmd)) {
         ti->error = "Failed to load metadata";
         r = PTR_ERR(clone->cmd);
         goto out_with_source_dev;
     }
 
     __set_clone_mode(clone, CM_WRITE);
 
     if (get_clone_mode(clone) != CM_WRITE) {
         ti->error = "Unable to get write access to metadata, please check/repair metadata";
         r = -EPERM;
         goto out_with_metadata;
     }
 
     clone->last_commit_jiffies = jiffies;
 
     /* Allocate hydration hash table */
     r = hash_table_init(clone);
     if (r) {
         ti->error = "Failed to allocate hydration hash table";
         goto out_with_metadata;
     }
 
     atomic_set(&clone->ios_in_flight, 0);
     init_waitqueue_head(&clone->hydration_stopped);
     spin_lock_init(&clone->lock);
     bio_list_init(&clone->deferred_bios);
     bio_list_init(&clone->deferred_discard_bios);
     bio_list_init(&clone->deferred_flush_bios);
     bio_list_init(&clone->deferred_flush_completions);
     clone->hydration_offset = 0;
     atomic_set(&clone->hydrations_in_flight, 0);
 
     clone->wq = alloc_workqueue("dm-" DM_MSG_PREFIX, WQ_MEM_RECLAIM, 0);
     if (!clone->wq) {
         ti->error = "Failed to allocate workqueue";
         r = -ENOMEM;
         goto out_with_ht;
     }
 
     INIT_WORK(&clone->worker, do_worker);
     INIT_DELAYED_WORK(&clone->waker, do_waker);
 
     clone->kcopyd_client = dm_kcopyd_client_create(&dm_kcopyd_throttle);
     if (IS_ERR(clone->kcopyd_client)) {
         r = PTR_ERR(clone->kcopyd_client);
         goto out_with_wq;
     }
 
     r = mempool_init_slab_pool(&clone->hydration_pool, MIN_HYDRATIONS,
                    _hydration_cache);
     if (r) {
         ti->error = "Failed to create dm_clone_region_hydration memory pool";
         goto out_with_kcopyd;
     }
 
     /* Save a copy of the table line */
     r = copy_ctr_args(clone, argc - 3, (const char **)argv + 3, &ti->error);
     if (r)
         goto out_with_mempool;
 
     mutex_init(&clone->commit_lock);
 
     /* Enable flushes */
     ti->num_flush_bios = 1;
     ti->flush_supported = true;
 
     /* Enable discards */
     ti->discards_supported = true;
     ti->num_discard_bios = 1;
 
     ti->private = clone;
 
     return 0;
 
 out_with_mempool:
     mempool_exit(&clone->hydration_pool);
 out_with_kcopyd:
     dm_kcopyd_client_destroy(clone->kcopyd_client);
 out_with_wq:
     destroy_workqueue(clone->wq);
 out_with_ht:
     hash_table_exit(clone);
 out_with_metadata:
     dm_clone_metadata_close(clone->cmd);
 out_with_source_dev:
     dm_put_device(ti, clone->source_dev);
 out_with_dest_dev:
     dm_put_device(ti, clone->dest_dev);
 out_with_meta_dev:
     dm_put_device(ti, clone->metadata_dev);
 out_with_clone:
     kfree(clone);
 
     return r;
 }
 
 static void clone_dtr(struct dm_target *ti)
 {
     unsigned int i;
     struct clone *clone = ti->private;
 
     mutex_destroy(&clone->commit_lock);
 
     for (i = 0; i < clone->nr_ctr_args; i++)
         kfree(clone->ctr_args[i]);
     kfree(clone->ctr_args);
 
     mempool_exit(&clone->hydration_pool);
     dm_kcopyd_client_destroy(clone->kcopyd_client);
     destroy_workqueue(clone->wq);
     hash_table_exit(clone);
     dm_clone_metadata_close(clone->cmd);
     dm_put_device(ti, clone->source_dev);
     dm_put_device(ti, clone->dest_dev);
     dm_put_device(ti, clone->metadata_dev);
 
     kfree(clone);
 }
 
 /*---------------------------------------------------------------------------*/
 
 static void clone_postsuspend(struct dm_target *ti)
 {
     struct clone *clone = ti->private;
 
     /*
      * To successfully suspend the device:
      *
      *  - We cancel the delayed work for periodic commits and wait for
      *    it to finish.
      *
      *  - We stop the background hydration, i.e. we prevent new region
      *    hydrations from starting.
      *
      *  - We wait for any in-flight hydrations to finish.
      *
      *  - We flush the workqueue.
      *
      *  - We commit the metadata.
      */
     cancel_delayed_work_sync(&clone->waker);
 
     set_bit(DM_CLONE_HYDRATION_SUSPENDED, &clone->flags);
 
     /*
      * Make sure set_bit() is ordered before atomic_read(), otherwise we
      * might race with do_hydration() and miss some started region
      * hydrations.
      *
      * This is paired with smp_mb__after_atomic() in do_hydration().
      */
     smp_mb__after_atomic();
 
     wait_event(clone->hydration_stopped, !atomic_read(&clone->hydrations_in_flight));
     flush_workqueue(clone->wq);
 
     (void) commit_metadata(clone, NULL);
 }
 
 static void clone_resume(struct dm_target *ti)
 {
     struct clone *clone = ti->private;
 
     clear_bit(DM_CLONE_HYDRATION_SUSPENDED, &clone->flags);
     do_waker(&clone->waker.work);
 }
 
 /*
  * If discard_passdown was enabled verify that the destination device supports
  * discards. Disable discard_passdown if not.
  */
 static void disable_passdown_if_not_supported(struct clone *clone)
 {
     struct block_device *dest_dev = clone->dest_dev->bdev;
     struct queue_limits *dest_limits = &bdev_get_queue(dest_dev)->limits;
     const char *reason = NULL;
 
     if (!test_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags))
         return;
 
     if (!bdev_max_discard_sectors(dest_dev))
         reason = "discard unsupported";
     else if (dest_limits->max_discard_sectors < clone->region_size)
         reason = "max discard sectors smaller than a region";
 
     if (reason) {
         DMWARN("Destination device (%pd) %s: Disabling discard passdown.",
                dest_dev, reason);
         clear_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags);
     }
 }
 
 static void set_discard_limits(struct clone *clone, struct queue_limits *limits)
 {
     struct block_device *dest_bdev = clone->dest_dev->bdev;
     struct queue_limits *dest_limits = &bdev_get_queue(dest_bdev)->limits;
 
     if (!test_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags)) {
         /* No passdown is done so we set our own virtual limits */
         limits->discard_granularity = clone->region_size << SECTOR_SHIFT;
         limits->max_discard_sectors = round_down(UINT_MAX >> SECTOR_SHIFT, clone->region_size);
         return;
     }
 
     /*
      * clone_iterate_devices() is stacking both the source and destination
      * device limits but discards aren't passed to the source device, so
      * inherit destination's limits.
      */
     limits->max_discard_sectors = dest_limits->max_discard_sectors;
     limits->max_hw_discard_sectors = dest_limits->max_hw_discard_sectors;
     limits->discard_granularity = dest_limits->discard_granularity;
     limits->discard_alignment = dest_limits->discard_alignment;
     limits->discard_misaligned = dest_limits->discard_misaligned;
     limits->max_discard_segments = dest_limits->max_discard_segments;
 }
 
 static void clone_io_hints(struct dm_target *ti, struct queue_limits *limits)
 {
     struct clone *clone = ti->private;
     u64 io_opt_sectors = limits->io_opt >> SECTOR_SHIFT;
 
     /*
      * If the system-determined stacked limits are compatible with
      * dm-clone's region size (io_opt is a factor) do not override them.
      */
     if (io_opt_sectors < clone->region_size ||
         do_div(io_opt_sectors, clone->region_size)) {
         blk_limits_io_min(limits, clone->region_size << SECTOR_SHIFT);
         blk_limits_io_opt(limits, clone->region_size << SECTOR_SHIFT);
     }
 
     disable_passdown_if_not_supported(clone);
     set_discard_limits(clone, limits);
 }
 
 static int clone_iterate_devices(struct dm_target *ti,
                  iterate_devices_callout_fn fn, void *data)
 {
     int ret;
     struct clone *clone = ti->private;
     struct dm_dev *dest_dev = clone->dest_dev;
     struct dm_dev *source_dev = clone->source_dev;
 
     ret = fn(ti, source_dev, 0, ti->len, data);
     if (!ret)
         ret = fn(ti, dest_dev, 0, ti->len, data);
     return ret;
 }
 
 /*
  * dm-clone message functions.
  */
 static void set_hydration_threshold(struct clone *clone, unsigned int nr_regions)
 {
     WRITE_ONCE(clone->hydration_threshold, nr_regions);
 
     /*
      * If user space sets hydration_threshold to zero then the hydration
      * will stop. If at a later time the hydration_threshold is increased
      * we must restart the hydration process by waking up the worker.
      */
     wake_worker(clone);
 }
 
 static void set_hydration_batch_size(struct clone *clone, unsigned int nr_regions)
 {
     WRITE_ONCE(clone->hydration_batch_size, nr_regions);
 }
 
 static void enable_hydration(struct clone *clone)
 {
     if (!test_and_set_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags))
         wake_worker(clone);
 }
 
 static void disable_hydration(struct clone *clone)
 {
     clear_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags);
 }
 
 static int clone_message(struct dm_target *ti, unsigned int argc, char **argv,
              char *result, unsigned int maxlen)
 {
     struct clone *clone = ti->private;
     unsigned int value;
 
     if (!argc)
         return -EINVAL;
 
     if (!strcasecmp(argv[0], "enable_hydration")) {
         enable_hydration(clone);
         return 0;
     }
 
     if (!strcasecmp(argv[0], "disable_hydration")) {
         disable_hydration(clone);
         return 0;
     }
 
     if (argc != 2)
         return -EINVAL;
 
     if (!strcasecmp(argv[0], "hydration_threshold")) {
         if (kstrtouint(argv[1], 10, &value))
             return -EINVAL;
 
         set_hydration_threshold(clone, value);
 
         return 0;
     }
 
     if (!strcasecmp(argv[0], "hydration_batch_size")) {
         if (kstrtouint(argv[1], 10, &value))
             return -EINVAL;
 
         set_hydration_batch_size(clone, value);
 
         return 0;
     }
 
     DMERR("%s: Unsupported message `%s'", clone_device_name(clone), argv[0]);
     return -EINVAL;
 }
 
 static struct target_type clone_target = {
     .name = "clone",
     .version = {1, 0, 0},
     .module = THIS_MODULE,
     .ctr = clone_ctr,
     .dtr =  clone_dtr,
     .map = clone_map,
     .end_io = clone_endio,
     .postsuspend = clone_postsuspend,
     .resume = clone_resume,
     .status = clone_status,
     .message = clone_message,
     .io_hints = clone_io_hints,
     .iterate_devices = clone_iterate_devices,
 };
 
 /*---------------------------------------------------------------------------*/
 
 /* Module functions */
 static int __init dm_clone_init(void)
 {
     int r;
 
     _hydration_cache = KMEM_CACHE(dm_clone_region_hydration, 0);
     if (!_hydration_cache)
         return -ENOMEM;
 
     r = dm_register_target(&clone_target);
     if (r < 0) {
         DMERR("Failed to register clone target");
         return r;
     }
 
     return 0;
 }
 
 static void __exit dm_clone_exit(void)
 {
     dm_unregister_target(&clone_target);
 
     kmem_cache_destroy(_hydration_cache);
     _hydration_cache = NULL;
 }
 
 /* Module hooks */
 module_init(dm_clone_init);
 module_exit(dm_clone_exit);
 
 MODULE_DESCRIPTION(DM_NAME " clone target");
 MODULE_AUTHOR("Nikos Tsironis <ntsironis@arrikto.com>");
 MODULE_LICENSE("GPL");

This page was automatically generated by the 2.1.0 LXR engine. The LXR team