OSCL-LXR linux-6.0.1/​drivers/​virtio/​virtio_mem.c	Source navigation Diff markup Identifier search General search
	Version: linux-6.0.1 spark-3.0.0 hadoop-3.2.0-src hadoop-3.3.0-src spark-2.4.7 linux-4.15.13 hadoop-2.7.4-src Revert   Change

 // SPDX-License-Identifier: GPL-2.0-or-later
 /*
  * Virtio-mem device driver.
  *
  * Copyright Red Hat, Inc. 2020
  *
  * Author(s): David Hildenbrand <david@redhat.com>
  */
 
 #include <linux/virtio.h>
 #include <linux/virtio_mem.h>
 #include <linux/workqueue.h>
 #include <linux/slab.h>
 #include <linux/module.h>
 #include <linux/mm.h>
 #include <linux/memory_hotplug.h>
 #include <linux/memory.h>
 #include <linux/hrtimer.h>
 #include <linux/crash_dump.h>
 #include <linux/mutex.h>
 #include <linux/bitmap.h>
 #include <linux/lockdep.h>
 #include <linux/log2.h>
 
 #include <acpi/acpi_numa.h>
 
 static bool unplug_online = true;
 module_param(unplug_online, bool, 0644);
 MODULE_PARM_DESC(unplug_online, "Try to unplug online memory");
 
 static bool force_bbm;
 module_param(force_bbm, bool, 0444);
 MODULE_PARM_DESC(force_bbm,
         "Force Big Block Mode. Default is 0 (auto-selection)");
 
 static unsigned long bbm_block_size;
 module_param(bbm_block_size, ulong, 0444);
 MODULE_PARM_DESC(bbm_block_size,
          "Big Block size in bytes. Default is 0 (auto-detection).");
 
 static bool bbm_safe_unplug = true;
 module_param(bbm_safe_unplug, bool, 0444);
 MODULE_PARM_DESC(bbm_safe_unplug,
          "Use a safe unplug mechanism in BBM, avoiding long/endless loops");
 
 /*
  * virtio-mem currently supports the following modes of operation:
  *
  * * Sub Block Mode (SBM): A Linux memory block spans 2..X subblocks (SB). The
  *   size of a Sub Block (SB) is determined based on the device block size, the
  *   pageblock size, and the maximum allocation granularity of the buddy.
  *   Subblocks within a Linux memory block might either be plugged or unplugged.
  *   Memory is added/removed to Linux MM in Linux memory block granularity.
  *
  * * Big Block Mode (BBM): A Big Block (BB) spans 1..X Linux memory blocks.
  *   Memory is added/removed to Linux MM in Big Block granularity.
  *
  * The mode is determined automatically based on the Linux memory block size
  * and the device block size.
  *
  * User space / core MM (auto onlining) is responsible for onlining added
  * Linux memory blocks - and for selecting a zone. Linux Memory Blocks are
  * always onlined separately, and all memory within a Linux memory block is
  * onlined to the same zone - virtio-mem relies on this behavior.
  */
 
 /*
  * State of a Linux memory block in SBM.
  */
 enum virtio_mem_sbm_mb_state {
     /* Unplugged, not added to Linux. Can be reused later. */
     VIRTIO_MEM_SBM_MB_UNUSED = 0,
     /* (Partially) plugged, not added to Linux. Error on add_memory(). */
     VIRTIO_MEM_SBM_MB_PLUGGED,
     /* Fully plugged, fully added to Linux, offline. */
     VIRTIO_MEM_SBM_MB_OFFLINE,
     /* Partially plugged, fully added to Linux, offline. */
     VIRTIO_MEM_SBM_MB_OFFLINE_PARTIAL,
     /* Fully plugged, fully added to Linux, onlined to a kernel zone. */
     VIRTIO_MEM_SBM_MB_KERNEL,
     /* Partially plugged, fully added to Linux, online to a kernel zone */
     VIRTIO_MEM_SBM_MB_KERNEL_PARTIAL,
     /* Fully plugged, fully added to Linux, onlined to ZONE_MOVABLE. */
     VIRTIO_MEM_SBM_MB_MOVABLE,
     /* Partially plugged, fully added to Linux, onlined to ZONE_MOVABLE. */
     VIRTIO_MEM_SBM_MB_MOVABLE_PARTIAL,
     VIRTIO_MEM_SBM_MB_COUNT
 };
 
 /*
  * State of a Big Block (BB) in BBM, covering 1..X Linux memory blocks.
  */
 enum virtio_mem_bbm_bb_state {
     /* Unplugged, not added to Linux. Can be reused later. */
     VIRTIO_MEM_BBM_BB_UNUSED = 0,
     /* Plugged, not added to Linux. Error on add_memory(). */
     VIRTIO_MEM_BBM_BB_PLUGGED,
     /* Plugged and added to Linux. */
     VIRTIO_MEM_BBM_BB_ADDED,
     /* All online parts are fake-offline, ready to remove. */
     VIRTIO_MEM_BBM_BB_FAKE_OFFLINE,
     VIRTIO_MEM_BBM_BB_COUNT
 };
 
 struct virtio_mem {
     struct virtio_device *vdev;
 
     /* We might first have to unplug all memory when starting up. */
     bool unplug_all_required;
 
     /* Workqueue that processes the plug/unplug requests. */
     struct work_struct wq;
     atomic_t wq_active;
     atomic_t config_changed;
 
     /* Virtqueue for guest->host requests. */
     struct virtqueue *vq;
 
     /* Wait for a host response to a guest request. */
     wait_queue_head_t host_resp;
 
     /* Space for one guest request and the host response. */
     struct virtio_mem_req req;
     struct virtio_mem_resp resp;
 
     /* The current size of the device. */
     uint64_t plugged_size;
     /* The requested size of the device. */
     uint64_t requested_size;
 
     /* The device block size (for communicating with the device). */
     uint64_t device_block_size;
     /* The determined node id for all memory of the device. */
     int nid;
     /* Physical start address of the memory region. */
     uint64_t addr;
     /* Maximum region size in bytes. */
     uint64_t region_size;
 
     /* The parent resource for all memory added via this device. */
     struct resource *parent_resource;
     /*
      * Copy of "System RAM (virtio_mem)" to be used for
      * add_memory_driver_managed().
      */
     const char *resource_name;
     /* Memory group identification. */
     int mgid;
 
     /*
      * We don't want to add too much memory if it's not getting onlined,
      * to avoid running OOM. Besides this threshold, we allow to have at
      * least two offline blocks at a time (whatever is bigger).
      */
 #define VIRTIO_MEM_DEFAULT_OFFLINE_THRESHOLD        (1024 * 1024 * 1024)
     atomic64_t offline_size;
     uint64_t offline_threshold;
 
     /* If set, the driver is in SBM, otherwise in BBM. */
     bool in_sbm;
 
     union {
         struct {
             /* Id of the first memory block of this device. */
             unsigned long first_mb_id;
             /* Id of the last usable memory block of this device. */
             unsigned long last_usable_mb_id;
             /* Id of the next memory bock to prepare when needed. */
             unsigned long next_mb_id;
 
             /* The subblock size. */
             uint64_t sb_size;
             /* The number of subblocks per Linux memory block. */
             uint32_t sbs_per_mb;
 
             /* Summary of all memory block states. */
             unsigned long mb_count[VIRTIO_MEM_SBM_MB_COUNT];
 
             /*
              * One byte state per memory block. Allocated via
              * vmalloc(). Resized (alloc+copy+free) on demand.
              *
              * With 128 MiB memory blocks, we have states for 512
              * GiB of memory in one 4 KiB page.
              */
             uint8_t *mb_states;
 
             /*
              * Bitmap: one bit per subblock. Allocated similar to
              * sbm.mb_states.
              *
              * A set bit means the corresponding subblock is
              * plugged, otherwise it's unblocked.
              *
              * With 4 MiB subblocks, we manage 128 GiB of memory
              * in one 4 KiB page.
              */
             unsigned long *sb_states;
         } sbm;
 
         struct {
             /* Id of the first big block of this device. */
             unsigned long first_bb_id;
             /* Id of the last usable big block of this device. */
             unsigned long last_usable_bb_id;
             /* Id of the next device bock to prepare when needed. */
             unsigned long next_bb_id;
 
             /* Summary of all big block states. */
             unsigned long bb_count[VIRTIO_MEM_BBM_BB_COUNT];
 
             /* One byte state per big block. See sbm.mb_states. */
             uint8_t *bb_states;
 
             /* The block size used for plugging/adding/removing. */
             uint64_t bb_size;
         } bbm;
     };
 
     /*
      * Mutex that protects the sbm.mb_count, sbm.mb_states,
      * sbm.sb_states, bbm.bb_count, and bbm.bb_states
      *
      * When this lock is held the pointers can't change, ONLINE and
      * OFFLINE blocks can't change the state and no subblocks will get
      * plugged/unplugged.
      *
      * In kdump mode, used to serialize requests, last_block_addr and
      * last_block_plugged.
      */
     struct mutex hotplug_mutex;
     bool hotplug_active;
 
     /* An error occurred we cannot handle - stop processing requests. */
     bool broken;
 
     /* Cached valued of is_kdump_kernel() when the device was probed. */
     bool in_kdump;
 
     /* The driver is being removed. */
     spinlock_t removal_lock;
     bool removing;
 
     /* Timer for retrying to plug/unplug memory. */
     struct hrtimer retry_timer;
     unsigned int retry_timer_ms;
 #define VIRTIO_MEM_RETRY_TIMER_MIN_MS       50000
 #define VIRTIO_MEM_RETRY_TIMER_MAX_MS       300000
 
     /* Memory notifier (online/offline events). */
     struct notifier_block memory_notifier;
 
 #ifdef CONFIG_PROC_VMCORE
     /* vmcore callback for /proc/vmcore handling in kdump mode */
     struct vmcore_cb vmcore_cb;
     uint64_t last_block_addr;
     bool last_block_plugged;
 #endif /* CONFIG_PROC_VMCORE */
 
     /* Next device in the list of virtio-mem devices. */
     struct list_head next;
 };
 
 /*
  * We have to share a single online_page callback among all virtio-mem
  * devices. We use RCU to iterate the list in the callback.
  */
 static DEFINE_MUTEX(virtio_mem_mutex);
 static LIST_HEAD(virtio_mem_devices);
 
 static void virtio_mem_online_page_cb(struct page *page, unsigned int order);
 static void virtio_mem_fake_offline_going_offline(unsigned long pfn,
                           unsigned long nr_pages);
 static void virtio_mem_fake_offline_cancel_offline(unsigned long pfn,
                            unsigned long nr_pages);
 static void virtio_mem_retry(struct virtio_mem *vm);
 static int virtio_mem_create_resource(struct virtio_mem *vm);
 static void virtio_mem_delete_resource(struct virtio_mem *vm);
 
 /*
  * Register a virtio-mem device so it will be considered for the online_page
  * callback.
  */
 static int register_virtio_mem_device(struct virtio_mem *vm)
 {
     int rc = 0;
 
     /* First device registers the callback. */
     mutex_lock(&virtio_mem_mutex);
     if (list_empty(&virtio_mem_devices))
         rc = set_online_page_callback(&virtio_mem_online_page_cb);
     if (!rc)
         list_add_rcu(&vm->next, &virtio_mem_devices);
     mutex_unlock(&virtio_mem_mutex);
 
     return rc;
 }
 
 /*
  * Unregister a virtio-mem device so it will no longer be considered for the
  * online_page callback.
  */
 static void unregister_virtio_mem_device(struct virtio_mem *vm)
 {
     /* Last device unregisters the callback. */
     mutex_lock(&virtio_mem_mutex);
     list_del_rcu(&vm->next);
     if (list_empty(&virtio_mem_devices))
         restore_online_page_callback(&virtio_mem_online_page_cb);
     mutex_unlock(&virtio_mem_mutex);
 
     synchronize_rcu();
 }
 
 /*
  * Calculate the memory block id of a given address.
  */
 static unsigned long virtio_mem_phys_to_mb_id(unsigned long addr)
 {
     return addr / memory_block_size_bytes();
 }
 
 /*
  * Calculate the physical start address of a given memory block id.
  */
 static unsigned long virtio_mem_mb_id_to_phys(unsigned long mb_id)
 {
     return mb_id * memory_block_size_bytes();
 }
 
 /*
  * Calculate the big block id of a given address.
  */
 static unsigned long virtio_mem_phys_to_bb_id(struct virtio_mem *vm,
                           uint64_t addr)
 {
     return addr / vm->bbm.bb_size;
 }
 
 /*
  * Calculate the physical start address of a given big block id.
  */
 static uint64_t virtio_mem_bb_id_to_phys(struct virtio_mem *vm,
                      unsigned long bb_id)
 {
     return bb_id * vm->bbm.bb_size;
 }
 
 /*
  * Calculate the subblock id of a given address.
  */
 static unsigned long virtio_mem_phys_to_sb_id(struct virtio_mem *vm,
                           unsigned long addr)
 {
     const unsigned long mb_id = virtio_mem_phys_to_mb_id(addr);
     const unsigned long mb_addr = virtio_mem_mb_id_to_phys(mb_id);
 
     return (addr - mb_addr) / vm->sbm.sb_size;
 }
 
 /*
  * Set the state of a big block, taking care of the state counter.
  */
 static void virtio_mem_bbm_set_bb_state(struct virtio_mem *vm,
                     unsigned long bb_id,
                     enum virtio_mem_bbm_bb_state state)
 {
     const unsigned long idx = bb_id - vm->bbm.first_bb_id;
     enum virtio_mem_bbm_bb_state old_state;
 
     old_state = vm->bbm.bb_states[idx];
     vm->bbm.bb_states[idx] = state;
 
     BUG_ON(vm->bbm.bb_count[old_state] == 0);
     vm->bbm.bb_count[old_state]--;
     vm->bbm.bb_count[state]++;
 }
 
 /*
  * Get the state of a big block.
  */
 static enum virtio_mem_bbm_bb_state virtio_mem_bbm_get_bb_state(struct virtio_mem *vm,
                                 unsigned long bb_id)
 {
     return vm->bbm.bb_states[bb_id - vm->bbm.first_bb_id];
 }
 
 /*
  * Prepare the big block state array for the next big block.
  */
 static int virtio_mem_bbm_bb_states_prepare_next_bb(struct virtio_mem *vm)
 {
     unsigned long old_bytes = vm->bbm.next_bb_id - vm->bbm.first_bb_id;
     unsigned long new_bytes = old_bytes + 1;
     int old_pages = PFN_UP(old_bytes);
     int new_pages = PFN_UP(new_bytes);
     uint8_t *new_array;
 
     if (vm->bbm.bb_states && old_pages == new_pages)
         return 0;
 
     new_array = vzalloc(new_pages * PAGE_SIZE);
     if (!new_array)
         return -ENOMEM;
 
     mutex_lock(&vm->hotplug_mutex);
     if (vm->bbm.bb_states)
         memcpy(new_array, vm->bbm.bb_states, old_pages * PAGE_SIZE);
     vfree(vm->bbm.bb_states);
     vm->bbm.bb_states = new_array;
     mutex_unlock(&vm->hotplug_mutex);
 
     return 0;
 }
 
 #define virtio_mem_bbm_for_each_bb(_vm, _bb_id, _state) \
     for (_bb_id = vm->bbm.first_bb_id; \
          _bb_id < vm->bbm.next_bb_id && _vm->bbm.bb_count[_state]; \
          _bb_id++) \
         if (virtio_mem_bbm_get_bb_state(_vm, _bb_id) == _state)
 
 #define virtio_mem_bbm_for_each_bb_rev(_vm, _bb_id, _state) \
     for (_bb_id = vm->bbm.next_bb_id - 1; \
          _bb_id >= vm->bbm.first_bb_id && _vm->bbm.bb_count[_state]; \
          _bb_id--) \
         if (virtio_mem_bbm_get_bb_state(_vm, _bb_id) == _state)
 
 /*
  * Set the state of a memory block, taking care of the state counter.
  */
 static void virtio_mem_sbm_set_mb_state(struct virtio_mem *vm,
                     unsigned long mb_id, uint8_t state)
 {
     const unsigned long idx = mb_id - vm->sbm.first_mb_id;
     uint8_t old_state;
 
     old_state = vm->sbm.mb_states[idx];
     vm->sbm.mb_states[idx] = state;
 
     BUG_ON(vm->sbm.mb_count[old_state] == 0);
     vm->sbm.mb_count[old_state]--;
     vm->sbm.mb_count[state]++;
 }
 
 /*
  * Get the state of a memory block.
  */
 static uint8_t virtio_mem_sbm_get_mb_state(struct virtio_mem *vm,
                        unsigned long mb_id)
 {
     const unsigned long idx = mb_id - vm->sbm.first_mb_id;
 
     return vm->sbm.mb_states[idx];
 }
 
 /*
  * Prepare the state array for the next memory block.
  */
 static int virtio_mem_sbm_mb_states_prepare_next_mb(struct virtio_mem *vm)
 {
     int old_pages = PFN_UP(vm->sbm.next_mb_id - vm->sbm.first_mb_id);
     int new_pages = PFN_UP(vm->sbm.next_mb_id - vm->sbm.first_mb_id + 1);
     uint8_t *new_array;
 
     if (vm->sbm.mb_states && old_pages == new_pages)
         return 0;
 
     new_array = vzalloc(new_pages * PAGE_SIZE);
     if (!new_array)
         return -ENOMEM;
 
     mutex_lock(&vm->hotplug_mutex);
     if (vm->sbm.mb_states)
         memcpy(new_array, vm->sbm.mb_states, old_pages * PAGE_SIZE);
     vfree(vm->sbm.mb_states);
     vm->sbm.mb_states = new_array;
     mutex_unlock(&vm->hotplug_mutex);
 
     return 0;
 }
 
 #define virtio_mem_sbm_for_each_mb(_vm, _mb_id, _state) \
     for (_mb_id = _vm->sbm.first_mb_id; \
          _mb_id < _vm->sbm.next_mb_id && _vm->sbm.mb_count[_state]; \
          _mb_id++) \
         if (virtio_mem_sbm_get_mb_state(_vm, _mb_id) == _state)
 
 #define virtio_mem_sbm_for_each_mb_rev(_vm, _mb_id, _state) \
     for (_mb_id = _vm->sbm.next_mb_id - 1; \
          _mb_id >= _vm->sbm.first_mb_id && _vm->sbm.mb_count[_state]; \
          _mb_id--) \
         if (virtio_mem_sbm_get_mb_state(_vm, _mb_id) == _state)
 
 /*
  * Calculate the bit number in the subblock bitmap for the given subblock
  * inside the given memory block.
  */
 static int virtio_mem_sbm_sb_state_bit_nr(struct virtio_mem *vm,
                       unsigned long mb_id, int sb_id)
 {
     return (mb_id - vm->sbm.first_mb_id) * vm->sbm.sbs_per_mb + sb_id;
 }
 
 /*
  * Mark all selected subblocks plugged.
  *
  * Will not modify the state of the memory block.
  */
 static void virtio_mem_sbm_set_sb_plugged(struct virtio_mem *vm,
                       unsigned long mb_id, int sb_id,
                       int count)
 {
     const int bit = virtio_mem_sbm_sb_state_bit_nr(vm, mb_id, sb_id);
 
     __bitmap_set(vm->sbm.sb_states, bit, count);
 }
 
 /*
  * Mark all selected subblocks unplugged.
  *
  * Will not modify the state of the memory block.
  */
 static void virtio_mem_sbm_set_sb_unplugged(struct virtio_mem *vm,
                         unsigned long mb_id, int sb_id,
                         int count)
 {
     const int bit = virtio_mem_sbm_sb_state_bit_nr(vm, mb_id, sb_id);
 
     __bitmap_clear(vm->sbm.sb_states, bit, count);
 }
 
 /*
  * Test if all selected subblocks are plugged.
  */
 static bool virtio_mem_sbm_test_sb_plugged(struct virtio_mem *vm,
                        unsigned long mb_id, int sb_id,
                        int count)
 {
     const int bit = virtio_mem_sbm_sb_state_bit_nr(vm, mb_id, sb_id);
 
     if (count == 1)
         return test_bit(bit, vm->sbm.sb_states);
 
     /* TODO: Helper similar to bitmap_set() */
     return find_next_zero_bit(vm->sbm.sb_states, bit + count, bit) >=
            bit + count;
 }
 
 /*
  * Test if all selected subblocks are unplugged.
  */
 static bool virtio_mem_sbm_test_sb_unplugged(struct virtio_mem *vm,
                          unsigned long mb_id, int sb_id,
                          int count)
 {
     const int bit = virtio_mem_sbm_sb_state_bit_nr(vm, mb_id, sb_id);
 
     /* TODO: Helper similar to bitmap_set() */
     return find_next_bit(vm->sbm.sb_states, bit + count, bit) >=
            bit + count;
 }
 
 /*
  * Find the first unplugged subblock. Returns vm->sbm.sbs_per_mb in case there is
  * none.
  */
 static int virtio_mem_sbm_first_unplugged_sb(struct virtio_mem *vm,
                         unsigned long mb_id)
 {
     const int bit = virtio_mem_sbm_sb_state_bit_nr(vm, mb_id, 0);
 
     return find_next_zero_bit(vm->sbm.sb_states,
                   bit + vm->sbm.sbs_per_mb, bit) - bit;
 }
 
 /*
  * Prepare the subblock bitmap for the next memory block.
  */
 static int virtio_mem_sbm_sb_states_prepare_next_mb(struct virtio_mem *vm)
 {
     const unsigned long old_nb_mb = vm->sbm.next_mb_id - vm->sbm.first_mb_id;
     const unsigned long old_nb_bits = old_nb_mb * vm->sbm.sbs_per_mb;
     const unsigned long new_nb_bits = (old_nb_mb + 1) * vm->sbm.sbs_per_mb;
     int old_pages = PFN_UP(BITS_TO_LONGS(old_nb_bits) * sizeof(long));
     int new_pages = PFN_UP(BITS_TO_LONGS(new_nb_bits) * sizeof(long));
     unsigned long *new_bitmap, *old_bitmap;
 
     if (vm->sbm.sb_states && old_pages == new_pages)
         return 0;
 
     new_bitmap = vzalloc(new_pages * PAGE_SIZE);
     if (!new_bitmap)
         return -ENOMEM;
 
     mutex_lock(&vm->hotplug_mutex);
     if (vm->sbm.sb_states)
         memcpy(new_bitmap, vm->sbm.sb_states, old_pages * PAGE_SIZE);
 
     old_bitmap = vm->sbm.sb_states;
     vm->sbm.sb_states = new_bitmap;
     mutex_unlock(&vm->hotplug_mutex);
 
     vfree(old_bitmap);
     return 0;
 }
 
 /*
  * Test if we could add memory without creating too much offline memory -
  * to avoid running OOM if memory is getting onlined deferred.
  */
 static bool virtio_mem_could_add_memory(struct virtio_mem *vm, uint64_t size)
 {
     if (WARN_ON_ONCE(size > vm->offline_threshold))
         return false;
 
     return atomic64_read(&vm->offline_size) + size <= vm->offline_threshold;
 }
 
 /*
  * Try adding memory to Linux. Will usually only fail if out of memory.
  *
  * Must not be called with the vm->hotplug_mutex held (possible deadlock with
  * onlining code).
  *
  * Will not modify the state of memory blocks in virtio-mem.
  */
 static int virtio_mem_add_memory(struct virtio_mem *vm, uint64_t addr,
                  uint64_t size)
 {
     int rc;
 
     /*
      * When force-unloading the driver and we still have memory added to
      * Linux, the resource name has to stay.
      */
     if (!vm->resource_name) {
         vm->resource_name = kstrdup_const("System RAM (virtio_mem)",
                           GFP_KERNEL);
         if (!vm->resource_name)
             return -ENOMEM;
     }
 
     dev_dbg(&vm->vdev->dev, "adding memory: 0x%llx - 0x%llx\n", addr,
         addr + size - 1);
     /* Memory might get onlined immediately. */
     atomic64_add(size, &vm->offline_size);
     rc = add_memory_driver_managed(vm->mgid, addr, size, vm->resource_name,
                        MHP_MERGE_RESOURCE | MHP_NID_IS_MGID);
     if (rc) {
         atomic64_sub(size, &vm->offline_size);
         dev_warn(&vm->vdev->dev, "adding memory failed: %d\n", rc);
         /*
          * TODO: Linux MM does not properly clean up yet in all cases
          * where adding of memory failed - especially on -ENOMEM.
          */
     }
     return rc;
 }
 
 /*
  * See virtio_mem_add_memory(): Try adding a single Linux memory block.
  */
 static int virtio_mem_sbm_add_mb(struct virtio_mem *vm, unsigned long mb_id)
 {
     const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id);
     const uint64_t size = memory_block_size_bytes();
 
     return virtio_mem_add_memory(vm, addr, size);
 }
 
 /*
  * See virtio_mem_add_memory(): Try adding a big block.
  */
 static int virtio_mem_bbm_add_bb(struct virtio_mem *vm, unsigned long bb_id)
 {
     const uint64_t addr = virtio_mem_bb_id_to_phys(vm, bb_id);
     const uint64_t size = vm->bbm.bb_size;
 
     return virtio_mem_add_memory(vm, addr, size);
 }
 
 /*
  * Try removing memory from Linux. Will only fail if memory blocks aren't
  * offline.
  *
  * Must not be called with the vm->hotplug_mutex held (possible deadlock with
  * onlining code).
  *
  * Will not modify the state of memory blocks in virtio-mem.
  */
 static int virtio_mem_remove_memory(struct virtio_mem *vm, uint64_t addr,
                     uint64_t size)
 {
     int rc;
 
     dev_dbg(&vm->vdev->dev, "removing memory: 0x%llx - 0x%llx\n", addr,
         addr + size - 1);
     rc = remove_memory(addr, size);
     if (!rc) {
         atomic64_sub(size, &vm->offline_size);
         /*
          * We might have freed up memory we can now unplug, retry
          * immediately instead of waiting.
          */
         virtio_mem_retry(vm);
     } else {
         dev_dbg(&vm->vdev->dev, "removing memory failed: %d\n", rc);
     }
     return rc;
 }
 
 /*
  * See virtio_mem_remove_memory(): Try removing a single Linux memory block.
  */
 static int virtio_mem_sbm_remove_mb(struct virtio_mem *vm, unsigned long mb_id)
 {
     const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id);
     const uint64_t size = memory_block_size_bytes();
 
     return virtio_mem_remove_memory(vm, addr, size);
 }
 
 /*
  * Try offlining and removing memory from Linux.
  *
  * Must not be called with the vm->hotplug_mutex held (possible deadlock with
  * onlining code).
  *
  * Will not modify the state of memory blocks in virtio-mem.
  */
 static int virtio_mem_offline_and_remove_memory(struct virtio_mem *vm,
                         uint64_t addr,
                         uint64_t size)
 {
     int rc;
 
     dev_dbg(&vm->vdev->dev,
         "offlining and removing memory: 0x%llx - 0x%llx\n", addr,
         addr + size - 1);
 
     rc = offline_and_remove_memory(addr, size);
     if (!rc) {
         atomic64_sub(size, &vm->offline_size);
         /*
          * We might have freed up memory we can now unplug, retry
          * immediately instead of waiting.
          */
         virtio_mem_retry(vm);
     } else {
         dev_dbg(&vm->vdev->dev,
             "offlining and removing memory failed: %d\n", rc);
     }
     return rc;
 }
 
 /*
  * See virtio_mem_offline_and_remove_memory(): Try offlining and removing
  * a single Linux memory block.
  */
 static int virtio_mem_sbm_offline_and_remove_mb(struct virtio_mem *vm,
                         unsigned long mb_id)
 {
     const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id);
     const uint64_t size = memory_block_size_bytes();
 
     return virtio_mem_offline_and_remove_memory(vm, addr, size);
 }
 
 /*
  * See virtio_mem_offline_and_remove_memory(): Try to offline and remove a
  * all Linux memory blocks covered by the big block.
  */
 static int virtio_mem_bbm_offline_and_remove_bb(struct virtio_mem *vm,
                         unsigned long bb_id)
 {
     const uint64_t addr = virtio_mem_bb_id_to_phys(vm, bb_id);
     const uint64_t size = vm->bbm.bb_size;
 
     return virtio_mem_offline_and_remove_memory(vm, addr, size);
 }
 
 /*
  * Trigger the workqueue so the device can perform its magic.
  */
 static void virtio_mem_retry(struct virtio_mem *vm)
 {
     unsigned long flags;
 
     spin_lock_irqsave(&vm->removal_lock, flags);
     if (!vm->removing)
         queue_work(system_freezable_wq, &vm->wq);
     spin_unlock_irqrestore(&vm->removal_lock, flags);
 }
 
 static int virtio_mem_translate_node_id(struct virtio_mem *vm, uint16_t node_id)
 {
     int node = NUMA_NO_NODE;
 
 #if defined(CONFIG_ACPI_NUMA)
     if (virtio_has_feature(vm->vdev, VIRTIO_MEM_F_ACPI_PXM))
         node = pxm_to_node(node_id);
 #endif
     return node;
 }
 
 /*
  * Test if a virtio-mem device overlaps with the given range. Can be called
  * from (notifier) callbacks lockless.
  */
 static bool virtio_mem_overlaps_range(struct virtio_mem *vm, uint64_t start,
                       uint64_t size)
 {
     return start < vm->addr + vm->region_size && vm->addr < start + size;
 }
 
 /*
  * Test if a virtio-mem device contains a given range. Can be called from
  * (notifier) callbacks lockless.
  */
 static bool virtio_mem_contains_range(struct virtio_mem *vm, uint64_t start,
                       uint64_t size)
 {
     return start >= vm->addr && start + size <= vm->addr + vm->region_size;
 }
 
 static int virtio_mem_sbm_notify_going_online(struct virtio_mem *vm,
                           unsigned long mb_id)
 {
     switch (virtio_mem_sbm_get_mb_state(vm, mb_id)) {
     case VIRTIO_MEM_SBM_MB_OFFLINE_PARTIAL:
     case VIRTIO_MEM_SBM_MB_OFFLINE:
         return NOTIFY_OK;
     default:
         break;
     }
     dev_warn_ratelimited(&vm->vdev->dev,
                  "memory block onlining denied\n");
     return NOTIFY_BAD;
 }
 
 static void virtio_mem_sbm_notify_offline(struct virtio_mem *vm,
                       unsigned long mb_id)
 {
     switch (virtio_mem_sbm_get_mb_state(vm, mb_id)) {
     case VIRTIO_MEM_SBM_MB_KERNEL_PARTIAL:
     case VIRTIO_MEM_SBM_MB_MOVABLE_PARTIAL:
         virtio_mem_sbm_set_mb_state(vm, mb_id,
                         VIRTIO_MEM_SBM_MB_OFFLINE_PARTIAL);
         break;
     case VIRTIO_MEM_SBM_MB_KERNEL:
     case VIRTIO_MEM_SBM_MB_MOVABLE:
         virtio_mem_sbm_set_mb_state(vm, mb_id,
                         VIRTIO_MEM_SBM_MB_OFFLINE);
         break;
     default:
         BUG();
         break;
     }
 }
 
 static void virtio_mem_sbm_notify_online(struct virtio_mem *vm,
                      unsigned long mb_id,
                      unsigned long start_pfn)
 {
     const bool is_movable = is_zone_movable_page(pfn_to_page(start_pfn));
     int new_state;
 
     switch (virtio_mem_sbm_get_mb_state(vm, mb_id)) {
     case VIRTIO_MEM_SBM_MB_OFFLINE_PARTIAL:
         new_state = VIRTIO_MEM_SBM_MB_KERNEL_PARTIAL;
         if (is_movable)
             new_state = VIRTIO_MEM_SBM_MB_MOVABLE_PARTIAL;
         break;
     case VIRTIO_MEM_SBM_MB_OFFLINE:
         new_state = VIRTIO_MEM_SBM_MB_KERNEL;
         if (is_movable)
             new_state = VIRTIO_MEM_SBM_MB_MOVABLE;
         break;
     default:
         BUG();
         break;
     }
     virtio_mem_sbm_set_mb_state(vm, mb_id, new_state);
 }
 
 static void virtio_mem_sbm_notify_going_offline(struct virtio_mem *vm,
                         unsigned long mb_id)
 {
     const unsigned long nr_pages = PFN_DOWN(vm->sbm.sb_size);
     unsigned long pfn;
     int sb_id;
 
     for (sb_id = 0; sb_id < vm->sbm.sbs_per_mb; sb_id++) {
         if (virtio_mem_sbm_test_sb_plugged(vm, mb_id, sb_id, 1))
             continue;
         pfn = PFN_DOWN(virtio_mem_mb_id_to_phys(mb_id) +
                    sb_id * vm->sbm.sb_size);
         virtio_mem_fake_offline_going_offline(pfn, nr_pages);
     }
 }
 
 static void virtio_mem_sbm_notify_cancel_offline(struct virtio_mem *vm,
                          unsigned long mb_id)
 {
     const unsigned long nr_pages = PFN_DOWN(vm->sbm.sb_size);
     unsigned long pfn;
     int sb_id;
 
     for (sb_id = 0; sb_id < vm->sbm.sbs_per_mb; sb_id++) {
         if (virtio_mem_sbm_test_sb_plugged(vm, mb_id, sb_id, 1))
             continue;
         pfn = PFN_DOWN(virtio_mem_mb_id_to_phys(mb_id) +
                    sb_id * vm->sbm.sb_size);
         virtio_mem_fake_offline_cancel_offline(pfn, nr_pages);
     }
 }
 
 static void virtio_mem_bbm_notify_going_offline(struct virtio_mem *vm,
                         unsigned long bb_id,
                         unsigned long pfn,
                         unsigned long nr_pages)
 {
     /*
      * When marked as "fake-offline", all online memory of this device block
      * is allocated by us. Otherwise, we don't have any memory allocated.
      */
     if (virtio_mem_bbm_get_bb_state(vm, bb_id) !=
         VIRTIO_MEM_BBM_BB_FAKE_OFFLINE)
         return;
     virtio_mem_fake_offline_going_offline(pfn, nr_pages);
 }
 
 static void virtio_mem_bbm_notify_cancel_offline(struct virtio_mem *vm,
                          unsigned long bb_id,
                          unsigned long pfn,
                          unsigned long nr_pages)
 {
     if (virtio_mem_bbm_get_bb_state(vm, bb_id) !=
         VIRTIO_MEM_BBM_BB_FAKE_OFFLINE)
         return;
     virtio_mem_fake_offline_cancel_offline(pfn, nr_pages);
 }
 
 /*
  * This callback will either be called synchronously from add_memory() or
  * asynchronously (e.g., triggered via user space). We have to be careful
  * with locking when calling add_memory().
  */
 static int virtio_mem_memory_notifier_cb(struct notifier_block *nb,
                      unsigned long action, void *arg)
 {
     struct virtio_mem *vm = container_of(nb, struct virtio_mem,
                          memory_notifier);
     struct memory_notify *mhp = arg;
     const unsigned long start = PFN_PHYS(mhp->start_pfn);
     const unsigned long size = PFN_PHYS(mhp->nr_pages);
     int rc = NOTIFY_OK;
     unsigned long id;
 
     if (!virtio_mem_overlaps_range(vm, start, size))
         return NOTIFY_DONE;
 
     if (vm->in_sbm) {
         id = virtio_mem_phys_to_mb_id(start);
         /*
          * In SBM, we add memory in separate memory blocks - we expect
          * it to be onlined/offlined in the same granularity. Bail out
          * if this ever changes.
          */
         if (WARN_ON_ONCE(size != memory_block_size_bytes() ||
                  !IS_ALIGNED(start, memory_block_size_bytes())))
             return NOTIFY_BAD;
     } else {
         id = virtio_mem_phys_to_bb_id(vm, start);
         /*
          * In BBM, we only care about onlining/offlining happening
          * within a single big block, we don't care about the
          * actual granularity as we don't track individual Linux
          * memory blocks.
          */
         if (WARN_ON_ONCE(id != virtio_mem_phys_to_bb_id(vm, start + size - 1)))
             return NOTIFY_BAD;
     }
 
     /*
      * Avoid circular locking lockdep warnings. We lock the mutex
      * e.g., in MEM_GOING_ONLINE and unlock it in MEM_ONLINE. The
      * blocking_notifier_call_chain() has it's own lock, which gets unlocked
      * between both notifier calls and will bail out. False positive.
      */
     lockdep_off();
 
     switch (action) {
     case MEM_GOING_OFFLINE:
         mutex_lock(&vm->hotplug_mutex);
         if (vm->removing) {
             rc = notifier_from_errno(-EBUSY);
             mutex_unlock(&vm->hotplug_mutex);
             break;
         }
         vm->hotplug_active = true;
         if (vm->in_sbm)
             virtio_mem_sbm_notify_going_offline(vm, id);
         else
             virtio_mem_bbm_notify_going_offline(vm, id,
                                 mhp->start_pfn,
                                 mhp->nr_pages);
         break;
     case MEM_GOING_ONLINE:
         mutex_lock(&vm->hotplug_mutex);
         if (vm->removing) {
             rc = notifier_from_errno(-EBUSY);
             mutex_unlock(&vm->hotplug_mutex);
             break;
         }
         vm->hotplug_active = true;
         if (vm->in_sbm)
             rc = virtio_mem_sbm_notify_going_online(vm, id);
         break;
     case MEM_OFFLINE:
         if (vm->in_sbm)
             virtio_mem_sbm_notify_offline(vm, id);
 
         atomic64_add(size, &vm->offline_size);
         /*
          * Trigger the workqueue. Now that we have some offline memory,
          * maybe we can handle pending unplug requests.
          */
         if (!unplug_online)
             virtio_mem_retry(vm);
 
         vm->hotplug_active = false;
         mutex_unlock(&vm->hotplug_mutex);
         break;
     case MEM_ONLINE:
         if (vm->in_sbm)
             virtio_mem_sbm_notify_online(vm, id, mhp->start_pfn);
 
         atomic64_sub(size, &vm->offline_size);
         /*
          * Start adding more memory once we onlined half of our
          * threshold. Don't trigger if it's possibly due to our actipn
          * (e.g., us adding memory which gets onlined immediately from
          * the core).
          */
         if (!atomic_read(&vm->wq_active) &&
             virtio_mem_could_add_memory(vm, vm->offline_threshold / 2))
             virtio_mem_retry(vm);
 
         vm->hotplug_active = false;
         mutex_unlock(&vm->hotplug_mutex);
         break;
     case MEM_CANCEL_OFFLINE:
         if (!vm->hotplug_active)
             break;
         if (vm->in_sbm)
             virtio_mem_sbm_notify_cancel_offline(vm, id);
         else
             virtio_mem_bbm_notify_cancel_offline(vm, id,
                                  mhp->start_pfn,
                                  mhp->nr_pages);
         vm->hotplug_active = false;
         mutex_unlock(&vm->hotplug_mutex);
         break;
     case MEM_CANCEL_ONLINE:
         if (!vm->hotplug_active)
             break;
         vm->hotplug_active = false;
         mutex_unlock(&vm->hotplug_mutex);
         break;
     default:
         break;
     }
 
     lockdep_on();
 
     return rc;
 }
 
 /*
  * Set a range of pages PG_offline. Remember pages that were never onlined
  * (via generic_online_page()) using PageDirty().
  */
 static void virtio_mem_set_fake_offline(unsigned long pfn,
                     unsigned long nr_pages, bool onlined)
 {
     page_offline_begin();
     for (; nr_pages--; pfn++) {
         struct page *page = pfn_to_page(pfn);
 
         __SetPageOffline(page);
         if (!onlined) {
             SetPageDirty(page);
             /* FIXME: remove after cleanups */
             ClearPageReserved(page);
         }
     }
     page_offline_end();
 }
 
 /*
  * Clear PG_offline from a range of pages. If the pages were never onlined,
  * (via generic_online_page()), clear PageDirty().
  */
 static void virtio_mem_clear_fake_offline(unsigned long pfn,
                       unsigned long nr_pages, bool onlined)
 {
     for (; nr_pages--; pfn++) {
         struct page *page = pfn_to_page(pfn);
 
         __ClearPageOffline(page);
         if (!onlined)
             ClearPageDirty(page);
     }
 }
 
 /*
  * Release a range of fake-offline pages to the buddy, effectively
  * fake-onlining them.
  */
 static void virtio_mem_fake_online(unsigned long pfn, unsigned long nr_pages)
 {
     unsigned long order = MAX_ORDER - 1;
     unsigned long i;
 
     /*
      * We might get called for ranges that don't cover properly aligned
      * MAX_ORDER - 1 pages; however, we can only online properly aligned
      * pages with an order of MAX_ORDER - 1 at maximum.
      */
     while (!IS_ALIGNED(pfn | nr_pages, 1 << order))
         order--;
 
     for (i = 0; i < nr_pages; i += 1 << order) {
         struct page *page = pfn_to_page(pfn + i);
 
         /*
          * If the page is PageDirty(), it was kept fake-offline when
          * onlining the memory block. Otherwise, it was allocated
          * using alloc_contig_range(). All pages in a subblock are
          * alike.
          */
         if (PageDirty(page)) {
             virtio_mem_clear_fake_offline(pfn + i, 1 << order, false);
             generic_online_page(page, order);
         } else {
             virtio_mem_clear_fake_offline(pfn + i, 1 << order, true);
             free_contig_range(pfn + i, 1 << order);
             adjust_managed_page_count(page, 1 << order);
         }
     }
 }
 
 /*
  * Try to allocate a range, marking pages fake-offline, effectively
  * fake-offlining them.
  */
 static int virtio_mem_fake_offline(unsigned long pfn, unsigned long nr_pages)
 {
     const bool is_movable = is_zone_movable_page(pfn_to_page(pfn));
     int rc, retry_count;
 
     /*
      * TODO: We want an alloc_contig_range() mode that tries to allocate
      * harder (e.g., dealing with temporarily pinned pages, PCP), especially
      * with ZONE_MOVABLE. So for now, retry a couple of times with
      * ZONE_MOVABLE before giving up - because that zone is supposed to give
      * some guarantees.
      */
     for (retry_count = 0; retry_count < 5; retry_count++) {
         rc = alloc_contig_range(pfn, pfn + nr_pages, MIGRATE_MOVABLE,
                     GFP_KERNEL);
         if (rc == -ENOMEM)
             /* whoops, out of memory */
             return rc;
         else if (rc && !is_movable)
             break;
         else if (rc)
             continue;
 
         virtio_mem_set_fake_offline(pfn, nr_pages, true);
         adjust_managed_page_count(pfn_to_page(pfn), -nr_pages);
         return 0;
     }
 
     return -EBUSY;
 }
 
 /*
  * Handle fake-offline pages when memory is going offline - such that the
  * pages can be skipped by mm-core when offlining.
  */
 static void virtio_mem_fake_offline_going_offline(unsigned long pfn,
                           unsigned long nr_pages)
 {
     struct page *page;
     unsigned long i;
 
     /*
      * Drop our reference to the pages so the memory can get offlined
      * and add the unplugged pages to the managed page counters (so
      * offlining code can correctly subtract them again).
      */
     adjust_managed_page_count(pfn_to_page(pfn), nr_pages);
     /* Drop our reference to the pages so the memory can get offlined. */
     for (i = 0; i < nr_pages; i++) {
         page = pfn_to_page(pfn + i);
         if (WARN_ON(!page_ref_dec_and_test(page)))
             dump_page(page, "fake-offline page referenced");
     }
 }
 
 /*
  * Handle fake-offline pages when memory offlining is canceled - to undo
  * what we did in virtio_mem_fake_offline_going_offline().
  */
 static void virtio_mem_fake_offline_cancel_offline(unsigned long pfn,
                            unsigned long nr_pages)
 {
     unsigned long i;
 
     /*
      * Get the reference we dropped when going offline and subtract the
      * unplugged pages from the managed page counters.
      */
     adjust_managed_page_count(pfn_to_page(pfn), -nr_pages);
     for (i = 0; i < nr_pages; i++)
         page_ref_inc(pfn_to_page(pfn + i));
 }
 
 static void virtio_mem_online_page(struct virtio_mem *vm,
                    struct page *page, unsigned int order)
 {
     const unsigned long start = page_to_phys(page);
     const unsigned long end = start + PFN_PHYS(1 << order);
     unsigned long addr, next, id, sb_id, count;
     bool do_online;
 
     /*
      * We can get called with any order up to MAX_ORDER - 1. If our
      * subblock size is smaller than that and we have a mixture of plugged
      * and unplugged subblocks within such a page, we have to process in
      * smaller granularity. In that case we'll adjust the order exactly once
      * within the loop.
      */
     for (addr = start; addr < end; ) {
         next = addr + PFN_PHYS(1 << order);
 
         if (vm->in_sbm) {
             id = virtio_mem_phys_to_mb_id(addr);
             sb_id = virtio_mem_phys_to_sb_id(vm, addr);
             count = virtio_mem_phys_to_sb_id(vm, next - 1) - sb_id + 1;
 
             if (virtio_mem_sbm_test_sb_plugged(vm, id, sb_id, count)) {
                 /* Fully plugged. */
                 do_online = true;
             } else if (count == 1 ||
                    virtio_mem_sbm_test_sb_unplugged(vm, id, sb_id, count)) {
                 /* Fully unplugged. */
                 do_online = false;
             } else {
                 /*
                  * Mixture, process sub-blocks instead. This
                  * will be at least the size of a pageblock.
                  * We'll run into this case exactly once.
                  */
                 order = ilog2(vm->sbm.sb_size) - PAGE_SHIFT;
                 do_online = virtio_mem_sbm_test_sb_plugged(vm, id, sb_id, 1);
                 continue;
             }
         } else {
             /*
              * If the whole block is marked fake offline, keep
              * everything that way.
              */
             id = virtio_mem_phys_to_bb_id(vm, addr);
             do_online = virtio_mem_bbm_get_bb_state(vm, id) !=
                     VIRTIO_MEM_BBM_BB_FAKE_OFFLINE;
         }
 
         if (do_online)
             generic_online_page(pfn_to_page(PFN_DOWN(addr)), order);
         else
             virtio_mem_set_fake_offline(PFN_DOWN(addr), 1 << order,
                             false);
         addr = next;
     }
 }
 
 static void virtio_mem_online_page_cb(struct page *page, unsigned int order)
 {
     const unsigned long addr = page_to_phys(page);
     struct virtio_mem *vm;
 
     rcu_read_lock();
     list_for_each_entry_rcu(vm, &virtio_mem_devices, next) {
         /*
          * Pages we're onlining will never cross memory blocks and,
          * therefore, not virtio-mem devices.
          */
         if (!virtio_mem_contains_range(vm, addr, PFN_PHYS(1 << order)))
             continue;
 
         /*
          * virtio_mem_set_fake_offline() might sleep. We can safely
          * drop the RCU lock at this point because the device
          * cannot go away. See virtio_mem_remove() how races
          * between memory onlining and device removal are handled.
          */
         rcu_read_unlock();
 
         virtio_mem_online_page(vm, page, order);
         return;
     }
     rcu_read_unlock();
 
     /* not virtio-mem memory, but e.g., a DIMM. online it */
     generic_online_page(page, order);
 }
 
 static uint64_t virtio_mem_send_request(struct virtio_mem *vm,
                     const struct virtio_mem_req *req)
 {
     struct scatterlist *sgs[2], sg_req, sg_resp;
     unsigned int len;
     int rc;
 
     /* don't use the request residing on the stack (vaddr) */
     vm->req = *req;
 
     /* out: buffer for request */
     sg_init_one(&sg_req, &vm->req, sizeof(vm->req));
     sgs[0] = &sg_req;
 
     /* in: buffer for response */
     sg_init_one(&sg_resp, &vm->resp, sizeof(vm->resp));
     sgs[1] = &sg_resp;
 
     rc = virtqueue_add_sgs(vm->vq, sgs, 1, 1, vm, GFP_KERNEL);
     if (rc < 0)
         return rc;
 
     virtqueue_kick(vm->vq);
 
     /* wait for a response */
     wait_event(vm->host_resp, virtqueue_get_buf(vm->vq, &len));
 
     return virtio16_to_cpu(vm->vdev, vm->resp.type);
 }
 
 static int virtio_mem_send_plug_request(struct virtio_mem *vm, uint64_t addr,
                     uint64_t size)
 {
     const uint64_t nb_vm_blocks = size / vm->device_block_size;
     const struct virtio_mem_req req = {
         .type = cpu_to_virtio16(vm->vdev, VIRTIO_MEM_REQ_PLUG),
         .u.plug.addr = cpu_to_virtio64(vm->vdev, addr),
         .u.plug.nb_blocks = cpu_to_virtio16(vm->vdev, nb_vm_blocks),
     };
     int rc = -ENOMEM;
 
     if (atomic_read(&vm->config_changed))
         return -EAGAIN;
 
     dev_dbg(&vm->vdev->dev, "plugging memory: 0x%llx - 0x%llx\n", addr,
         addr + size - 1);
 
     switch (virtio_mem_send_request(vm, &req)) {
     case VIRTIO_MEM_RESP_ACK:
         vm->plugged_size += size;
         return 0;
     case VIRTIO_MEM_RESP_NACK:
         rc = -EAGAIN;
         break;
     case VIRTIO_MEM_RESP_BUSY:
         rc = -ETXTBSY;
         break;
     case VIRTIO_MEM_RESP_ERROR:
         rc = -EINVAL;
         break;
     default:
         break;
     }
 
     dev_dbg(&vm->vdev->dev, "plugging memory failed: %d\n", rc);
     return rc;
 }
 
 static int virtio_mem_send_unplug_request(struct virtio_mem *vm, uint64_t addr,
                       uint64_t size)
 {
     const uint64_t nb_vm_blocks = size / vm->device_block_size;
     const struct virtio_mem_req req = {
         .type = cpu_to_virtio16(vm->vdev, VIRTIO_MEM_REQ_UNPLUG),
         .u.unplug.addr = cpu_to_virtio64(vm->vdev, addr),
         .u.unplug.nb_blocks = cpu_to_virtio16(vm->vdev, nb_vm_blocks),
     };
     int rc = -ENOMEM;
 
     if (atomic_read(&vm->config_changed))
         return -EAGAIN;
 
     dev_dbg(&vm->vdev->dev, "unplugging memory: 0x%llx - 0x%llx\n", addr,
         addr + size - 1);
 
     switch (virtio_mem_send_request(vm, &req)) {
     case VIRTIO_MEM_RESP_ACK:
         vm->plugged_size -= size;
         return 0;
     case VIRTIO_MEM_RESP_BUSY:
         rc = -ETXTBSY;
         break;
     case VIRTIO_MEM_RESP_ERROR:
         rc = -EINVAL;
         break;
     default:
         break;
     }
 
     dev_dbg(&vm->vdev->dev, "unplugging memory failed: %d\n", rc);
     return rc;
 }
 
 static int virtio_mem_send_unplug_all_request(struct virtio_mem *vm)
 {
     const struct virtio_mem_req req = {
         .type = cpu_to_virtio16(vm->vdev, VIRTIO_MEM_REQ_UNPLUG_ALL),
     };
     int rc = -ENOMEM;
 
     dev_dbg(&vm->vdev->dev, "unplugging all memory");
 
     switch (virtio_mem_send_request(vm, &req)) {
     case VIRTIO_MEM_RESP_ACK:
         vm->unplug_all_required = false;
         vm->plugged_size = 0;
         /* usable region might have shrunk */
         atomic_set(&vm->config_changed, 1);
         return 0;
     case VIRTIO_MEM_RESP_BUSY:
         rc = -ETXTBSY;
         break;
     default:
         break;
     }
 
     dev_dbg(&vm->vdev->dev, "unplugging all memory failed: %d\n", rc);
     return rc;
 }
 
 /*
  * Plug selected subblocks. Updates the plugged state, but not the state
  * of the memory block.
  */
 static int virtio_mem_sbm_plug_sb(struct virtio_mem *vm, unsigned long mb_id,
                   int sb_id, int count)
 {
     const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id) +
                   sb_id * vm->sbm.sb_size;
     const uint64_t size = count * vm->sbm.sb_size;
     int rc;
 
     rc = virtio_mem_send_plug_request(vm, addr, size);
     if (!rc)
         virtio_mem_sbm_set_sb_plugged(vm, mb_id, sb_id, count);
     return rc;
 }
 
 /*
  * Unplug selected subblocks. Updates the plugged state, but not the state
  * of the memory block.
  */
 static int virtio_mem_sbm_unplug_sb(struct virtio_mem *vm, unsigned long mb_id,
                     int sb_id, int count)
 {
     const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id) +
                   sb_id * vm->sbm.sb_size;
     const uint64_t size = count * vm->sbm.sb_size;
     int rc;
 
     rc = virtio_mem_send_unplug_request(vm, addr, size);
     if (!rc)
         virtio_mem_sbm_set_sb_unplugged(vm, mb_id, sb_id, count);
     return rc;
 }
 
 /*
  * Request to unplug a big block.
  *
  * Will not modify the state of the big block.
  */
 static int virtio_mem_bbm_unplug_bb(struct virtio_mem *vm, unsigned long bb_id)
 {
     const uint64_t addr = virtio_mem_bb_id_to_phys(vm, bb_id);
     const uint64_t size = vm->bbm.bb_size;
 
     return virtio_mem_send_unplug_request(vm, addr, size);
 }
 
 /*
  * Request to plug a big block.
  *
  * Will not modify the state of the big block.
  */
 static int virtio_mem_bbm_plug_bb(struct virtio_mem *vm, unsigned long bb_id)
 {
     const uint64_t addr = virtio_mem_bb_id_to_phys(vm, bb_id);
     const uint64_t size = vm->bbm.bb_size;
 
     return virtio_mem_send_plug_request(vm, addr, size);
 }
 
 /*
  * Unplug the desired number of plugged subblocks of a offline or not-added
  * memory block. Will fail if any subblock cannot get unplugged (instead of
  * skipping it).
  *
  * Will not modify the state of the memory block.
  *
  * Note: can fail after some subblocks were unplugged.
  */
 static int virtio_mem_sbm_unplug_any_sb_raw(struct virtio_mem *vm,
                         unsigned long mb_id, uint64_t *nb_sb)
 {
     int sb_id, count;
     int rc;
 
     sb_id = vm->sbm.sbs_per_mb - 1;
     while (*nb_sb) {
         /* Find the next candidate subblock */
         while (sb_id >= 0 &&
                virtio_mem_sbm_test_sb_unplugged(vm, mb_id, sb_id, 1))
             sb_id--;
         if (sb_id < 0)
             break;
         /* Try to unplug multiple subblocks at a time */
         count = 1;
         while (count < *nb_sb && sb_id > 0 &&
                virtio_mem_sbm_test_sb_plugged(vm, mb_id, sb_id - 1, 1)) {
             count++;
             sb_id--;
         }
 
         rc = virtio_mem_sbm_unplug_sb(vm, mb_id, sb_id, count);
         if (rc)
             return rc;
         *nb_sb -= count;
         sb_id--;
     }
 
     return 0;
 }
 
 /*
  * Unplug all plugged subblocks of an offline or not-added memory block.
  *
  * Will not modify the state of the memory block.
  *
  * Note: can fail after some subblocks were unplugged.
  */
 static int virtio_mem_sbm_unplug_mb(struct virtio_mem *vm, unsigned long mb_id)
 {
     uint64_t nb_sb = vm->sbm.sbs_per_mb;
 
     return virtio_mem_sbm_unplug_any_sb_raw(vm, mb_id, &nb_sb);
 }
 
 /*
  * Prepare tracking data for the next memory block.
  */
 static int virtio_mem_sbm_prepare_next_mb(struct virtio_mem *vm,
                       unsigned long *mb_id)
 {
     int rc;
 
     if (vm->sbm.next_mb_id > vm->sbm.last_usable_mb_id)
         return -ENOSPC;
 
     /* Resize the state array if required. */
     rc = virtio_mem_sbm_mb_states_prepare_next_mb(vm);
     if (rc)
         return rc;
 
     /* Resize the subblock bitmap if required. */
     rc = virtio_mem_sbm_sb_states_prepare_next_mb(vm);
     if (rc)
         return rc;
 
     vm->sbm.mb_count[VIRTIO_MEM_SBM_MB_UNUSED]++;
     *mb_id = vm->sbm.next_mb_id++;
     return 0;
 }
 
 /*
  * Try to plug the desired number of subblocks and add the memory block
  * to Linux.
  *
  * Will modify the state of the memory block.
  */
 static int virtio_mem_sbm_plug_and_add_mb(struct virtio_mem *vm,
                       unsigned long mb_id, uint64_t *nb_sb)
 {
     const int count = min_t(int, *nb_sb, vm->sbm.sbs_per_mb);
     int rc;
 
     if (WARN_ON_ONCE(!count))
         return -EINVAL;
 
     /*
      * Plug the requested number of subblocks before adding it to linux,
      * so that onlining will directly online all plugged subblocks.
      */
     rc = virtio_mem_sbm_plug_sb(vm, mb_id, 0, count);
     if (rc)
         return rc;
 
     /*
      * Mark the block properly offline before adding it to Linux,
      * so the memory notifiers will find the block in the right state.
      */
     if (count == vm->sbm.sbs_per_mb)
         virtio_mem_sbm_set_mb_state(vm, mb_id,
                         VIRTIO_MEM_SBM_MB_OFFLINE);
     else
         virtio_mem_sbm_set_mb_state(vm, mb_id,
                         VIRTIO_MEM_SBM_MB_OFFLINE_PARTIAL);
 
     /* Add the memory block to linux - if that fails, try to unplug. */
     rc = virtio_mem_sbm_add_mb(vm, mb_id);
     if (rc) {
         int new_state = VIRTIO_MEM_SBM_MB_UNUSED;
 
         if (virtio_mem_sbm_unplug_sb(vm, mb_id, 0, count))
             new_state = VIRTIO_MEM_SBM_MB_PLUGGED;
         virtio_mem_sbm_set_mb_state(vm, mb_id, new_state);
         return rc;
     }
 
     *nb_sb -= count;
     return 0;
 }
 
 /*
  * Try to plug the desired number of subblocks of a memory block that
  * is already added to Linux.
  *
  * Will modify the state of the memory block.
  *
  * Note: Can fail after some subblocks were successfully plugged.
  */
 static int virtio_mem_sbm_plug_any_sb(struct virtio_mem *vm,
                       unsigned long mb_id, uint64_t *nb_sb)
 {
     const int old_state = virtio_mem_sbm_get_mb_state(vm, mb_id);
     unsigned long pfn, nr_pages;
     int sb_id, count;
     int rc;
 
     if (WARN_ON_ONCE(!*nb_sb))
         return -EINVAL;
 
     while (*nb_sb) {
         sb_id = virtio_mem_sbm_first_unplugged_sb(vm, mb_id);
         if (sb_id >= vm->sbm.sbs_per_mb)
             break;
         count = 1;
         while (count < *nb_sb &&
                sb_id + count < vm->sbm.sbs_per_mb &&
                !virtio_mem_sbm_test_sb_plugged(vm, mb_id, sb_id + count, 1))
             count++;
 
         rc = virtio_mem_sbm_plug_sb(vm, mb_id, sb_id, count);
         if (rc)
             return rc;
         *nb_sb -= count;
         if (old_state == VIRTIO_MEM_SBM_MB_OFFLINE_PARTIAL)
             continue;
 
         /* fake-online the pages if the memory block is online */
         pfn = PFN_DOWN(virtio_mem_mb_id_to_phys(mb_id) +
                    sb_id * vm->sbm.sb_size);
         nr_pages = PFN_DOWN(count * vm->sbm.sb_size);
         virtio_mem_fake_online(pfn, nr_pages);
     }
 
     if (virtio_mem_sbm_test_sb_plugged(vm, mb_id, 0, vm->sbm.sbs_per_mb))
         virtio_mem_sbm_set_mb_state(vm, mb_id, old_state - 1);
 
     return 0;
 }
 
 static int virtio_mem_sbm_plug_request(struct virtio_mem *vm, uint64_t diff)
 {
     const int mb_states[] = {
         VIRTIO_MEM_SBM_MB_KERNEL_PARTIAL,
         VIRTIO_MEM_SBM_MB_MOVABLE_PARTIAL,
         VIRTIO_MEM_SBM_MB_OFFLINE_PARTIAL,
     };
     uint64_t nb_sb = diff / vm->sbm.sb_size;
     unsigned long mb_id;
     int rc, i;
 
     if (!nb_sb)
         return 0;
 
     /* Don't race with onlining/offlining */
     mutex_lock(&vm->hotplug_mutex);
 
     for (i = 0; i < ARRAY_SIZE(mb_states); i++) {
         virtio_mem_sbm_for_each_mb(vm, mb_id, mb_states[i]) {
             rc = virtio_mem_sbm_plug_any_sb(vm, mb_id, &nb_sb);
             if (rc || !nb_sb)
                 goto out_unlock;
             cond_resched();
         }
     }
 
     /*
      * We won't be working on online/offline memory blocks from this point,
      * so we can't race with memory onlining/offlining. Drop the mutex.
      */
     mutex_unlock(&vm->hotplug_mutex);
 
     /* Try to plug and add unused blocks */
     virtio_mem_sbm_for_each_mb(vm, mb_id, VIRTIO_MEM_SBM_MB_UNUSED) {
         if (!virtio_mem_could_add_memory(vm, memory_block_size_bytes()))
             return -ENOSPC;
 
         rc = virtio_mem_sbm_plug_and_add_mb(vm, mb_id, &nb_sb);
         if (rc || !nb_sb)
             return rc;
         cond_resched();
     }
 
     /* Try to prepare, plug and add new blocks */
     while (nb_sb) {
         if (!virtio_mem_could_add_memory(vm, memory_block_size_bytes()))
             return -ENOSPC;
 
         rc = virtio_mem_sbm_prepare_next_mb(vm, &mb_id);
         if (rc)
             return rc;
         rc = virtio_mem_sbm_plug_and_add_mb(vm, mb_id, &nb_sb);
         if (rc)
             return rc;
         cond_resched();
     }
 
     return 0;
 out_unlock:
     mutex_unlock(&vm->hotplug_mutex);
     return rc;
 }
 
 /*
  * Plug a big block and add it to Linux.
  *
  * Will modify the state of the big block.
  */
 static int virtio_mem_bbm_plug_and_add_bb(struct virtio_mem *vm,
                       unsigned long bb_id)
 {
     int rc;
 
     if (WARN_ON_ONCE(virtio_mem_bbm_get_bb_state(vm, bb_id) !=
              VIRTIO_MEM_BBM_BB_UNUSED))
         return -EINVAL;
 
     rc = virtio_mem_bbm_plug_bb(vm, bb_id);
     if (rc)
         return rc;
     virtio_mem_bbm_set_bb_state(vm, bb_id, VIRTIO_MEM_BBM_BB_ADDED);
 
     rc = virtio_mem_bbm_add_bb(vm, bb_id);
     if (rc) {
         if (!virtio_mem_bbm_unplug_bb(vm, bb_id))
             virtio_mem_bbm_set_bb_state(vm, bb_id,
                             VIRTIO_MEM_BBM_BB_UNUSED);
         else
             /* Retry from the main loop. */
             virtio_mem_bbm_set_bb_state(vm, bb_id,
                             VIRTIO_MEM_BBM_BB_PLUGGED);
         return rc;
     }
     return 0;
 }
 
 /*
  * Prepare tracking data for the next big block.
  */
 static int virtio_mem_bbm_prepare_next_bb(struct virtio_mem *vm,
                       unsigned long *bb_id)
 {
     int rc;
 
     if (vm->bbm.next_bb_id > vm->bbm.last_usable_bb_id)
         return -ENOSPC;
 
     /* Resize the big block state array if required. */
     rc = virtio_mem_bbm_bb_states_prepare_next_bb(vm);
     if (rc)
         return rc;
 
     vm->bbm.bb_count[VIRTIO_MEM_BBM_BB_UNUSED]++;
     *bb_id = vm->bbm.next_bb_id;
     vm->bbm.next_bb_id++;
     return 0;
 }
 
 static int virtio_mem_bbm_plug_request(struct virtio_mem *vm, uint64_t diff)
 {
     uint64_t nb_bb = diff / vm->bbm.bb_size;
     unsigned long bb_id;
     int rc;
 
     if (!nb_bb)
         return 0;
 
     /* Try to plug and add unused big blocks */
     virtio_mem_bbm_for_each_bb(vm, bb_id, VIRTIO_MEM_BBM_BB_UNUSED) {
         if (!virtio_mem_could_add_memory(vm, vm->bbm.bb_size))
             return -ENOSPC;
 
         rc = virtio_mem_bbm_plug_and_add_bb(vm, bb_id);
         if (!rc)
             nb_bb--;
         if (rc || !nb_bb)
             return rc;
         cond_resched();
     }
 
     /* Try to prepare, plug and add new big blocks */
     while (nb_bb) {
         if (!virtio_mem_could_add_memory(vm, vm->bbm.bb_size))
             return -ENOSPC;
 
         rc = virtio_mem_bbm_prepare_next_bb(vm, &bb_id);
         if (rc)
             return rc;
         rc = virtio_mem_bbm_plug_and_add_bb(vm, bb_id);
         if (!rc)
             nb_bb--;
         if (rc)
             return rc;
         cond_resched();
     }
 
     return 0;
 }
 
 /*
  * Try to plug the requested amount of memory.
  */
 static int virtio_mem_plug_request(struct virtio_mem *vm, uint64_t diff)
 {
     if (vm->in_sbm)
         return virtio_mem_sbm_plug_request(vm, diff);
     return virtio_mem_bbm_plug_request(vm, diff);
 }
 
 /*
  * Unplug the desired number of plugged subblocks of an offline memory block.
  * Will fail if any subblock cannot get unplugged (instead of skipping it).
  *
  * Will modify the state of the memory block. Might temporarily drop the
  * hotplug_mutex.
  *
  * Note: Can fail after some subblocks were successfully unplugged.
  */
 static int virtio_mem_sbm_unplug_any_sb_offline(struct virtio_mem *vm,
                         unsigned long mb_id,
                         uint64_t *nb_sb)
 {
     int rc;
 
     rc = virtio_mem_sbm_unplug_any_sb_raw(vm, mb_id, nb_sb);
 
     /* some subblocks might have been unplugged even on failure */
     if (!virtio_mem_sbm_test_sb_plugged(vm, mb_id, 0, vm->sbm.sbs_per_mb))
         virtio_mem_sbm_set_mb_state(vm, mb_id,
                         VIRTIO_MEM_SBM_MB_OFFLINE_PARTIAL);
     if (rc)
         return rc;
 
     if (virtio_mem_sbm_test_sb_unplugged(vm, mb_id, 0, vm->sbm.sbs_per_mb)) {
         /*
          * Remove the block from Linux - this should never fail.
          * Hinder the block from getting onlined by marking it
          * unplugged. Temporarily drop the mutex, so
          * any pending GOING_ONLINE requests can be serviced/rejected.
          */
         virtio_mem_sbm_set_mb_state(vm, mb_id,
                         VIRTIO_MEM_SBM_MB_UNUSED);
 
         mutex_unlock(&vm->hotplug_mutex);
         rc = virtio_mem_sbm_remove_mb(vm, mb_id);
         BUG_ON(rc);
         mutex_lock(&vm->hotplug_mutex);
     }
     return 0;
 }
 
 /*
  * Unplug the given plugged subblocks of an online memory block.
  *
  * Will modify the state of the memory block.
  */
 static int virtio_mem_sbm_unplug_sb_online(struct virtio_mem *vm,
                        unsigned long mb_id, int sb_id,
                        int count)
 {
     const unsigned long nr_pages = PFN_DOWN(vm->sbm.sb_size) * count;
     const int old_state = virtio_mem_sbm_get_mb_state(vm, mb_id);
     unsigned long start_pfn;
     int rc;
 
     start_pfn = PFN_DOWN(virtio_mem_mb_id_to_phys(mb_id) +
                  sb_id * vm->sbm.sb_size);
 
     rc = virtio_mem_fake_offline(start_pfn, nr_pages);
     if (rc)
         return rc;
 
     /* Try to unplug the allocated memory */
     rc = virtio_mem_sbm_unplug_sb(vm, mb_id, sb_id, count);
     if (rc) {
         /* Return the memory to the buddy. */
         virtio_mem_fake_online(start_pfn, nr_pages);
         return rc;
     }
 
     switch (old_state) {
     case VIRTIO_MEM_SBM_MB_KERNEL:
         virtio_mem_sbm_set_mb_state(vm, mb_id,
                         VIRTIO_MEM_SBM_MB_KERNEL_PARTIAL);
         break;
     case VIRTIO_MEM_SBM_MB_MOVABLE:
         virtio_mem_sbm_set_mb_state(vm, mb_id,
                         VIRTIO_MEM_SBM_MB_MOVABLE_PARTIAL);
         break;
     }
 
     return 0;
 }
 
 /*
  * Unplug the desired number of plugged subblocks of an online memory block.
  * Will skip subblock that are busy.
  *
  * Will modify the state of the memory block. Might temporarily drop the
  * hotplug_mutex.
  *
  * Note: Can fail after some subblocks were successfully unplugged. Can
  *       return 0 even if subblocks were busy and could not get unplugged.
  */
 static int virtio_mem_sbm_unplug_any_sb_online(struct virtio_mem *vm,
                            unsigned long mb_id,
                            uint64_t *nb_sb)
 {
     int rc, sb_id;
 
     /* If possible, try to unplug the complete block in one shot. */
     if (*nb_sb >= vm->sbm.sbs_per_mb &&
         virtio_mem_sbm_test_sb_plugged(vm, mb_id, 0, vm->sbm.sbs_per_mb)) {
         rc = virtio_mem_sbm_unplug_sb_online(vm, mb_id, 0,
                              vm->sbm.sbs_per_mb);
         if (!rc) {
             *nb_sb -= vm->sbm.sbs_per_mb;
             goto unplugged;
         } else if (rc != -EBUSY)
             return rc;
     }
 
     /* Fallback to single subblocks. */
     for (sb_id = vm->sbm.sbs_per_mb - 1; sb_id >= 0 && *nb_sb; sb_id--) {
         /* Find the next candidate subblock */
         while (sb_id >= 0 &&
                !virtio_mem_sbm_test_sb_plugged(vm, mb_id, sb_id, 1))
             sb_id--;
         if (sb_id < 0)
             break;
 
         rc = virtio_mem_sbm_unplug_sb_online(vm, mb_id, sb_id, 1);
         if (rc == -EBUSY)
             continue;
         else if (rc)
             return rc;
         *nb_sb -= 1;
     }
 
 unplugged:
     /*
      * Once all subblocks of a memory block were unplugged, offline and
      * remove it. This will usually not fail, as no memory is in use
      * anymore - however some other notifiers might NACK the request.
      */
     if (virtio_mem_sbm_test_sb_unplugged(vm, mb_id, 0, vm->sbm.sbs_per_mb)) {
         mutex_unlock(&vm->hotplug_mutex);
         rc = virtio_mem_sbm_offline_and_remove_mb(vm, mb_id);
         mutex_lock(&vm->hotplug_mutex);
         if (!rc)
             virtio_mem_sbm_set_mb_state(vm, mb_id,
                             VIRTIO_MEM_SBM_MB_UNUSED);
     }
 
     return 0;
 }
 
 /*
  * Unplug the desired number of plugged subblocks of a memory block that is
  * already added to Linux. Will skip subblock of online memory blocks that are
  * busy (by the OS). Will fail if any subblock that's not busy cannot get
  * unplugged.
  *
  * Will modify the state of the memory block. Might temporarily drop the
  * hotplug_mutex.
  *
  * Note: Can fail after some subblocks were successfully unplugged. Can
  *       return 0 even if subblocks were busy and could not get unplugged.
  */
 static int virtio_mem_sbm_unplug_any_sb(struct virtio_mem *vm,
                     unsigned long mb_id,
                     uint64_t *nb_sb)
 {
     const int old_state = virtio_mem_sbm_get_mb_state(vm, mb_id);
 
     switch (old_state) {
     case VIRTIO_MEM_SBM_MB_KERNEL_PARTIAL:
     case VIRTIO_MEM_SBM_MB_KERNEL:
     case VIRTIO_MEM_SBM_MB_MOVABLE_PARTIAL:
     case VIRTIO_MEM_SBM_MB_MOVABLE:
         return virtio_mem_sbm_unplug_any_sb_online(vm, mb_id, nb_sb);
     case VIRTIO_MEM_SBM_MB_OFFLINE_PARTIAL:
     case VIRTIO_MEM_SBM_MB_OFFLINE:
         return virtio_mem_sbm_unplug_any_sb_offline(vm, mb_id, nb_sb);
     }
     return -EINVAL;
 }
 
 static int virtio_mem_sbm_unplug_request(struct virtio_mem *vm, uint64_t diff)
 {
     const int mb_states[] = {
         VIRTIO_MEM_SBM_MB_OFFLINE_PARTIAL,
         VIRTIO_MEM_SBM_MB_OFFLINE,
         VIRTIO_MEM_SBM_MB_MOVABLE_PARTIAL,
         VIRTIO_MEM_SBM_MB_KERNEL_PARTIAL,
         VIRTIO_MEM_SBM_MB_MOVABLE,
         VIRTIO_MEM_SBM_MB_KERNEL,
     };
     uint64_t nb_sb = diff / vm->sbm.sb_size;
     unsigned long mb_id;
     int rc, i;
 
     if (!nb_sb)
         return 0;
 
     /*
      * We'll drop the mutex a couple of times when it is safe to do so.
      * This might result in some blocks switching the state (online/offline)
      * and we could miss them in this run - we will retry again later.
      */
     mutex_lock(&vm->hotplug_mutex);
 
     /*
      * We try unplug from partially plugged blocks first, to try removing
      * whole memory blocks along with metadata. We prioritize ZONE_MOVABLE
      * as it's more reliable to unplug memory and remove whole memory
      * blocks, and we don't want to trigger a zone imbalances by
      * accidentially removing too much kernel memory.
      */
     for (i = 0; i < ARRAY_SIZE(mb_states); i++) {
         virtio_mem_sbm_for_each_mb_rev(vm, mb_id, mb_states[i]) {
             rc = virtio_mem_sbm_unplug_any_sb(vm, mb_id, &nb_sb);
             if (rc || !nb_sb)
                 goto out_unlock;
             mutex_unlock(&vm->hotplug_mutex);
             cond_resched();
             mutex_lock(&vm->hotplug_mutex);
         }
         if (!unplug_online && i == 1) {
             mutex_unlock(&vm->hotplug_mutex);
             return 0;
         }
     }
 
     mutex_unlock(&vm->hotplug_mutex);
     return nb_sb ? -EBUSY : 0;
 out_unlock:
     mutex_unlock(&vm->hotplug_mutex);
     return rc;
 }
 
 /*
  * Try to offline and remove a big block from Linux and unplug it. Will fail
  * with -EBUSY if some memory is busy and cannot get unplugged.
  *
  * Will modify the state of the memory block. Might temporarily drop the
  * hotplug_mutex.
  */
 static int virtio_mem_bbm_offline_remove_and_unplug_bb(struct virtio_mem *vm,
                                unsigned long bb_id)
 {
     const unsigned long start_pfn = PFN_DOWN(virtio_mem_bb_id_to_phys(vm, bb_id));
     const unsigned long nr_pages = PFN_DOWN(vm->bbm.bb_size);
     unsigned long end_pfn = start_pfn + nr_pages;
     unsigned long pfn;
     struct page *page;
     int rc;
 
     if (WARN_ON_ONCE(virtio_mem_bbm_get_bb_state(vm, bb_id) !=
              VIRTIO_MEM_BBM_BB_ADDED))
         return -EINVAL;
 
     if (bbm_safe_unplug) {
         /*
          * Start by fake-offlining all memory. Once we marked the device
          * block as fake-offline, all newly onlined memory will
          * automatically be kept fake-offline. Protect from concurrent
          * onlining/offlining until we have a consistent state.
          */
         mutex_lock(&vm->hotplug_mutex);
         virtio_mem_bbm_set_bb_state(vm, bb_id,
                         VIRTIO_MEM_BBM_BB_FAKE_OFFLINE);
 
         for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
             page = pfn_to_online_page(pfn);
             if (!page)
                 continue;
 
             rc = virtio_mem_fake_offline(pfn, PAGES_PER_SECTION);
             if (rc) {
                 end_pfn = pfn;
                 goto rollback_safe_unplug;
             }
         }
         mutex_unlock(&vm->hotplug_mutex);
     }
 
     rc = virtio_mem_bbm_offline_and_remove_bb(vm, bb_id);
     if (rc) {
         if (bbm_safe_unplug) {
             mutex_lock(&vm->hotplug_mutex);
             goto rollback_safe_unplug;
         }
         return rc;
     }
 
     rc = virtio_mem_bbm_unplug_bb(vm, bb_id);
     if (rc)
         virtio_mem_bbm_set_bb_state(vm, bb_id,
                         VIRTIO_MEM_BBM_BB_PLUGGED);
     else
         virtio_mem_bbm_set_bb_state(vm, bb_id,
                         VIRTIO_MEM_BBM_BB_UNUSED);
     return rc;
 
 rollback_safe_unplug:
     for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
         page = pfn_to_online_page(pfn);
         if (!page)
             continue;
         virtio_mem_fake_online(pfn, PAGES_PER_SECTION);
     }
     virtio_mem_bbm_set_bb_state(vm, bb_id, VIRTIO_MEM_BBM_BB_ADDED);
     mutex_unlock(&vm->hotplug_mutex);
     return rc;
 }
 
 /*
  * Test if a big block is completely offline.
  */
 static bool virtio_mem_bbm_bb_is_offline(struct virtio_mem *vm,
                      unsigned long bb_id)
 {
     const unsigned long start_pfn = PFN_DOWN(virtio_mem_bb_id_to_phys(vm, bb_id));
     const unsigned long nr_pages = PFN_DOWN(vm->bbm.bb_size);
     unsigned long pfn;
 
     for (pfn = start_pfn; pfn < start_pfn + nr_pages;
          pfn += PAGES_PER_SECTION) {
         if (pfn_to_online_page(pfn))
             return false;
     }
 
     return true;
 }
 
 /*
  * Test if a big block is completely onlined to ZONE_MOVABLE (or offline).
  */
 static bool virtio_mem_bbm_bb_is_movable(struct virtio_mem *vm,
                      unsigned long bb_id)
 {
     const unsigned long start_pfn = PFN_DOWN(virtio_mem_bb_id_to_phys(vm, bb_id));
     const unsigned long nr_pages = PFN_DOWN(vm->bbm.bb_size);
     struct page *page;
     unsigned long pfn;
 
     for (pfn = start_pfn; pfn < start_pfn + nr_pages;
          pfn += PAGES_PER_SECTION) {
         page = pfn_to_online_page(pfn);
         if (!page)
             continue;
         if (page_zonenum(page) != ZONE_MOVABLE)
             return false;
     }
 
     return true;
 }
 
 static int virtio_mem_bbm_unplug_request(struct virtio_mem *vm, uint64_t diff)
 {
     uint64_t nb_bb = diff / vm->bbm.bb_size;
     uint64_t bb_id;
     int rc, i;
 
     if (!nb_bb)
         return 0;
 
     /*
      * Try to unplug big blocks. Similar to SBM, start with offline
      * big blocks.
      */
     for (i = 0; i < 3; i++) {
         virtio_mem_bbm_for_each_bb_rev(vm, bb_id, VIRTIO_MEM_BBM_BB_ADDED) {
             cond_resched();
 
             /*
              * As we're holding no locks, these checks are racy,
              * but we don't care.
              */
             if (i == 0 && !virtio_mem_bbm_bb_is_offline(vm, bb_id))
                 continue;
             if (i == 1 && !virtio_mem_bbm_bb_is_movable(vm, bb_id))
                 continue;
             rc = virtio_mem_bbm_offline_remove_and_unplug_bb(vm, bb_id);
             if (rc == -EBUSY)
                 continue;
             if (!rc)
                 nb_bb--;
             if (rc || !nb_bb)
                 return rc;
         }
         if (i == 0 && !unplug_online)
             return 0;
     }
 
     return nb_bb ? -EBUSY : 0;
 }
 
 /*
  * Try to unplug the requested amount of memory.
  */
 static int virtio_mem_unplug_request(struct virtio_mem *vm, uint64_t diff)
 {
     if (vm->in_sbm)
         return virtio_mem_sbm_unplug_request(vm, diff);
     return virtio_mem_bbm_unplug_request(vm, diff);
 }
 
 /*
  * Try to unplug all blocks that couldn't be unplugged before, for example,
  * because the hypervisor was busy.
  */
 static int virtio_mem_unplug_pending_mb(struct virtio_mem *vm)
 {
     unsigned long id;
     int rc;
 
     if (!vm->in_sbm) {
         virtio_mem_bbm_for_each_bb(vm, id,
                        VIRTIO_MEM_BBM_BB_PLUGGED) {
             rc = virtio_mem_bbm_unplug_bb(vm, id);
             if (rc)
                 return rc;
             virtio_mem_bbm_set_bb_state(vm, id,
                             VIRTIO_MEM_BBM_BB_UNUSED);
         }
         return 0;
     }
 
     virtio_mem_sbm_for_each_mb(vm, id, VIRTIO_MEM_SBM_MB_PLUGGED) {
         rc = virtio_mem_sbm_unplug_mb(vm, id);
         if (rc)
             return rc;
         virtio_mem_sbm_set_mb_state(vm, id,
                         VIRTIO_MEM_SBM_MB_UNUSED);
     }
 
     return 0;
 }
 
 /*
  * Update all parts of the config that could have changed.
  */
 static void virtio_mem_refresh_config(struct virtio_mem *vm)
 {
     const struct range pluggable_range = mhp_get_pluggable_range(true);
     uint64_t new_plugged_size, usable_region_size, end_addr;
 
     /* the plugged_size is just a reflection of what _we_ did previously */
     virtio_cread_le(vm->vdev, struct virtio_mem_config, plugged_size,
             &new_plugged_size);
     if (WARN_ON_ONCE(new_plugged_size != vm->plugged_size))
         vm->plugged_size = new_plugged_size;
 
     /* calculate the last usable memory block id */
     virtio_cread_le(vm->vdev, struct virtio_mem_config,
             usable_region_size, &usable_region_size);
     end_addr = min(vm->addr + usable_region_size - 1,
                pluggable_range.end);
 
     if (vm->in_sbm) {
         vm->sbm.last_usable_mb_id = virtio_mem_phys_to_mb_id(end_addr);
         if (!IS_ALIGNED(end_addr + 1, memory_block_size_bytes()))
             vm->sbm.last_usable_mb_id--;
     } else {
         vm->bbm.last_usable_bb_id = virtio_mem_phys_to_bb_id(vm,
                                      end_addr);
         if (!IS_ALIGNED(end_addr + 1, vm->bbm.bb_size))
             vm->bbm.last_usable_bb_id--;
     }
     /*
      * If we cannot plug any of our device memory (e.g., nothing in the
      * usable region is addressable), the last usable memory block id will
      * be smaller than the first usable memory block id. We'll stop
      * attempting to add memory with -ENOSPC from our main loop.
      */
 
     /* see if there is a request to change the size */
     virtio_cread_le(vm->vdev, struct virtio_mem_config, requested_size,
             &vm->requested_size);
 
     dev_info(&vm->vdev->dev, "plugged size: 0x%llx", vm->plugged_size);
     dev_info(&vm->vdev->dev, "requested size: 0x%llx", vm->requested_size);
 }
 
 /*
  * Workqueue function for handling plug/unplug requests and config updates.
  */
 static void virtio_mem_run_wq(struct work_struct *work)
 {
     struct virtio_mem *vm = container_of(work, struct virtio_mem, wq);
     uint64_t diff;
     int rc;
 
     if (unlikely(vm->in_kdump)) {
         dev_warn_once(&vm->vdev->dev,
                  "unexpected workqueue run in kdump kernel\n");
         return;
     }
 
     hrtimer_cancel(&vm->retry_timer);
 
     if (vm->broken)
         return;
 
     atomic_set(&vm->wq_active, 1);
 retry:
     rc = 0;
 
     /* Make sure we start with a clean state if there are leftovers. */
     if (unlikely(vm->unplug_all_required))
         rc = virtio_mem_send_unplug_all_request(vm);
 
     if (atomic_read(&vm->config_changed)) {
         atomic_set(&vm->config_changed, 0);
         virtio_mem_refresh_config(vm);
     }
 
     /* Unplug any leftovers from previous runs */
     if (!rc)
         rc = virtio_mem_unplug_pending_mb(vm);
 
     if (!rc && vm->requested_size != vm->plugged_size) {
         if (vm->requested_size > vm->plugged_size) {
             diff = vm->requested_size - vm->plugged_size;
             rc = virtio_mem_plug_request(vm, diff);
         } else {
             diff = vm->plugged_size - vm->requested_size;
             rc = virtio_mem_unplug_request(vm, diff);
         }
     }
 
     switch (rc) {
     case 0:
         vm->retry_timer_ms = VIRTIO_MEM_RETRY_TIMER_MIN_MS;
         break;
     case -ENOSPC:
         /*
          * We cannot add any more memory (alignment, physical limit)
          * or we have too many offline memory blocks.
          */
         break;
     case -ETXTBSY:
         /*
          * The hypervisor cannot process our request right now
          * (e.g., out of memory, migrating);
          */
     case -EBUSY:
         /*
          * We cannot free up any memory to unplug it (all plugged memory
          * is busy).
          */
     case -ENOMEM:
         /* Out of memory, try again later. */
         hrtimer_start(&vm->retry_timer, ms_to_ktime(vm->retry_timer_ms),
                   HRTIMER_MODE_REL);
         break;
     case -EAGAIN:
         /* Retry immediately (e.g., the config changed). */
         goto retry;
     default:
         /* Unknown error, mark as broken */
         dev_err(&vm->vdev->dev,
             "unknown error, marking device broken: %d\n", rc);
         vm->broken = true;
     }
 
     atomic_set(&vm->wq_active, 0);
 }
 
 static enum hrtimer_restart virtio_mem_timer_expired(struct hrtimer *timer)
 {
     struct virtio_mem *vm = container_of(timer, struct virtio_mem,
                          retry_timer);
 
     virtio_mem_retry(vm);
     vm->retry_timer_ms = min_t(unsigned int, vm->retry_timer_ms * 2,
                    VIRTIO_MEM_RETRY_TIMER_MAX_MS);
     return HRTIMER_NORESTART;
 }
 
 static void virtio_mem_handle_response(struct virtqueue *vq)
 {
     struct virtio_mem *vm = vq->vdev->priv;
 
     wake_up(&vm->host_resp);
 }
 
 static int virtio_mem_init_vq(struct virtio_mem *vm)
 {
     struct virtqueue *vq;
 
     vq = virtio_find_single_vq(vm->vdev, virtio_mem_handle_response,
                    "guest-request");
     if (IS_ERR(vq))
         return PTR_ERR(vq);
     vm->vq = vq;
 
     return 0;
 }
 
 static int virtio_mem_init_hotplug(struct virtio_mem *vm)
 {
     const struct range pluggable_range = mhp_get_pluggable_range(true);
     uint64_t unit_pages, sb_size, addr;
     int rc;
 
     /* bad device setup - warn only */
     if (!IS_ALIGNED(vm->addr, memory_block_size_bytes()))
         dev_warn(&vm->vdev->dev,
              "The alignment of the physical start address can make some memory unusable.\n");
     if (!IS_ALIGNED(vm->addr + vm->region_size, memory_block_size_bytes()))
         dev_warn(&vm->vdev->dev,
              "The alignment of the physical end address can make some memory unusable.\n");
     if (vm->addr < pluggable_range.start ||
         vm->addr + vm->region_size - 1 > pluggable_range.end)
         dev_warn(&vm->vdev->dev,
              "Some device memory is not addressable/pluggable. This can make some memory unusable.\n");
 
     /* Prepare the offline threshold - make sure we can add two blocks. */
     vm->offline_threshold = max_t(uint64_t, 2 * memory_block_size_bytes(),
                       VIRTIO_MEM_DEFAULT_OFFLINE_THRESHOLD);
 
     /*
      * alloc_contig_range() works reliably with pageblock
      * granularity on ZONE_NORMAL, use pageblock_nr_pages.
      */
     sb_size = PAGE_SIZE * pageblock_nr_pages;
     sb_size = max_t(uint64_t, vm->device_block_size, sb_size);
 
     if (sb_size < memory_block_size_bytes() && !force_bbm) {
         /* SBM: At least two subblocks per Linux memory block. */
         vm->in_sbm = true;
         vm->sbm.sb_size = sb_size;
         vm->sbm.sbs_per_mb = memory_block_size_bytes() /
                      vm->sbm.sb_size;
 
         /* Round up to the next full memory block */
         addr = max_t(uint64_t, vm->addr, pluggable_range.start) +
                memory_block_size_bytes() - 1;
         vm->sbm.first_mb_id = virtio_mem_phys_to_mb_id(addr);
         vm->sbm.next_mb_id = vm->sbm.first_mb_id;
     } else {
         /* BBM: At least one Linux memory block. */
         vm->bbm.bb_size = max_t(uint64_t, vm->device_block_size,
                     memory_block_size_bytes());
 
         if (bbm_block_size) {
             if (!is_power_of_2(bbm_block_size)) {
                 dev_warn(&vm->vdev->dev,
                      "bbm_block_size is not a power of 2");
             } else if (bbm_block_size < vm->bbm.bb_size) {
                 dev_warn(&vm->vdev->dev,
                      "bbm_block_size is too small");
             } else {
                 vm->bbm.bb_size = bbm_block_size;
             }
         }
 
         /* Round up to the next aligned big block */
         addr = max_t(uint64_t, vm->addr, pluggable_range.start) +
                vm->bbm.bb_size - 1;
         vm->bbm.first_bb_id = virtio_mem_phys_to_bb_id(vm, addr);
         vm->bbm.next_bb_id = vm->bbm.first_bb_id;
 
         /* Make sure we can add two big blocks. */
         vm->offline_threshold = max_t(uint64_t, 2 * vm->bbm.bb_size,
                           vm->offline_threshold);
     }
 
     dev_info(&vm->vdev->dev, "memory block size: 0x%lx",
          memory_block_size_bytes());
     if (vm->in_sbm)
         dev_info(&vm->vdev->dev, "subblock size: 0x%llx",
              (unsigned long long)vm->sbm.sb_size);
     else
         dev_info(&vm->vdev->dev, "big block size: 0x%llx",
              (unsigned long long)vm->bbm.bb_size);
 
     /* create the parent resource for all memory */
     rc = virtio_mem_create_resource(vm);
     if (rc)
         return rc;
 
     /* use a single dynamic memory group to cover the whole memory device */
     if (vm->in_sbm)
         unit_pages = PHYS_PFN(memory_block_size_bytes());
     else
         unit_pages = PHYS_PFN(vm->bbm.bb_size);
     rc = memory_group_register_dynamic(vm->nid, unit_pages);
     if (rc < 0)
         goto out_del_resource;
     vm->mgid = rc;
 
     /*
      * If we still have memory plugged, we have to unplug all memory first.
      * Registering our parent resource makes sure that this memory isn't
      * actually in use (e.g., trying to reload the driver).
      */
     if (vm->plugged_size) {
         vm->unplug_all_required = true;
         dev_info(&vm->vdev->dev, "unplugging all memory is required\n");
     }
 
     /* register callbacks */
     vm->memory_notifier.notifier_call = virtio_mem_memory_notifier_cb;
     rc = register_memory_notifier(&vm->memory_notifier);
     if (rc)
         goto out_unreg_group;
     rc = register_virtio_mem_device(vm);
     if (rc)
         goto out_unreg_mem;
 
     return 0;
 out_unreg_mem:
     unregister_memory_notifier(&vm->memory_notifier);
 out_unreg_group:
     memory_group_unregister(vm->mgid);
 out_del_resource:
     virtio_mem_delete_resource(vm);
     return rc;
 }
 
 #ifdef CONFIG_PROC_VMCORE
 static int virtio_mem_send_state_request(struct virtio_mem *vm, uint64_t addr,
                      uint64_t size)
 {
     const uint64_t nb_vm_blocks = size / vm->device_block_size;
     const struct virtio_mem_req req = {
         .type = cpu_to_virtio16(vm->vdev, VIRTIO_MEM_REQ_STATE),
         .u.state.addr = cpu_to_virtio64(vm->vdev, addr),
         .u.state.nb_blocks = cpu_to_virtio16(vm->vdev, nb_vm_blocks),
     };
     int rc = -ENOMEM;
 
     dev_dbg(&vm->vdev->dev, "requesting state: 0x%llx - 0x%llx\n", addr,
         addr + size - 1);
 
     switch (virtio_mem_send_request(vm, &req)) {
     case VIRTIO_MEM_RESP_ACK:
         return virtio16_to_cpu(vm->vdev, vm->resp.u.state.state);
     case VIRTIO_MEM_RESP_ERROR:
         rc = -EINVAL;
         break;
     default:
         break;
     }
 
     dev_dbg(&vm->vdev->dev, "requesting state failed: %d\n", rc);
     return rc;
 }
 
 static bool virtio_mem_vmcore_pfn_is_ram(struct vmcore_cb *cb,
                      unsigned long pfn)
 {
     struct virtio_mem *vm = container_of(cb, struct virtio_mem,
                          vmcore_cb);
     uint64_t addr = PFN_PHYS(pfn);
     bool is_ram;
     int rc;
 
     if (!virtio_mem_contains_range(vm, addr, PAGE_SIZE))
         return true;
     if (!vm->plugged_size)
         return false;
 
     /*
      * We have to serialize device requests and access to the information
      * about the block queried last.
      */
     mutex_lock(&vm->hotplug_mutex);
 
     addr = ALIGN_DOWN(addr, vm->device_block_size);
     if (addr != vm->last_block_addr) {
         rc = virtio_mem_send_state_request(vm, addr,
                            vm->device_block_size);
         /* On any kind of error, we're going to signal !ram. */
         if (rc == VIRTIO_MEM_STATE_PLUGGED)
             vm->last_block_plugged = true;
         else
             vm->last_block_plugged = false;
         vm->last_block_addr = addr;
     }
 
     is_ram = vm->last_block_plugged;
     mutex_unlock(&vm->hotplug_mutex);
     return is_ram;
 }
 #endif /* CONFIG_PROC_VMCORE */
 
 static int virtio_mem_init_kdump(struct virtio_mem *vm)
 {
 #ifdef CONFIG_PROC_VMCORE
     dev_info(&vm->vdev->dev, "memory hot(un)plug disabled in kdump kernel\n");
     vm->vmcore_cb.pfn_is_ram = virtio_mem_vmcore_pfn_is_ram;
     register_vmcore_cb(&vm->vmcore_cb);
     return 0;
 #else /* CONFIG_PROC_VMCORE */
     dev_warn(&vm->vdev->dev, "disabled in kdump kernel without vmcore\n");
     return -EBUSY;
 #endif /* CONFIG_PROC_VMCORE */
 }
 
 static int virtio_mem_init(struct virtio_mem *vm)
 {
     uint16_t node_id;
 
     if (!vm->vdev->config->get) {
         dev_err(&vm->vdev->dev, "config access disabled\n");
         return -EINVAL;
     }
 
     /* Fetch all properties that can't change. */
     virtio_cread_le(vm->vdev, struct virtio_mem_config, plugged_size,
             &vm->plugged_size);
     virtio_cread_le(vm->vdev, struct virtio_mem_config, block_size,
             &vm->device_block_size);
     virtio_cread_le(vm->vdev, struct virtio_mem_config, node_id,
             &node_id);
     vm->nid = virtio_mem_translate_node_id(vm, node_id);
     virtio_cread_le(vm->vdev, struct virtio_mem_config, addr, &vm->addr);
     virtio_cread_le(vm->vdev, struct virtio_mem_config, region_size,
             &vm->region_size);
 
     /* Determine the nid for the device based on the lowest address. */
     if (vm->nid == NUMA_NO_NODE)
         vm->nid = memory_add_physaddr_to_nid(vm->addr);
 
     dev_info(&vm->vdev->dev, "start address: 0x%llx", vm->addr);
     dev_info(&vm->vdev->dev, "region size: 0x%llx", vm->region_size);
     dev_info(&vm->vdev->dev, "device block size: 0x%llx",
          (unsigned long long)vm->device_block_size);
     if (vm->nid != NUMA_NO_NODE && IS_ENABLED(CONFIG_NUMA))
         dev_info(&vm->vdev->dev, "nid: %d", vm->nid);
 
     /*
      * We don't want to (un)plug or reuse any memory when in kdump. The
      * memory is still accessible (but not exposed to Linux).
      */
     if (vm->in_kdump)
         return virtio_mem_init_kdump(vm);
     return virtio_mem_init_hotplug(vm);
 }
 
 static int virtio_mem_create_resource(struct virtio_mem *vm)
 {
     /*
      * When force-unloading the driver and removing the device, we
      * could have a garbage pointer. Duplicate the string.
      */
     const char *name = kstrdup(dev_name(&vm->vdev->dev), GFP_KERNEL);
 
     if (!name)
         return -ENOMEM;
 
     /* Disallow mapping device memory via /dev/mem completely. */
     vm->parent_resource = __request_mem_region(vm->addr, vm->region_size,
                            name, IORESOURCE_SYSTEM_RAM |
                            IORESOURCE_EXCLUSIVE);
     if (!vm->parent_resource) {
         kfree(name);
         dev_warn(&vm->vdev->dev, "could not reserve device region\n");
         dev_info(&vm->vdev->dev,
              "reloading the driver is not supported\n");
         return -EBUSY;
     }
 
     /* The memory is not actually busy - make add_memory() work. */
     vm->parent_resource->flags &= ~IORESOURCE_BUSY;
     return 0;
 }
 
 static void virtio_mem_delete_resource(struct virtio_mem *vm)
 {
     const char *name;
 
     if (!vm->parent_resource)
         return;
 
     name = vm->parent_resource->name;
     release_resource(vm->parent_resource);
     kfree(vm->parent_resource);
     kfree(name);
     vm->parent_resource = NULL;
 }
 
 static int virtio_mem_range_has_system_ram(struct resource *res, void *arg)
 {
     return 1;
 }
 
 static bool virtio_mem_has_memory_added(struct virtio_mem *vm)
 {
     const unsigned long flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY;
 
     return walk_iomem_res_desc(IORES_DESC_NONE, flags, vm->addr,
                    vm->addr + vm->region_size, NULL,
                    virtio_mem_range_has_system_ram) == 1;
 }
 
 static int virtio_mem_probe(struct virtio_device *vdev)
 {
     struct virtio_mem *vm;
     int rc;
 
     BUILD_BUG_ON(sizeof(struct virtio_mem_req) != 24);
     BUILD_BUG_ON(sizeof(struct virtio_mem_resp) != 10);
 
     vdev->priv = vm = kzalloc(sizeof(*vm), GFP_KERNEL);
     if (!vm)
         return -ENOMEM;
 
     init_waitqueue_head(&vm->host_resp);
     vm->vdev = vdev;
     INIT_WORK(&vm->wq, virtio_mem_run_wq);
     mutex_init(&vm->hotplug_mutex);
     INIT_LIST_HEAD(&vm->next);
     spin_lock_init(&vm->removal_lock);
     hrtimer_init(&vm->retry_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
     vm->retry_timer.function = virtio_mem_timer_expired;
     vm->retry_timer_ms = VIRTIO_MEM_RETRY_TIMER_MIN_MS;
     vm->in_kdump = is_kdump_kernel();
 
     /* register the virtqueue */
     rc = virtio_mem_init_vq(vm);
     if (rc)
         goto out_free_vm;
 
     /* initialize the device by querying the config */
     rc = virtio_mem_init(vm);
     if (rc)
         goto out_del_vq;
 
     virtio_device_ready(vdev);
 
     /* trigger a config update to start processing the requested_size */
     if (!vm->in_kdump) {
         atomic_set(&vm->config_changed, 1);
         queue_work(system_freezable_wq, &vm->wq);
     }
 
     return 0;
 out_del_vq:
     vdev->config->del_vqs(vdev);
 out_free_vm:
     kfree(vm);
     vdev->priv = NULL;
 
     return rc;
 }
 
 static void virtio_mem_deinit_hotplug(struct virtio_mem *vm)
 {
     unsigned long mb_id;
     int rc;
 
     /*
      * Make sure the workqueue won't be triggered anymore and no memory
      * blocks can be onlined/offlined until we're finished here.
      */
     mutex_lock(&vm->hotplug_mutex);
     spin_lock_irq(&vm->removal_lock);
     vm->removing = true;
     spin_unlock_irq(&vm->removal_lock);
     mutex_unlock(&vm->hotplug_mutex);
 
     /* wait until the workqueue stopped */
     cancel_work_sync(&vm->wq);
     hrtimer_cancel(&vm->retry_timer);
 
     if (vm->in_sbm) {
         /*
          * After we unregistered our callbacks, user space can online
          * partially plugged offline blocks. Make sure to remove them.
          */
         virtio_mem_sbm_for_each_mb(vm, mb_id,
                        VIRTIO_MEM_SBM_MB_OFFLINE_PARTIAL) {
             rc = virtio_mem_sbm_remove_mb(vm, mb_id);
             BUG_ON(rc);
             virtio_mem_sbm_set_mb_state(vm, mb_id,
                             VIRTIO_MEM_SBM_MB_UNUSED);
         }
         /*
          * After we unregistered our callbacks, user space can no longer
          * offline partially plugged online memory blocks. No need to
          * worry about them.
          */
     }
 
     /* unregister callbacks */
     unregister_virtio_mem_device(vm);
     unregister_memory_notifier(&vm->memory_notifier);
 
     /*
      * There is no way we could reliably remove all memory we have added to
      * the system. And there is no way to stop the driver/device from going
      * away. Warn at least.
      */
     if (virtio_mem_has_memory_added(vm)) {
         dev_warn(&vm->vdev->dev,
              "device still has system memory added\n");
     } else {
         virtio_mem_delete_resource(vm);
         kfree_const(vm->resource_name);
         memory_group_unregister(vm->mgid);
     }
 
     /* remove all tracking data - no locking needed */
     if (vm->in_sbm) {
         vfree(vm->sbm.mb_states);
         vfree(vm->sbm.sb_states);
     } else {
         vfree(vm->bbm.bb_states);
     }
 }
 
 static void virtio_mem_deinit_kdump(struct virtio_mem *vm)
 {
 #ifdef CONFIG_PROC_VMCORE
     unregister_vmcore_cb(&vm->vmcore_cb);
 #endif /* CONFIG_PROC_VMCORE */
 }
 
 static void virtio_mem_remove(struct virtio_device *vdev)
 {
     struct virtio_mem *vm = vdev->priv;
 
     if (vm->in_kdump)
         virtio_mem_deinit_kdump(vm);
     else
         virtio_mem_deinit_hotplug(vm);
 
     /* reset the device and cleanup the queues */
     virtio_reset_device(vdev);
     vdev->config->del_vqs(vdev);
 
     kfree(vm);
     vdev->priv = NULL;
 }
 
 static void virtio_mem_config_changed(struct virtio_device *vdev)
 {
     struct virtio_mem *vm = vdev->priv;
 
     if (unlikely(vm->in_kdump))
         return;
 
     atomic_set(&vm->config_changed, 1);
     virtio_mem_retry(vm);
 }
 
 #ifdef CONFIG_PM_SLEEP
 static int virtio_mem_freeze(struct virtio_device *vdev)
 {
     /*
      * When restarting the VM, all memory is usually unplugged. Don't
      * allow to suspend/hibernate.
      */
     dev_err(&vdev->dev, "save/restore not supported.\n");
     return -EPERM;
 }
 
 static int virtio_mem_restore(struct virtio_device *vdev)
 {
     return -EPERM;
 }
 #endif
 
 static unsigned int virtio_mem_features[] = {
 #if defined(CONFIG_NUMA) && defined(CONFIG_ACPI_NUMA)
     VIRTIO_MEM_F_ACPI_PXM,
 #endif
     VIRTIO_MEM_F_UNPLUGGED_INACCESSIBLE,
 };
 
 static const struct virtio_device_id virtio_mem_id_table[] = {
     { VIRTIO_ID_MEM, VIRTIO_DEV_ANY_ID },
     { 0 },
 };
 
 static struct virtio_driver virtio_mem_driver = {
     .feature_table = virtio_mem_features,
     .feature_table_size = ARRAY_SIZE(virtio_mem_features),
     .driver.name = KBUILD_MODNAME,
     .driver.owner = THIS_MODULE,
     .id_table = virtio_mem_id_table,
     .probe = virtio_mem_probe,
     .remove = virtio_mem_remove,
     .config_changed = virtio_mem_config_changed,
 #ifdef CONFIG_PM_SLEEP
     .freeze =   virtio_mem_freeze,
     .restore =  virtio_mem_restore,
 #endif
 };
 
 module_virtio_driver(virtio_mem_driver);
 MODULE_DEVICE_TABLE(virtio, virtio_mem_id_table);
 MODULE_AUTHOR("David Hildenbrand <david@redhat.com>");
 MODULE_DESCRIPTION("Virtio-mem driver");
 MODULE_LICENSE("GPL");