drivers/dax/kmem.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /* Copyright(c) 2016-2019 Intel Corporation. All rights reserved. */
0003 #include <linux/memremap.h>
0004 #include <linux/pagemap.h>
0005 #include <linux/memory.h>
0006 #include <linux/module.h>
0007 #include <linux/device.h>
0008 #include <linux/pfn_t.h>
0009 #include <linux/slab.h>
0010 #include <linux/dax.h>
0011 #include <linux/fs.h>
0012 #include <linux/mm.h>
0013 #include <linux/mman.h>
0014 #include "dax-private.h"
0015 #include "bus.h"
0016
0017 /* Memory resource name used for add_memory_driver_managed(). */
0018 static const char *kmem_name;
0019 /* Set if any memory will remain added when the driver will be unloaded. */
0020 static bool any_hotremove_failed;
0021
0022 static int dax_kmem_range(struct dev_dax *dev_dax, int i, struct range *r)
0023 {
0024     struct dev_dax_range *dax_range = &dev_dax->ranges[i];
0025     struct range *range = &dax_range->range;
0026
0027     /* memory-block align the hotplug range */
0028     r->start = ALIGN(range->start, memory_block_size_bytes());
0029     r->end = ALIGN_DOWN(range->end + 1, memory_block_size_bytes()) - 1;
0030     if (r->start >= r->end) {
0031         r->start = range->start;
0032         r->end = range->end;
0033         return -ENOSPC;
0034     }
0035     return 0;
0036 }
0037
0038 struct dax_kmem_data {
0039     const char *res_name;
0040     int mgid;
0041     struct resource *res[];
0042 };
0043
0044 static int dev_dax_kmem_probe(struct dev_dax *dev_dax)
0045 {
0046     struct device *dev = &dev_dax->dev;
0047     unsigned long total_len = 0;
0048     struct dax_kmem_data *data;
0049     int i, rc, mapped = 0;
0050     int numa_node;
0051
0052     /*
0053      * Ensure good NUMA information for the persistent memory.
0054      * Without this check, there is a risk that slow memory
0055      * could be mixed in a node with faster memory, causing
0056      * unavoidable performance issues.
0057      */
0058     numa_node = dev_dax->target_node;
0059     if (numa_node < 0) {
0060         dev_warn(dev, "rejecting DAX region with invalid node: %d\n",
0061                 numa_node);
0062         return -EINVAL;
0063     }
0064
0065     for (i = 0; i < dev_dax->nr_range; i++) {
0066         struct range range;
0067
0068         rc = dax_kmem_range(dev_dax, i, &range);
0069         if (rc) {
0070             dev_info(dev, "mapping%d: %#llx-%#llx too small after alignment\n",
0071                     i, range.start, range.end);
0072             continue;
0073         }
0074         total_len += range_len(&range);
0075     }
0076
0077     if (!total_len) {
0078         dev_warn(dev, "rejecting DAX region without any memory after alignment\n");
0079         return -EINVAL;
0080     }
0081
0082     data = kzalloc(struct_size(data, res, dev_dax->nr_range), GFP_KERNEL);
0083     if (!data)
0084         return -ENOMEM;
0085
0086     rc = -ENOMEM;
0087     data->res_name = kstrdup(dev_name(dev), GFP_KERNEL);
0088     if (!data->res_name)
0089         goto err_res_name;
0090
0091     rc = memory_group_register_static(numa_node, total_len);
0092     if (rc < 0)
0093         goto err_reg_mgid;
0094     data->mgid = rc;
0095
0096     for (i = 0; i < dev_dax->nr_range; i++) {
0097         struct resource *res;
0098         struct range range;
0099
0100         rc = dax_kmem_range(dev_dax, i, &range);
0101         if (rc)
0102             continue;
0103
0104         /* Region is permanently reserved if hotremove fails. */
0105         res = request_mem_region(range.start, range_len(&range), data->res_name);
0106         if (!res) {
0107             dev_warn(dev, "mapping%d: %#llx-%#llx could not reserve region\n",
0108                     i, range.start, range.end);
0109             /*
0110              * Once some memory has been onlined we can't
0111              * assume that it can be un-onlined safely.
0112              */
0113             if (mapped)
0114                 continue;
0115             rc = -EBUSY;
0116             goto err_request_mem;
0117         }
0118         data->res[i] = res;
0119
0120         /*
0121          * Set flags appropriate for System RAM.  Leave ..._BUSY clear
0122          * so that add_memory() can add a child resource.  Do not
0123          * inherit flags from the parent since it may set new flags
0124          * unknown to us that will break add_memory() below.
0125          */
0126         res->flags = IORESOURCE_SYSTEM_RAM;
0127
0128         /*
0129          * Ensure that future kexec'd kernels will not treat
0130          * this as RAM automatically.
0131          */
0132         rc = add_memory_driver_managed(data->mgid, range.start,
0133                 range_len(&range), kmem_name, MHP_NID_IS_MGID);
0134
0135         if (rc) {
0136             dev_warn(dev, "mapping%d: %#llx-%#llx memory add failed\n",
0137                     i, range.start, range.end);
0138             release_resource(res);
0139             kfree(res);
0140             data->res[i] = NULL;
0141             if (mapped)
0142                 continue;
0143             goto err_request_mem;
0144         }
0145         mapped++;
0146     }
0147
0148     dev_set_drvdata(dev, data);
0149
0150     return 0;
0151
0152 err_request_mem:
0153     memory_group_unregister(data->mgid);
0154 err_reg_mgid:
0155     kfree(data->res_name);
0156 err_res_name:
0157     kfree(data);
0158     return rc;
0159 }
0160
0161 #ifdef CONFIG_MEMORY_HOTREMOVE
0162 static void dev_dax_kmem_remove(struct dev_dax *dev_dax)
0163 {
0164     int i, success = 0;
0165     struct device *dev = &dev_dax->dev;
0166     struct dax_kmem_data *data = dev_get_drvdata(dev);
0167
0168     /*
0169      * We have one shot for removing memory, if some memory blocks were not
0170      * offline prior to calling this function remove_memory() will fail, and
0171      * there is no way to hotremove this memory until reboot because device
0172      * unbind will succeed even if we return failure.
0173      */
0174     for (i = 0; i < dev_dax->nr_range; i++) {
0175         struct range range;
0176         int rc;
0177
0178         rc = dax_kmem_range(dev_dax, i, &range);
0179         if (rc)
0180             continue;
0181
0182         rc = remove_memory(range.start, range_len(&range));
0183         if (rc == 0) {
0184             release_resource(data->res[i]);
0185             kfree(data->res[i]);
0186             data->res[i] = NULL;
0187             success++;
0188             continue;
0189         }
0190         any_hotremove_failed = true;
0191         dev_err(dev,
0192             "mapping%d: %#llx-%#llx cannot be hotremoved until the next reboot\n",
0193                 i, range.start, range.end);
0194     }
0195
0196     if (success >= dev_dax->nr_range) {
0197         memory_group_unregister(data->mgid);
0198         kfree(data->res_name);
0199         kfree(data);
0200         dev_set_drvdata(dev, NULL);
0201     }
0202 }
0203 #else
0204 static void dev_dax_kmem_remove(struct dev_dax *dev_dax)
0205 {
0206     /*
0207      * Without hotremove purposely leak the request_mem_region() for the
0208      * device-dax range and return '0' to ->remove() attempts. The removal
0209      * of the device from the driver always succeeds, but the region is
0210      * permanently pinned as reserved by the unreleased
0211      * request_mem_region().
0212      */
0213     any_hotremove_failed = true;
0214 }
0215 #endif /* CONFIG_MEMORY_HOTREMOVE */
0216
0217 static struct dax_device_driver device_dax_kmem_driver = {
0218     .probe = dev_dax_kmem_probe,
0219     .remove = dev_dax_kmem_remove,
0220 };
0221
0222 static int __init dax_kmem_init(void)
0223 {
0224     int rc;
0225
0226     /* Resource name is permanently allocated if any hotremove fails. */
0227     kmem_name = kstrdup_const("System RAM (kmem)", GFP_KERNEL);
0228     if (!kmem_name)
0229         return -ENOMEM;
0230
0231     rc = dax_driver_register(&device_dax_kmem_driver);
0232     if (rc)
0233         kfree_const(kmem_name);
0234     return rc;
0235 }
0236
0237 static void __exit dax_kmem_exit(void)
0238 {
0239     dax_driver_unregister(&device_dax_kmem_driver);
0240     if (!any_hotremove_failed)
0241         kfree_const(kmem_name);
0242 }
0243
0244 MODULE_AUTHOR("Intel Corporation");
0245 MODULE_LICENSE("GPL v2");
0246 module_init(dax_kmem_init);
0247 module_exit(dax_kmem_exit);
0248 MODULE_ALIAS_DAX_DEVICE(0);