Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0
0002 /* Copyright(c) 2017-2018 Intel Corporation. All rights reserved. */
0003 #include <linux/memremap.h>
0004 #include <linux/device.h>
0005 #include <linux/mutex.h>
0006 #include <linux/list.h>
0007 #include <linux/slab.h>
0008 #include <linux/dax.h>
0009 #include <linux/io.h>
0010 #include "dax-private.h"
0011 #include "bus.h"
0012 
0013 static DEFINE_MUTEX(dax_bus_lock);
0014 
0015 #define DAX_NAME_LEN 30
0016 struct dax_id {
0017     struct list_head list;
0018     char dev_name[DAX_NAME_LEN];
0019 };
0020 
0021 static int dax_bus_uevent(struct device *dev, struct kobj_uevent_env *env)
0022 {
0023     /*
0024      * We only ever expect to handle device-dax instances, i.e. the
0025      * @type argument to MODULE_ALIAS_DAX_DEVICE() is always zero
0026      */
0027     return add_uevent_var(env, "MODALIAS=" DAX_DEVICE_MODALIAS_FMT, 0);
0028 }
0029 
0030 static struct dax_device_driver *to_dax_drv(struct device_driver *drv)
0031 {
0032     return container_of(drv, struct dax_device_driver, drv);
0033 }
0034 
0035 static struct dax_id *__dax_match_id(struct dax_device_driver *dax_drv,
0036         const char *dev_name)
0037 {
0038     struct dax_id *dax_id;
0039 
0040     lockdep_assert_held(&dax_bus_lock);
0041 
0042     list_for_each_entry(dax_id, &dax_drv->ids, list)
0043         if (sysfs_streq(dax_id->dev_name, dev_name))
0044             return dax_id;
0045     return NULL;
0046 }
0047 
0048 static int dax_match_id(struct dax_device_driver *dax_drv, struct device *dev)
0049 {
0050     int match;
0051 
0052     mutex_lock(&dax_bus_lock);
0053     match = !!__dax_match_id(dax_drv, dev_name(dev));
0054     mutex_unlock(&dax_bus_lock);
0055 
0056     return match;
0057 }
0058 
0059 enum id_action {
0060     ID_REMOVE,
0061     ID_ADD,
0062 };
0063 
0064 static ssize_t do_id_store(struct device_driver *drv, const char *buf,
0065         size_t count, enum id_action action)
0066 {
0067     struct dax_device_driver *dax_drv = to_dax_drv(drv);
0068     unsigned int region_id, id;
0069     char devname[DAX_NAME_LEN];
0070     struct dax_id *dax_id;
0071     ssize_t rc = count;
0072     int fields;
0073 
0074     fields = sscanf(buf, "dax%d.%d", &region_id, &id);
0075     if (fields != 2)
0076         return -EINVAL;
0077     sprintf(devname, "dax%d.%d", region_id, id);
0078     if (!sysfs_streq(buf, devname))
0079         return -EINVAL;
0080 
0081     mutex_lock(&dax_bus_lock);
0082     dax_id = __dax_match_id(dax_drv, buf);
0083     if (!dax_id) {
0084         if (action == ID_ADD) {
0085             dax_id = kzalloc(sizeof(*dax_id), GFP_KERNEL);
0086             if (dax_id) {
0087                 strncpy(dax_id->dev_name, buf, DAX_NAME_LEN);
0088                 list_add(&dax_id->list, &dax_drv->ids);
0089             } else
0090                 rc = -ENOMEM;
0091         }
0092     } else if (action == ID_REMOVE) {
0093         list_del(&dax_id->list);
0094         kfree(dax_id);
0095     }
0096     mutex_unlock(&dax_bus_lock);
0097 
0098     if (rc < 0)
0099         return rc;
0100     if (action == ID_ADD)
0101         rc = driver_attach(drv);
0102     if (rc)
0103         return rc;
0104     return count;
0105 }
0106 
0107 static ssize_t new_id_store(struct device_driver *drv, const char *buf,
0108         size_t count)
0109 {
0110     return do_id_store(drv, buf, count, ID_ADD);
0111 }
0112 static DRIVER_ATTR_WO(new_id);
0113 
0114 static ssize_t remove_id_store(struct device_driver *drv, const char *buf,
0115         size_t count)
0116 {
0117     return do_id_store(drv, buf, count, ID_REMOVE);
0118 }
0119 static DRIVER_ATTR_WO(remove_id);
0120 
0121 static struct attribute *dax_drv_attrs[] = {
0122     &driver_attr_new_id.attr,
0123     &driver_attr_remove_id.attr,
0124     NULL,
0125 };
0126 ATTRIBUTE_GROUPS(dax_drv);
0127 
0128 static int dax_bus_match(struct device *dev, struct device_driver *drv);
0129 
0130 /*
0131  * Static dax regions are regions created by an external subsystem
0132  * nvdimm where a single range is assigned. Its boundaries are by the external
0133  * subsystem and are usually limited to one physical memory range. For example,
0134  * for PMEM it is usually defined by NVDIMM Namespace boundaries (i.e. a
0135  * single contiguous range)
0136  *
0137  * On dynamic dax regions, the assigned region can be partitioned by dax core
0138  * into multiple subdivisions. A subdivision is represented into one
0139  * /dev/daxN.M device composed by one or more potentially discontiguous ranges.
0140  *
0141  * When allocating a dax region, drivers must set whether it's static
0142  * (IORESOURCE_DAX_STATIC).  On static dax devices, the @pgmap is pre-assigned
0143  * to dax core when calling devm_create_dev_dax(), whereas in dynamic dax
0144  * devices it is NULL but afterwards allocated by dax core on device ->probe().
0145  * Care is needed to make sure that dynamic dax devices are torn down with a
0146  * cleared @pgmap field (see kill_dev_dax()).
0147  */
0148 static bool is_static(struct dax_region *dax_region)
0149 {
0150     return (dax_region->res.flags & IORESOURCE_DAX_STATIC) != 0;
0151 }
0152 
0153 bool static_dev_dax(struct dev_dax *dev_dax)
0154 {
0155     return is_static(dev_dax->region);
0156 }
0157 EXPORT_SYMBOL_GPL(static_dev_dax);
0158 
0159 static u64 dev_dax_size(struct dev_dax *dev_dax)
0160 {
0161     u64 size = 0;
0162     int i;
0163 
0164     device_lock_assert(&dev_dax->dev);
0165 
0166     for (i = 0; i < dev_dax->nr_range; i++)
0167         size += range_len(&dev_dax->ranges[i].range);
0168 
0169     return size;
0170 }
0171 
0172 static int dax_bus_probe(struct device *dev)
0173 {
0174     struct dax_device_driver *dax_drv = to_dax_drv(dev->driver);
0175     struct dev_dax *dev_dax = to_dev_dax(dev);
0176     struct dax_region *dax_region = dev_dax->region;
0177     int rc;
0178 
0179     if (dev_dax_size(dev_dax) == 0 || dev_dax->id < 0)
0180         return -ENXIO;
0181 
0182     rc = dax_drv->probe(dev_dax);
0183 
0184     if (rc || is_static(dax_region))
0185         return rc;
0186 
0187     /*
0188      * Track new seed creation only after successful probe of the
0189      * previous seed.
0190      */
0191     if (dax_region->seed == dev)
0192         dax_region->seed = NULL;
0193 
0194     return 0;
0195 }
0196 
0197 static void dax_bus_remove(struct device *dev)
0198 {
0199     struct dax_device_driver *dax_drv = to_dax_drv(dev->driver);
0200     struct dev_dax *dev_dax = to_dev_dax(dev);
0201 
0202     if (dax_drv->remove)
0203         dax_drv->remove(dev_dax);
0204 }
0205 
0206 static struct bus_type dax_bus_type = {
0207     .name = "dax",
0208     .uevent = dax_bus_uevent,
0209     .match = dax_bus_match,
0210     .probe = dax_bus_probe,
0211     .remove = dax_bus_remove,
0212     .drv_groups = dax_drv_groups,
0213 };
0214 
0215 static int dax_bus_match(struct device *dev, struct device_driver *drv)
0216 {
0217     struct dax_device_driver *dax_drv = to_dax_drv(drv);
0218 
0219     /*
0220      * All but the 'device-dax' driver, which has 'match_always'
0221      * set, requires an exact id match.
0222      */
0223     if (dax_drv->match_always)
0224         return 1;
0225 
0226     return dax_match_id(dax_drv, dev);
0227 }
0228 
0229 /*
0230  * Rely on the fact that drvdata is set before the attributes are
0231  * registered, and that the attributes are unregistered before drvdata
0232  * is cleared to assume that drvdata is always valid.
0233  */
0234 static ssize_t id_show(struct device *dev,
0235         struct device_attribute *attr, char *buf)
0236 {
0237     struct dax_region *dax_region = dev_get_drvdata(dev);
0238 
0239     return sprintf(buf, "%d\n", dax_region->id);
0240 }
0241 static DEVICE_ATTR_RO(id);
0242 
0243 static ssize_t region_size_show(struct device *dev,
0244         struct device_attribute *attr, char *buf)
0245 {
0246     struct dax_region *dax_region = dev_get_drvdata(dev);
0247 
0248     return sprintf(buf, "%llu\n", (unsigned long long)
0249             resource_size(&dax_region->res));
0250 }
0251 static struct device_attribute dev_attr_region_size = __ATTR(size, 0444,
0252         region_size_show, NULL);
0253 
0254 static ssize_t region_align_show(struct device *dev,
0255         struct device_attribute *attr, char *buf)
0256 {
0257     struct dax_region *dax_region = dev_get_drvdata(dev);
0258 
0259     return sprintf(buf, "%u\n", dax_region->align);
0260 }
0261 static struct device_attribute dev_attr_region_align =
0262         __ATTR(align, 0400, region_align_show, NULL);
0263 
0264 #define for_each_dax_region_resource(dax_region, res) \
0265     for (res = (dax_region)->res.child; res; res = res->sibling)
0266 
0267 static unsigned long long dax_region_avail_size(struct dax_region *dax_region)
0268 {
0269     resource_size_t size = resource_size(&dax_region->res);
0270     struct resource *res;
0271 
0272     device_lock_assert(dax_region->dev);
0273 
0274     for_each_dax_region_resource(dax_region, res)
0275         size -= resource_size(res);
0276     return size;
0277 }
0278 
0279 static ssize_t available_size_show(struct device *dev,
0280         struct device_attribute *attr, char *buf)
0281 {
0282     struct dax_region *dax_region = dev_get_drvdata(dev);
0283     unsigned long long size;
0284 
0285     device_lock(dev);
0286     size = dax_region_avail_size(dax_region);
0287     device_unlock(dev);
0288 
0289     return sprintf(buf, "%llu\n", size);
0290 }
0291 static DEVICE_ATTR_RO(available_size);
0292 
0293 static ssize_t seed_show(struct device *dev,
0294         struct device_attribute *attr, char *buf)
0295 {
0296     struct dax_region *dax_region = dev_get_drvdata(dev);
0297     struct device *seed;
0298     ssize_t rc;
0299 
0300     if (is_static(dax_region))
0301         return -EINVAL;
0302 
0303     device_lock(dev);
0304     seed = dax_region->seed;
0305     rc = sprintf(buf, "%s\n", seed ? dev_name(seed) : "");
0306     device_unlock(dev);
0307 
0308     return rc;
0309 }
0310 static DEVICE_ATTR_RO(seed);
0311 
0312 static ssize_t create_show(struct device *dev,
0313         struct device_attribute *attr, char *buf)
0314 {
0315     struct dax_region *dax_region = dev_get_drvdata(dev);
0316     struct device *youngest;
0317     ssize_t rc;
0318 
0319     if (is_static(dax_region))
0320         return -EINVAL;
0321 
0322     device_lock(dev);
0323     youngest = dax_region->youngest;
0324     rc = sprintf(buf, "%s\n", youngest ? dev_name(youngest) : "");
0325     device_unlock(dev);
0326 
0327     return rc;
0328 }
0329 
0330 static ssize_t create_store(struct device *dev, struct device_attribute *attr,
0331         const char *buf, size_t len)
0332 {
0333     struct dax_region *dax_region = dev_get_drvdata(dev);
0334     unsigned long long avail;
0335     ssize_t rc;
0336     int val;
0337 
0338     if (is_static(dax_region))
0339         return -EINVAL;
0340 
0341     rc = kstrtoint(buf, 0, &val);
0342     if (rc)
0343         return rc;
0344     if (val != 1)
0345         return -EINVAL;
0346 
0347     device_lock(dev);
0348     avail = dax_region_avail_size(dax_region);
0349     if (avail == 0)
0350         rc = -ENOSPC;
0351     else {
0352         struct dev_dax_data data = {
0353             .dax_region = dax_region,
0354             .size = 0,
0355             .id = -1,
0356         };
0357         struct dev_dax *dev_dax = devm_create_dev_dax(&data);
0358 
0359         if (IS_ERR(dev_dax))
0360             rc = PTR_ERR(dev_dax);
0361         else {
0362             /*
0363              * In support of crafting multiple new devices
0364              * simultaneously multiple seeds can be created,
0365              * but only the first one that has not been
0366              * successfully bound is tracked as the region
0367              * seed.
0368              */
0369             if (!dax_region->seed)
0370                 dax_region->seed = &dev_dax->dev;
0371             dax_region->youngest = &dev_dax->dev;
0372             rc = len;
0373         }
0374     }
0375     device_unlock(dev);
0376 
0377     return rc;
0378 }
0379 static DEVICE_ATTR_RW(create);
0380 
0381 void kill_dev_dax(struct dev_dax *dev_dax)
0382 {
0383     struct dax_device *dax_dev = dev_dax->dax_dev;
0384     struct inode *inode = dax_inode(dax_dev);
0385 
0386     kill_dax(dax_dev);
0387     unmap_mapping_range(inode->i_mapping, 0, 0, 1);
0388 
0389     /*
0390      * Dynamic dax region have the pgmap allocated via dev_kzalloc()
0391      * and thus freed by devm. Clear the pgmap to not have stale pgmap
0392      * ranges on probe() from previous reconfigurations of region devices.
0393      */
0394     if (!static_dev_dax(dev_dax))
0395         dev_dax->pgmap = NULL;
0396 }
0397 EXPORT_SYMBOL_GPL(kill_dev_dax);
0398 
0399 static void trim_dev_dax_range(struct dev_dax *dev_dax)
0400 {
0401     int i = dev_dax->nr_range - 1;
0402     struct range *range = &dev_dax->ranges[i].range;
0403     struct dax_region *dax_region = dev_dax->region;
0404 
0405     device_lock_assert(dax_region->dev);
0406     dev_dbg(&dev_dax->dev, "delete range[%d]: %#llx:%#llx\n", i,
0407         (unsigned long long)range->start,
0408         (unsigned long long)range->end);
0409 
0410     __release_region(&dax_region->res, range->start, range_len(range));
0411     if (--dev_dax->nr_range == 0) {
0412         kfree(dev_dax->ranges);
0413         dev_dax->ranges = NULL;
0414     }
0415 }
0416 
0417 static void free_dev_dax_ranges(struct dev_dax *dev_dax)
0418 {
0419     while (dev_dax->nr_range)
0420         trim_dev_dax_range(dev_dax);
0421 }
0422 
0423 static void unregister_dev_dax(void *dev)
0424 {
0425     struct dev_dax *dev_dax = to_dev_dax(dev);
0426 
0427     dev_dbg(dev, "%s\n", __func__);
0428 
0429     kill_dev_dax(dev_dax);
0430     free_dev_dax_ranges(dev_dax);
0431     device_del(dev);
0432     put_device(dev);
0433 }
0434 
0435 /* a return value >= 0 indicates this invocation invalidated the id */
0436 static int __free_dev_dax_id(struct dev_dax *dev_dax)
0437 {
0438     struct dax_region *dax_region = dev_dax->region;
0439     struct device *dev = &dev_dax->dev;
0440     int rc = dev_dax->id;
0441 
0442     device_lock_assert(dev);
0443 
0444     if (is_static(dax_region) || dev_dax->id < 0)
0445         return -1;
0446     ida_free(&dax_region->ida, dev_dax->id);
0447     dev_dax->id = -1;
0448     return rc;
0449 }
0450 
0451 static int free_dev_dax_id(struct dev_dax *dev_dax)
0452 {
0453     struct device *dev = &dev_dax->dev;
0454     int rc;
0455 
0456     device_lock(dev);
0457     rc = __free_dev_dax_id(dev_dax);
0458     device_unlock(dev);
0459     return rc;
0460 }
0461 
0462 static ssize_t delete_store(struct device *dev, struct device_attribute *attr,
0463         const char *buf, size_t len)
0464 {
0465     struct dax_region *dax_region = dev_get_drvdata(dev);
0466     struct dev_dax *dev_dax;
0467     struct device *victim;
0468     bool do_del = false;
0469     int rc;
0470 
0471     if (is_static(dax_region))
0472         return -EINVAL;
0473 
0474     victim = device_find_child_by_name(dax_region->dev, buf);
0475     if (!victim)
0476         return -ENXIO;
0477 
0478     device_lock(dev);
0479     device_lock(victim);
0480     dev_dax = to_dev_dax(victim);
0481     if (victim->driver || dev_dax_size(dev_dax))
0482         rc = -EBUSY;
0483     else {
0484         /*
0485          * Invalidate the device so it does not become active
0486          * again, but always preserve device-id-0 so that
0487          * /sys/bus/dax/ is guaranteed to be populated while any
0488          * dax_region is registered.
0489          */
0490         if (dev_dax->id > 0) {
0491             do_del = __free_dev_dax_id(dev_dax) >= 0;
0492             rc = len;
0493             if (dax_region->seed == victim)
0494                 dax_region->seed = NULL;
0495             if (dax_region->youngest == victim)
0496                 dax_region->youngest = NULL;
0497         } else
0498             rc = -EBUSY;
0499     }
0500     device_unlock(victim);
0501 
0502     /* won the race to invalidate the device, clean it up */
0503     if (do_del)
0504         devm_release_action(dev, unregister_dev_dax, victim);
0505     device_unlock(dev);
0506     put_device(victim);
0507 
0508     return rc;
0509 }
0510 static DEVICE_ATTR_WO(delete);
0511 
0512 static umode_t dax_region_visible(struct kobject *kobj, struct attribute *a,
0513         int n)
0514 {
0515     struct device *dev = container_of(kobj, struct device, kobj);
0516     struct dax_region *dax_region = dev_get_drvdata(dev);
0517 
0518     if (is_static(dax_region))
0519         if (a == &dev_attr_available_size.attr
0520                 || a == &dev_attr_create.attr
0521                 || a == &dev_attr_seed.attr
0522                 || a == &dev_attr_delete.attr)
0523             return 0;
0524     return a->mode;
0525 }
0526 
0527 static struct attribute *dax_region_attributes[] = {
0528     &dev_attr_available_size.attr,
0529     &dev_attr_region_size.attr,
0530     &dev_attr_region_align.attr,
0531     &dev_attr_create.attr,
0532     &dev_attr_seed.attr,
0533     &dev_attr_delete.attr,
0534     &dev_attr_id.attr,
0535     NULL,
0536 };
0537 
0538 static const struct attribute_group dax_region_attribute_group = {
0539     .name = "dax_region",
0540     .attrs = dax_region_attributes,
0541     .is_visible = dax_region_visible,
0542 };
0543 
0544 static const struct attribute_group *dax_region_attribute_groups[] = {
0545     &dax_region_attribute_group,
0546     NULL,
0547 };
0548 
0549 static void dax_region_free(struct kref *kref)
0550 {
0551     struct dax_region *dax_region;
0552 
0553     dax_region = container_of(kref, struct dax_region, kref);
0554     kfree(dax_region);
0555 }
0556 
0557 void dax_region_put(struct dax_region *dax_region)
0558 {
0559     kref_put(&dax_region->kref, dax_region_free);
0560 }
0561 EXPORT_SYMBOL_GPL(dax_region_put);
0562 
0563 static void dax_region_unregister(void *region)
0564 {
0565     struct dax_region *dax_region = region;
0566 
0567     sysfs_remove_groups(&dax_region->dev->kobj,
0568             dax_region_attribute_groups);
0569     dax_region_put(dax_region);
0570 }
0571 
0572 struct dax_region *alloc_dax_region(struct device *parent, int region_id,
0573         struct range *range, int target_node, unsigned int align,
0574         unsigned long flags)
0575 {
0576     struct dax_region *dax_region;
0577 
0578     /*
0579      * The DAX core assumes that it can store its private data in
0580      * parent->driver_data. This WARN is a reminder / safeguard for
0581      * developers of device-dax drivers.
0582      */
0583     if (dev_get_drvdata(parent)) {
0584         dev_WARN(parent, "dax core failed to setup private data\n");
0585         return NULL;
0586     }
0587 
0588     if (!IS_ALIGNED(range->start, align)
0589             || !IS_ALIGNED(range_len(range), align))
0590         return NULL;
0591 
0592     dax_region = kzalloc(sizeof(*dax_region), GFP_KERNEL);
0593     if (!dax_region)
0594         return NULL;
0595 
0596     dev_set_drvdata(parent, dax_region);
0597     kref_init(&dax_region->kref);
0598     dax_region->id = region_id;
0599     dax_region->align = align;
0600     dax_region->dev = parent;
0601     dax_region->target_node = target_node;
0602     ida_init(&dax_region->ida);
0603     dax_region->res = (struct resource) {
0604         .start = range->start,
0605         .end = range->end,
0606         .flags = IORESOURCE_MEM | flags,
0607     };
0608 
0609     if (sysfs_create_groups(&parent->kobj, dax_region_attribute_groups)) {
0610         kfree(dax_region);
0611         return NULL;
0612     }
0613 
0614     kref_get(&dax_region->kref);
0615     if (devm_add_action_or_reset(parent, dax_region_unregister, dax_region))
0616         return NULL;
0617     return dax_region;
0618 }
0619 EXPORT_SYMBOL_GPL(alloc_dax_region);
0620 
0621 static void dax_mapping_release(struct device *dev)
0622 {
0623     struct dax_mapping *mapping = to_dax_mapping(dev);
0624     struct dev_dax *dev_dax = to_dev_dax(dev->parent);
0625 
0626     ida_free(&dev_dax->ida, mapping->id);
0627     kfree(mapping);
0628 }
0629 
0630 static void unregister_dax_mapping(void *data)
0631 {
0632     struct device *dev = data;
0633     struct dax_mapping *mapping = to_dax_mapping(dev);
0634     struct dev_dax *dev_dax = to_dev_dax(dev->parent);
0635     struct dax_region *dax_region = dev_dax->region;
0636 
0637     dev_dbg(dev, "%s\n", __func__);
0638 
0639     device_lock_assert(dax_region->dev);
0640 
0641     dev_dax->ranges[mapping->range_id].mapping = NULL;
0642     mapping->range_id = -1;
0643 
0644     device_del(dev);
0645     put_device(dev);
0646 }
0647 
0648 static struct dev_dax_range *get_dax_range(struct device *dev)
0649 {
0650     struct dax_mapping *mapping = to_dax_mapping(dev);
0651     struct dev_dax *dev_dax = to_dev_dax(dev->parent);
0652     struct dax_region *dax_region = dev_dax->region;
0653 
0654     device_lock(dax_region->dev);
0655     if (mapping->range_id < 0) {
0656         device_unlock(dax_region->dev);
0657         return NULL;
0658     }
0659 
0660     return &dev_dax->ranges[mapping->range_id];
0661 }
0662 
0663 static void put_dax_range(struct dev_dax_range *dax_range)
0664 {
0665     struct dax_mapping *mapping = dax_range->mapping;
0666     struct dev_dax *dev_dax = to_dev_dax(mapping->dev.parent);
0667     struct dax_region *dax_region = dev_dax->region;
0668 
0669     device_unlock(dax_region->dev);
0670 }
0671 
0672 static ssize_t start_show(struct device *dev,
0673         struct device_attribute *attr, char *buf)
0674 {
0675     struct dev_dax_range *dax_range;
0676     ssize_t rc;
0677 
0678     dax_range = get_dax_range(dev);
0679     if (!dax_range)
0680         return -ENXIO;
0681     rc = sprintf(buf, "%#llx\n", dax_range->range.start);
0682     put_dax_range(dax_range);
0683 
0684     return rc;
0685 }
0686 static DEVICE_ATTR(start, 0400, start_show, NULL);
0687 
0688 static ssize_t end_show(struct device *dev,
0689         struct device_attribute *attr, char *buf)
0690 {
0691     struct dev_dax_range *dax_range;
0692     ssize_t rc;
0693 
0694     dax_range = get_dax_range(dev);
0695     if (!dax_range)
0696         return -ENXIO;
0697     rc = sprintf(buf, "%#llx\n", dax_range->range.end);
0698     put_dax_range(dax_range);
0699 
0700     return rc;
0701 }
0702 static DEVICE_ATTR(end, 0400, end_show, NULL);
0703 
0704 static ssize_t pgoff_show(struct device *dev,
0705         struct device_attribute *attr, char *buf)
0706 {
0707     struct dev_dax_range *dax_range;
0708     ssize_t rc;
0709 
0710     dax_range = get_dax_range(dev);
0711     if (!dax_range)
0712         return -ENXIO;
0713     rc = sprintf(buf, "%#lx\n", dax_range->pgoff);
0714     put_dax_range(dax_range);
0715 
0716     return rc;
0717 }
0718 static DEVICE_ATTR(page_offset, 0400, pgoff_show, NULL);
0719 
0720 static struct attribute *dax_mapping_attributes[] = {
0721     &dev_attr_start.attr,
0722     &dev_attr_end.attr,
0723     &dev_attr_page_offset.attr,
0724     NULL,
0725 };
0726 
0727 static const struct attribute_group dax_mapping_attribute_group = {
0728     .attrs = dax_mapping_attributes,
0729 };
0730 
0731 static const struct attribute_group *dax_mapping_attribute_groups[] = {
0732     &dax_mapping_attribute_group,
0733     NULL,
0734 };
0735 
0736 static struct device_type dax_mapping_type = {
0737     .release = dax_mapping_release,
0738     .groups = dax_mapping_attribute_groups,
0739 };
0740 
0741 static int devm_register_dax_mapping(struct dev_dax *dev_dax, int range_id)
0742 {
0743     struct dax_region *dax_region = dev_dax->region;
0744     struct dax_mapping *mapping;
0745     struct device *dev;
0746     int rc;
0747 
0748     device_lock_assert(dax_region->dev);
0749 
0750     if (dev_WARN_ONCE(&dev_dax->dev, !dax_region->dev->driver,
0751                 "region disabled\n"))
0752         return -ENXIO;
0753 
0754     mapping = kzalloc(sizeof(*mapping), GFP_KERNEL);
0755     if (!mapping)
0756         return -ENOMEM;
0757     mapping->range_id = range_id;
0758     mapping->id = ida_alloc(&dev_dax->ida, GFP_KERNEL);
0759     if (mapping->id < 0) {
0760         kfree(mapping);
0761         return -ENOMEM;
0762     }
0763     dev_dax->ranges[range_id].mapping = mapping;
0764     dev = &mapping->dev;
0765     device_initialize(dev);
0766     dev->parent = &dev_dax->dev;
0767     dev->type = &dax_mapping_type;
0768     dev_set_name(dev, "mapping%d", mapping->id);
0769     rc = device_add(dev);
0770     if (rc) {
0771         put_device(dev);
0772         return rc;
0773     }
0774 
0775     rc = devm_add_action_or_reset(dax_region->dev, unregister_dax_mapping,
0776             dev);
0777     if (rc)
0778         return rc;
0779     return 0;
0780 }
0781 
0782 static int alloc_dev_dax_range(struct dev_dax *dev_dax, u64 start,
0783         resource_size_t size)
0784 {
0785     struct dax_region *dax_region = dev_dax->region;
0786     struct resource *res = &dax_region->res;
0787     struct device *dev = &dev_dax->dev;
0788     struct dev_dax_range *ranges;
0789     unsigned long pgoff = 0;
0790     struct resource *alloc;
0791     int i, rc;
0792 
0793     device_lock_assert(dax_region->dev);
0794 
0795     /* handle the seed alloc special case */
0796     if (!size) {
0797         if (dev_WARN_ONCE(dev, dev_dax->nr_range,
0798                     "0-size allocation must be first\n"))
0799             return -EBUSY;
0800         /* nr_range == 0 is elsewhere special cased as 0-size device */
0801         return 0;
0802     }
0803 
0804     alloc = __request_region(res, start, size, dev_name(dev), 0);
0805     if (!alloc)
0806         return -ENOMEM;
0807 
0808     ranges = krealloc(dev_dax->ranges, sizeof(*ranges)
0809             * (dev_dax->nr_range + 1), GFP_KERNEL);
0810     if (!ranges) {
0811         __release_region(res, alloc->start, resource_size(alloc));
0812         return -ENOMEM;
0813     }
0814 
0815     for (i = 0; i < dev_dax->nr_range; i++)
0816         pgoff += PHYS_PFN(range_len(&ranges[i].range));
0817     dev_dax->ranges = ranges;
0818     ranges[dev_dax->nr_range++] = (struct dev_dax_range) {
0819         .pgoff = pgoff,
0820         .range = {
0821             .start = alloc->start,
0822             .end = alloc->end,
0823         },
0824     };
0825 
0826     dev_dbg(dev, "alloc range[%d]: %pa:%pa\n", dev_dax->nr_range - 1,
0827             &alloc->start, &alloc->end);
0828     /*
0829      * A dev_dax instance must be registered before mapping device
0830      * children can be added. Defer to devm_create_dev_dax() to add
0831      * the initial mapping device.
0832      */
0833     if (!device_is_registered(&dev_dax->dev))
0834         return 0;
0835 
0836     rc = devm_register_dax_mapping(dev_dax, dev_dax->nr_range - 1);
0837     if (rc)
0838         trim_dev_dax_range(dev_dax);
0839 
0840     return rc;
0841 }
0842 
0843 static int adjust_dev_dax_range(struct dev_dax *dev_dax, struct resource *res, resource_size_t size)
0844 {
0845     int last_range = dev_dax->nr_range - 1;
0846     struct dev_dax_range *dax_range = &dev_dax->ranges[last_range];
0847     struct dax_region *dax_region = dev_dax->region;
0848     bool is_shrink = resource_size(res) > size;
0849     struct range *range = &dax_range->range;
0850     struct device *dev = &dev_dax->dev;
0851     int rc;
0852 
0853     device_lock_assert(dax_region->dev);
0854 
0855     if (dev_WARN_ONCE(dev, !size, "deletion is handled by dev_dax_shrink\n"))
0856         return -EINVAL;
0857 
0858     rc = adjust_resource(res, range->start, size);
0859     if (rc)
0860         return rc;
0861 
0862     *range = (struct range) {
0863         .start = range->start,
0864         .end = range->start + size - 1,
0865     };
0866 
0867     dev_dbg(dev, "%s range[%d]: %#llx:%#llx\n", is_shrink ? "shrink" : "extend",
0868             last_range, (unsigned long long) range->start,
0869             (unsigned long long) range->end);
0870 
0871     return 0;
0872 }
0873 
0874 static ssize_t size_show(struct device *dev,
0875         struct device_attribute *attr, char *buf)
0876 {
0877     struct dev_dax *dev_dax = to_dev_dax(dev);
0878     unsigned long long size;
0879 
0880     device_lock(dev);
0881     size = dev_dax_size(dev_dax);
0882     device_unlock(dev);
0883 
0884     return sprintf(buf, "%llu\n", size);
0885 }
0886 
0887 static bool alloc_is_aligned(struct dev_dax *dev_dax, resource_size_t size)
0888 {
0889     /*
0890      * The minimum mapping granularity for a device instance is a
0891      * single subsection, unless the arch says otherwise.
0892      */
0893     return IS_ALIGNED(size, max_t(unsigned long, dev_dax->align, memremap_compat_align()));
0894 }
0895 
0896 static int dev_dax_shrink(struct dev_dax *dev_dax, resource_size_t size)
0897 {
0898     resource_size_t to_shrink = dev_dax_size(dev_dax) - size;
0899     struct dax_region *dax_region = dev_dax->region;
0900     struct device *dev = &dev_dax->dev;
0901     int i;
0902 
0903     for (i = dev_dax->nr_range - 1; i >= 0; i--) {
0904         struct range *range = &dev_dax->ranges[i].range;
0905         struct dax_mapping *mapping = dev_dax->ranges[i].mapping;
0906         struct resource *adjust = NULL, *res;
0907         resource_size_t shrink;
0908 
0909         shrink = min_t(u64, to_shrink, range_len(range));
0910         if (shrink >= range_len(range)) {
0911             devm_release_action(dax_region->dev,
0912                     unregister_dax_mapping, &mapping->dev);
0913             trim_dev_dax_range(dev_dax);
0914             to_shrink -= shrink;
0915             if (!to_shrink)
0916                 break;
0917             continue;
0918         }
0919 
0920         for_each_dax_region_resource(dax_region, res)
0921             if (strcmp(res->name, dev_name(dev)) == 0
0922                     && res->start == range->start) {
0923                 adjust = res;
0924                 break;
0925             }
0926 
0927         if (dev_WARN_ONCE(dev, !adjust || i != dev_dax->nr_range - 1,
0928                     "failed to find matching resource\n"))
0929             return -ENXIO;
0930         return adjust_dev_dax_range(dev_dax, adjust, range_len(range)
0931                 - shrink);
0932     }
0933     return 0;
0934 }
0935 
0936 /*
0937  * Only allow adjustments that preserve the relative pgoff of existing
0938  * allocations. I.e. the dev_dax->ranges array is ordered by increasing pgoff.
0939  */
0940 static bool adjust_ok(struct dev_dax *dev_dax, struct resource *res)
0941 {
0942     struct dev_dax_range *last;
0943     int i;
0944 
0945     if (dev_dax->nr_range == 0)
0946         return false;
0947     if (strcmp(res->name, dev_name(&dev_dax->dev)) != 0)
0948         return false;
0949     last = &dev_dax->ranges[dev_dax->nr_range - 1];
0950     if (last->range.start != res->start || last->range.end != res->end)
0951         return false;
0952     for (i = 0; i < dev_dax->nr_range - 1; i++) {
0953         struct dev_dax_range *dax_range = &dev_dax->ranges[i];
0954 
0955         if (dax_range->pgoff > last->pgoff)
0956             return false;
0957     }
0958 
0959     return true;
0960 }
0961 
0962 static ssize_t dev_dax_resize(struct dax_region *dax_region,
0963         struct dev_dax *dev_dax, resource_size_t size)
0964 {
0965     resource_size_t avail = dax_region_avail_size(dax_region), to_alloc;
0966     resource_size_t dev_size = dev_dax_size(dev_dax);
0967     struct resource *region_res = &dax_region->res;
0968     struct device *dev = &dev_dax->dev;
0969     struct resource *res, *first;
0970     resource_size_t alloc = 0;
0971     int rc;
0972 
0973     if (dev->driver)
0974         return -EBUSY;
0975     if (size == dev_size)
0976         return 0;
0977     if (size > dev_size && size - dev_size > avail)
0978         return -ENOSPC;
0979     if (size < dev_size)
0980         return dev_dax_shrink(dev_dax, size);
0981 
0982     to_alloc = size - dev_size;
0983     if (dev_WARN_ONCE(dev, !alloc_is_aligned(dev_dax, to_alloc),
0984             "resize of %pa misaligned\n", &to_alloc))
0985         return -ENXIO;
0986 
0987     /*
0988      * Expand the device into the unused portion of the region. This
0989      * may involve adjusting the end of an existing resource, or
0990      * allocating a new resource.
0991      */
0992 retry:
0993     first = region_res->child;
0994     if (!first)
0995         return alloc_dev_dax_range(dev_dax, dax_region->res.start, to_alloc);
0996 
0997     rc = -ENOSPC;
0998     for (res = first; res; res = res->sibling) {
0999         struct resource *next = res->sibling;
1000 
1001         /* space at the beginning of the region */
1002         if (res == first && res->start > dax_region->res.start) {
1003             alloc = min(res->start - dax_region->res.start, to_alloc);
1004             rc = alloc_dev_dax_range(dev_dax, dax_region->res.start, alloc);
1005             break;
1006         }
1007 
1008         alloc = 0;
1009         /* space between allocations */
1010         if (next && next->start > res->end + 1)
1011             alloc = min(next->start - (res->end + 1), to_alloc);
1012 
1013         /* space at the end of the region */
1014         if (!alloc && !next && res->end < region_res->end)
1015             alloc = min(region_res->end - res->end, to_alloc);
1016 
1017         if (!alloc)
1018             continue;
1019 
1020         if (adjust_ok(dev_dax, res)) {
1021             rc = adjust_dev_dax_range(dev_dax, res, resource_size(res) + alloc);
1022             break;
1023         }
1024         rc = alloc_dev_dax_range(dev_dax, res->end + 1, alloc);
1025         break;
1026     }
1027     if (rc)
1028         return rc;
1029     to_alloc -= alloc;
1030     if (to_alloc)
1031         goto retry;
1032     return 0;
1033 }
1034 
1035 static ssize_t size_store(struct device *dev, struct device_attribute *attr,
1036         const char *buf, size_t len)
1037 {
1038     ssize_t rc;
1039     unsigned long long val;
1040     struct dev_dax *dev_dax = to_dev_dax(dev);
1041     struct dax_region *dax_region = dev_dax->region;
1042 
1043     rc = kstrtoull(buf, 0, &val);
1044     if (rc)
1045         return rc;
1046 
1047     if (!alloc_is_aligned(dev_dax, val)) {
1048         dev_dbg(dev, "%s: size: %lld misaligned\n", __func__, val);
1049         return -EINVAL;
1050     }
1051 
1052     device_lock(dax_region->dev);
1053     if (!dax_region->dev->driver) {
1054         device_unlock(dax_region->dev);
1055         return -ENXIO;
1056     }
1057     device_lock(dev);
1058     rc = dev_dax_resize(dax_region, dev_dax, val);
1059     device_unlock(dev);
1060     device_unlock(dax_region->dev);
1061 
1062     return rc == 0 ? len : rc;
1063 }
1064 static DEVICE_ATTR_RW(size);
1065 
1066 static ssize_t range_parse(const char *opt, size_t len, struct range *range)
1067 {
1068     unsigned long long addr = 0;
1069     char *start, *end, *str;
1070     ssize_t rc = -EINVAL;
1071 
1072     str = kstrdup(opt, GFP_KERNEL);
1073     if (!str)
1074         return rc;
1075 
1076     end = str;
1077     start = strsep(&end, "-");
1078     if (!start || !end)
1079         goto err;
1080 
1081     rc = kstrtoull(start, 16, &addr);
1082     if (rc)
1083         goto err;
1084     range->start = addr;
1085 
1086     rc = kstrtoull(end, 16, &addr);
1087     if (rc)
1088         goto err;
1089     range->end = addr;
1090 
1091 err:
1092     kfree(str);
1093     return rc;
1094 }
1095 
1096 static ssize_t mapping_store(struct device *dev, struct device_attribute *attr,
1097         const char *buf, size_t len)
1098 {
1099     struct dev_dax *dev_dax = to_dev_dax(dev);
1100     struct dax_region *dax_region = dev_dax->region;
1101     size_t to_alloc;
1102     struct range r;
1103     ssize_t rc;
1104 
1105     rc = range_parse(buf, len, &r);
1106     if (rc)
1107         return rc;
1108 
1109     rc = -ENXIO;
1110     device_lock(dax_region->dev);
1111     if (!dax_region->dev->driver) {
1112         device_unlock(dax_region->dev);
1113         return rc;
1114     }
1115     device_lock(dev);
1116 
1117     to_alloc = range_len(&r);
1118     if (alloc_is_aligned(dev_dax, to_alloc))
1119         rc = alloc_dev_dax_range(dev_dax, r.start, to_alloc);
1120     device_unlock(dev);
1121     device_unlock(dax_region->dev);
1122 
1123     return rc == 0 ? len : rc;
1124 }
1125 static DEVICE_ATTR_WO(mapping);
1126 
1127 static ssize_t align_show(struct device *dev,
1128         struct device_attribute *attr, char *buf)
1129 {
1130     struct dev_dax *dev_dax = to_dev_dax(dev);
1131 
1132     return sprintf(buf, "%d\n", dev_dax->align);
1133 }
1134 
1135 static ssize_t dev_dax_validate_align(struct dev_dax *dev_dax)
1136 {
1137     struct device *dev = &dev_dax->dev;
1138     int i;
1139 
1140     for (i = 0; i < dev_dax->nr_range; i++) {
1141         size_t len = range_len(&dev_dax->ranges[i].range);
1142 
1143         if (!alloc_is_aligned(dev_dax, len)) {
1144             dev_dbg(dev, "%s: align %u invalid for range %d\n",
1145                 __func__, dev_dax->align, i);
1146             return -EINVAL;
1147         }
1148     }
1149 
1150     return 0;
1151 }
1152 
1153 static ssize_t align_store(struct device *dev, struct device_attribute *attr,
1154         const char *buf, size_t len)
1155 {
1156     struct dev_dax *dev_dax = to_dev_dax(dev);
1157     struct dax_region *dax_region = dev_dax->region;
1158     unsigned long val, align_save;
1159     ssize_t rc;
1160 
1161     rc = kstrtoul(buf, 0, &val);
1162     if (rc)
1163         return -ENXIO;
1164 
1165     if (!dax_align_valid(val))
1166         return -EINVAL;
1167 
1168     device_lock(dax_region->dev);
1169     if (!dax_region->dev->driver) {
1170         device_unlock(dax_region->dev);
1171         return -ENXIO;
1172     }
1173 
1174     device_lock(dev);
1175     if (dev->driver) {
1176         rc = -EBUSY;
1177         goto out_unlock;
1178     }
1179 
1180     align_save = dev_dax->align;
1181     dev_dax->align = val;
1182     rc = dev_dax_validate_align(dev_dax);
1183     if (rc)
1184         dev_dax->align = align_save;
1185 out_unlock:
1186     device_unlock(dev);
1187     device_unlock(dax_region->dev);
1188     return rc == 0 ? len : rc;
1189 }
1190 static DEVICE_ATTR_RW(align);
1191 
1192 static int dev_dax_target_node(struct dev_dax *dev_dax)
1193 {
1194     struct dax_region *dax_region = dev_dax->region;
1195 
1196     return dax_region->target_node;
1197 }
1198 
1199 static ssize_t target_node_show(struct device *dev,
1200         struct device_attribute *attr, char *buf)
1201 {
1202     struct dev_dax *dev_dax = to_dev_dax(dev);
1203 
1204     return sprintf(buf, "%d\n", dev_dax_target_node(dev_dax));
1205 }
1206 static DEVICE_ATTR_RO(target_node);
1207 
1208 static ssize_t resource_show(struct device *dev,
1209         struct device_attribute *attr, char *buf)
1210 {
1211     struct dev_dax *dev_dax = to_dev_dax(dev);
1212     struct dax_region *dax_region = dev_dax->region;
1213     unsigned long long start;
1214 
1215     if (dev_dax->nr_range < 1)
1216         start = dax_region->res.start;
1217     else
1218         start = dev_dax->ranges[0].range.start;
1219 
1220     return sprintf(buf, "%#llx\n", start);
1221 }
1222 static DEVICE_ATTR(resource, 0400, resource_show, NULL);
1223 
1224 static ssize_t modalias_show(struct device *dev, struct device_attribute *attr,
1225         char *buf)
1226 {
1227     /*
1228      * We only ever expect to handle device-dax instances, i.e. the
1229      * @type argument to MODULE_ALIAS_DAX_DEVICE() is always zero
1230      */
1231     return sprintf(buf, DAX_DEVICE_MODALIAS_FMT "\n", 0);
1232 }
1233 static DEVICE_ATTR_RO(modalias);
1234 
1235 static ssize_t numa_node_show(struct device *dev,
1236         struct device_attribute *attr, char *buf)
1237 {
1238     return sprintf(buf, "%d\n", dev_to_node(dev));
1239 }
1240 static DEVICE_ATTR_RO(numa_node);
1241 
1242 static umode_t dev_dax_visible(struct kobject *kobj, struct attribute *a, int n)
1243 {
1244     struct device *dev = container_of(kobj, struct device, kobj);
1245     struct dev_dax *dev_dax = to_dev_dax(dev);
1246     struct dax_region *dax_region = dev_dax->region;
1247 
1248     if (a == &dev_attr_target_node.attr && dev_dax_target_node(dev_dax) < 0)
1249         return 0;
1250     if (a == &dev_attr_numa_node.attr && !IS_ENABLED(CONFIG_NUMA))
1251         return 0;
1252     if (a == &dev_attr_mapping.attr && is_static(dax_region))
1253         return 0;
1254     if ((a == &dev_attr_align.attr ||
1255          a == &dev_attr_size.attr) && is_static(dax_region))
1256         return 0444;
1257     return a->mode;
1258 }
1259 
1260 static struct attribute *dev_dax_attributes[] = {
1261     &dev_attr_modalias.attr,
1262     &dev_attr_size.attr,
1263     &dev_attr_mapping.attr,
1264     &dev_attr_target_node.attr,
1265     &dev_attr_align.attr,
1266     &dev_attr_resource.attr,
1267     &dev_attr_numa_node.attr,
1268     NULL,
1269 };
1270 
1271 static const struct attribute_group dev_dax_attribute_group = {
1272     .attrs = dev_dax_attributes,
1273     .is_visible = dev_dax_visible,
1274 };
1275 
1276 static const struct attribute_group *dax_attribute_groups[] = {
1277     &dev_dax_attribute_group,
1278     NULL,
1279 };
1280 
1281 static void dev_dax_release(struct device *dev)
1282 {
1283     struct dev_dax *dev_dax = to_dev_dax(dev);
1284     struct dax_region *dax_region = dev_dax->region;
1285     struct dax_device *dax_dev = dev_dax->dax_dev;
1286 
1287     put_dax(dax_dev);
1288     free_dev_dax_id(dev_dax);
1289     dax_region_put(dax_region);
1290     kfree(dev_dax->pgmap);
1291     kfree(dev_dax);
1292 }
1293 
1294 static const struct device_type dev_dax_type = {
1295     .release = dev_dax_release,
1296     .groups = dax_attribute_groups,
1297 };
1298 
1299 struct dev_dax *devm_create_dev_dax(struct dev_dax_data *data)
1300 {
1301     struct dax_region *dax_region = data->dax_region;
1302     struct device *parent = dax_region->dev;
1303     struct dax_device *dax_dev;
1304     struct dev_dax *dev_dax;
1305     struct inode *inode;
1306     struct device *dev;
1307     int rc;
1308 
1309     dev_dax = kzalloc(sizeof(*dev_dax), GFP_KERNEL);
1310     if (!dev_dax)
1311         return ERR_PTR(-ENOMEM);
1312 
1313     if (is_static(dax_region)) {
1314         if (dev_WARN_ONCE(parent, data->id < 0,
1315                 "dynamic id specified to static region\n")) {
1316             rc = -EINVAL;
1317             goto err_id;
1318         }
1319 
1320         dev_dax->id = data->id;
1321     } else {
1322         if (dev_WARN_ONCE(parent, data->id >= 0,
1323                 "static id specified to dynamic region\n")) {
1324             rc = -EINVAL;
1325             goto err_id;
1326         }
1327 
1328         rc = ida_alloc(&dax_region->ida, GFP_KERNEL);
1329         if (rc < 0)
1330             goto err_id;
1331         dev_dax->id = rc;
1332     }
1333 
1334     dev_dax->region = dax_region;
1335     dev = &dev_dax->dev;
1336     device_initialize(dev);
1337     dev_set_name(dev, "dax%d.%d", dax_region->id, dev_dax->id);
1338 
1339     rc = alloc_dev_dax_range(dev_dax, dax_region->res.start, data->size);
1340     if (rc)
1341         goto err_range;
1342 
1343     if (data->pgmap) {
1344         dev_WARN_ONCE(parent, !is_static(dax_region),
1345             "custom dev_pagemap requires a static dax_region\n");
1346 
1347         dev_dax->pgmap = kmemdup(data->pgmap,
1348                 sizeof(struct dev_pagemap), GFP_KERNEL);
1349         if (!dev_dax->pgmap) {
1350             rc = -ENOMEM;
1351             goto err_pgmap;
1352         }
1353     }
1354 
1355     /*
1356      * No dax_operations since there is no access to this device outside of
1357      * mmap of the resulting character device.
1358      */
1359     dax_dev = alloc_dax(dev_dax, NULL);
1360     if (IS_ERR(dax_dev)) {
1361         rc = PTR_ERR(dax_dev);
1362         goto err_alloc_dax;
1363     }
1364     set_dax_synchronous(dax_dev);
1365     set_dax_nocache(dax_dev);
1366     set_dax_nomc(dax_dev);
1367 
1368     /* a device_dax instance is dead while the driver is not attached */
1369     kill_dax(dax_dev);
1370 
1371     dev_dax->dax_dev = dax_dev;
1372     dev_dax->target_node = dax_region->target_node;
1373     dev_dax->align = dax_region->align;
1374     ida_init(&dev_dax->ida);
1375     kref_get(&dax_region->kref);
1376 
1377     inode = dax_inode(dax_dev);
1378     dev->devt = inode->i_rdev;
1379     dev->bus = &dax_bus_type;
1380     dev->parent = parent;
1381     dev->type = &dev_dax_type;
1382 
1383     rc = device_add(dev);
1384     if (rc) {
1385         kill_dev_dax(dev_dax);
1386         put_device(dev);
1387         return ERR_PTR(rc);
1388     }
1389 
1390     rc = devm_add_action_or_reset(dax_region->dev, unregister_dev_dax, dev);
1391     if (rc)
1392         return ERR_PTR(rc);
1393 
1394     /* register mapping device for the initial allocation range */
1395     if (dev_dax->nr_range && range_len(&dev_dax->ranges[0].range)) {
1396         rc = devm_register_dax_mapping(dev_dax, 0);
1397         if (rc)
1398             return ERR_PTR(rc);
1399     }
1400 
1401     return dev_dax;
1402 
1403 err_alloc_dax:
1404     kfree(dev_dax->pgmap);
1405 err_pgmap:
1406     free_dev_dax_ranges(dev_dax);
1407 err_range:
1408     free_dev_dax_id(dev_dax);
1409 err_id:
1410     kfree(dev_dax);
1411 
1412     return ERR_PTR(rc);
1413 }
1414 EXPORT_SYMBOL_GPL(devm_create_dev_dax);
1415 
1416 static int match_always_count;
1417 
1418 int __dax_driver_register(struct dax_device_driver *dax_drv,
1419         struct module *module, const char *mod_name)
1420 {
1421     struct device_driver *drv = &dax_drv->drv;
1422     int rc = 0;
1423 
1424     /*
1425      * dax_bus_probe() calls dax_drv->probe() unconditionally.
1426      * So better be safe than sorry and ensure it is provided.
1427      */
1428     if (!dax_drv->probe)
1429         return -EINVAL;
1430 
1431     INIT_LIST_HEAD(&dax_drv->ids);
1432     drv->owner = module;
1433     drv->name = mod_name;
1434     drv->mod_name = mod_name;
1435     drv->bus = &dax_bus_type;
1436 
1437     /* there can only be one default driver */
1438     mutex_lock(&dax_bus_lock);
1439     match_always_count += dax_drv->match_always;
1440     if (match_always_count > 1) {
1441         match_always_count--;
1442         WARN_ON(1);
1443         rc = -EINVAL;
1444     }
1445     mutex_unlock(&dax_bus_lock);
1446     if (rc)
1447         return rc;
1448 
1449     rc = driver_register(drv);
1450     if (rc && dax_drv->match_always) {
1451         mutex_lock(&dax_bus_lock);
1452         match_always_count -= dax_drv->match_always;
1453         mutex_unlock(&dax_bus_lock);
1454     }
1455 
1456     return rc;
1457 }
1458 EXPORT_SYMBOL_GPL(__dax_driver_register);
1459 
1460 void dax_driver_unregister(struct dax_device_driver *dax_drv)
1461 {
1462     struct device_driver *drv = &dax_drv->drv;
1463     struct dax_id *dax_id, *_id;
1464 
1465     mutex_lock(&dax_bus_lock);
1466     match_always_count -= dax_drv->match_always;
1467     list_for_each_entry_safe(dax_id, _id, &dax_drv->ids, list) {
1468         list_del(&dax_id->list);
1469         kfree(dax_id);
1470     }
1471     mutex_unlock(&dax_bus_lock);
1472     driver_unregister(drv);
1473 }
1474 EXPORT_SYMBOL_GPL(dax_driver_unregister);
1475 
1476 int __init dax_bus_init(void)
1477 {
1478     return bus_register(&dax_bus_type);
1479 }
1480 
1481 void __exit dax_bus_exit(void)
1482 {
1483     bus_unregister(&dax_bus_type);
1484 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!