Back to home page

OSCL-LXR

 
 

    


0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  * RDMA Network Block Driver
0004  *
0005  * Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
0006  * Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
0007  * Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
0008  */
0009 
0010 #undef pr_fmt
0011 #define pr_fmt(fmt) KBUILD_MODNAME " L" __stringify(__LINE__) ": " fmt
0012 
0013 #include <linux/types.h>
0014 #include <linux/ctype.h>
0015 #include <linux/parser.h>
0016 #include <linux/module.h>
0017 #include <linux/in6.h>
0018 #include <linux/fs.h>
0019 #include <linux/uaccess.h>
0020 #include <linux/device.h>
0021 #include <rdma/ib.h>
0022 #include <rdma/rdma_cm.h>
0023 
0024 #include "rnbd-clt.h"
0025 
0026 static struct device *rnbd_dev;
0027 static struct class *rnbd_dev_class;
0028 static struct kobject *rnbd_devs_kobj;
0029 
0030 enum {
0031     RNBD_OPT_ERR        = 0,
0032     RNBD_OPT_DEST_PORT  = 1 << 0,
0033     RNBD_OPT_PATH       = 1 << 1,
0034     RNBD_OPT_DEV_PATH   = 1 << 2,
0035     RNBD_OPT_ACCESS_MODE    = 1 << 3,
0036     RNBD_OPT_SESSNAME   = 1 << 6,
0037     RNBD_OPT_NR_POLL_QUEUES = 1 << 7,
0038 };
0039 
0040 static const unsigned int rnbd_opt_mandatory[] = {
0041     RNBD_OPT_DEV_PATH,
0042     RNBD_OPT_SESSNAME,
0043 };
0044 
0045 static const match_table_t rnbd_opt_tokens = {
0046     {RNBD_OPT_PATH,         "path=%s"       },
0047     {RNBD_OPT_DEV_PATH,     "device_path=%s"    },
0048     {RNBD_OPT_DEST_PORT,        "dest_port=%d"      },
0049     {RNBD_OPT_ACCESS_MODE,      "access_mode=%s"    },
0050     {RNBD_OPT_SESSNAME,     "sessname=%s"       },
0051     {RNBD_OPT_NR_POLL_QUEUES,   "nr_poll_queues=%d" },
0052     {RNBD_OPT_ERR,          NULL            },
0053 };
0054 
0055 struct rnbd_map_options {
0056     char *sessname;
0057     struct rtrs_addr *paths;
0058     size_t *path_cnt;
0059     char *pathname;
0060     u16 *dest_port;
0061     enum rnbd_access_mode *access_mode;
0062     u32 *nr_poll_queues;
0063 };
0064 
0065 static int rnbd_clt_parse_map_options(const char *buf, size_t max_path_cnt,
0066                        struct rnbd_map_options *opt)
0067 {
0068     char *options, *sep_opt;
0069     char *p;
0070     substring_t args[MAX_OPT_ARGS];
0071     int opt_mask = 0;
0072     int token;
0073     int ret = -EINVAL;
0074     int nr_poll_queues = 0;
0075     int dest_port = 0;
0076     int p_cnt = 0;
0077     int i;
0078 
0079     options = kstrdup(buf, GFP_KERNEL);
0080     if (!options)
0081         return -ENOMEM;
0082 
0083     sep_opt = strstrip(options);
0084     while ((p = strsep(&sep_opt, " ")) != NULL) {
0085         if (!*p)
0086             continue;
0087 
0088         token = match_token(p, rnbd_opt_tokens, args);
0089         opt_mask |= token;
0090 
0091         switch (token) {
0092         case RNBD_OPT_SESSNAME:
0093             p = match_strdup(args);
0094             if (!p) {
0095                 ret = -ENOMEM;
0096                 goto out;
0097             }
0098             if (strlen(p) > NAME_MAX) {
0099                 pr_err("map_device: sessname too long\n");
0100                 ret = -EINVAL;
0101                 kfree(p);
0102                 goto out;
0103             }
0104             strscpy(opt->sessname, p, NAME_MAX);
0105             kfree(p);
0106             break;
0107 
0108         case RNBD_OPT_PATH:
0109             if (p_cnt >= max_path_cnt) {
0110                 pr_err("map_device: too many (> %zu) paths provided\n",
0111                        max_path_cnt);
0112                 ret = -ENOMEM;
0113                 goto out;
0114             }
0115             p = match_strdup(args);
0116             if (!p) {
0117                 ret = -ENOMEM;
0118                 goto out;
0119             }
0120 
0121             ret = rtrs_addr_to_sockaddr(p, strlen(p),
0122                             *opt->dest_port,
0123                             &opt->paths[p_cnt]);
0124             if (ret) {
0125                 pr_err("Can't parse path %s: %d\n", p, ret);
0126                 kfree(p);
0127                 goto out;
0128             }
0129 
0130             p_cnt++;
0131 
0132             kfree(p);
0133             break;
0134 
0135         case RNBD_OPT_DEV_PATH:
0136             p = match_strdup(args);
0137             if (!p) {
0138                 ret = -ENOMEM;
0139                 goto out;
0140             }
0141             if (strlen(p) > NAME_MAX) {
0142                 pr_err("map_device: Device path too long\n");
0143                 ret = -EINVAL;
0144                 kfree(p);
0145                 goto out;
0146             }
0147             strscpy(opt->pathname, p, NAME_MAX);
0148             kfree(p);
0149             break;
0150 
0151         case RNBD_OPT_DEST_PORT:
0152             if (match_int(args, &dest_port) || dest_port < 0 ||
0153                 dest_port > 65535) {
0154                 pr_err("bad destination port number parameter '%d'\n",
0155                        dest_port);
0156                 ret = -EINVAL;
0157                 goto out;
0158             }
0159             *opt->dest_port = dest_port;
0160             break;
0161 
0162         case RNBD_OPT_ACCESS_MODE:
0163             p = match_strdup(args);
0164             if (!p) {
0165                 ret = -ENOMEM;
0166                 goto out;
0167             }
0168 
0169             if (!strcmp(p, "ro")) {
0170                 *opt->access_mode = RNBD_ACCESS_RO;
0171             } else if (!strcmp(p, "rw")) {
0172                 *opt->access_mode = RNBD_ACCESS_RW;
0173             } else if (!strcmp(p, "migration")) {
0174                 *opt->access_mode = RNBD_ACCESS_MIGRATION;
0175             } else {
0176                 pr_err("map_device: Invalid access_mode: '%s'\n",
0177                        p);
0178                 ret = -EINVAL;
0179                 kfree(p);
0180                 goto out;
0181             }
0182 
0183             kfree(p);
0184             break;
0185 
0186         case RNBD_OPT_NR_POLL_QUEUES:
0187             if (match_int(args, &nr_poll_queues) || nr_poll_queues < -1 ||
0188                 nr_poll_queues > (int)nr_cpu_ids) {
0189                 pr_err("bad nr_poll_queues parameter '%d'\n",
0190                        nr_poll_queues);
0191                 ret = -EINVAL;
0192                 goto out;
0193             }
0194             if (nr_poll_queues == -1)
0195                 nr_poll_queues = nr_cpu_ids;
0196             *opt->nr_poll_queues = nr_poll_queues;
0197             break;
0198 
0199         default:
0200             pr_err("map_device: Unknown parameter or missing value '%s'\n",
0201                    p);
0202             ret = -EINVAL;
0203             goto out;
0204         }
0205     }
0206 
0207     for (i = 0; i < ARRAY_SIZE(rnbd_opt_mandatory); i++) {
0208         if ((opt_mask & rnbd_opt_mandatory[i])) {
0209             ret = 0;
0210         } else {
0211             pr_err("map_device: Parameters missing\n");
0212             ret = -EINVAL;
0213             break;
0214         }
0215     }
0216 
0217 out:
0218     *opt->path_cnt = p_cnt;
0219     kfree(options);
0220     return ret;
0221 }
0222 
0223 static ssize_t state_show(struct kobject *kobj,
0224               struct kobj_attribute *attr, char *page)
0225 {
0226     struct rnbd_clt_dev *dev;
0227 
0228     dev = container_of(kobj, struct rnbd_clt_dev, kobj);
0229 
0230     switch (dev->dev_state) {
0231     case DEV_STATE_INIT:
0232         return sysfs_emit(page, "init\n");
0233     case DEV_STATE_MAPPED:
0234         /* TODO fix cli tool before changing to proper state */
0235         return sysfs_emit(page, "open\n");
0236     case DEV_STATE_MAPPED_DISCONNECTED:
0237         /* TODO fix cli tool before changing to proper state */
0238         return sysfs_emit(page, "closed\n");
0239     case DEV_STATE_UNMAPPED:
0240         return sysfs_emit(page, "unmapped\n");
0241     default:
0242         return sysfs_emit(page, "unknown\n");
0243     }
0244 }
0245 
0246 static struct kobj_attribute rnbd_clt_state_attr = __ATTR_RO(state);
0247 
0248 static ssize_t nr_poll_queues_show(struct kobject *kobj,
0249                    struct kobj_attribute *attr, char *page)
0250 {
0251     struct rnbd_clt_dev *dev;
0252 
0253     dev = container_of(kobj, struct rnbd_clt_dev, kobj);
0254 
0255     return sysfs_emit(page, "%d\n", dev->nr_poll_queues);
0256 }
0257 
0258 static struct kobj_attribute rnbd_clt_nr_poll_queues =
0259     __ATTR_RO(nr_poll_queues);
0260 
0261 static ssize_t mapping_path_show(struct kobject *kobj,
0262                  struct kobj_attribute *attr, char *page)
0263 {
0264     struct rnbd_clt_dev *dev;
0265 
0266     dev = container_of(kobj, struct rnbd_clt_dev, kobj);
0267 
0268     return sysfs_emit(page, "%s\n", dev->pathname);
0269 }
0270 
0271 static struct kobj_attribute rnbd_clt_mapping_path_attr =
0272     __ATTR_RO(mapping_path);
0273 
0274 static ssize_t access_mode_show(struct kobject *kobj,
0275                 struct kobj_attribute *attr, char *page)
0276 {
0277     struct rnbd_clt_dev *dev;
0278 
0279     dev = container_of(kobj, struct rnbd_clt_dev, kobj);
0280 
0281     return sysfs_emit(page, "%s\n", rnbd_access_mode_str(dev->access_mode));
0282 }
0283 
0284 static struct kobj_attribute rnbd_clt_access_mode =
0285     __ATTR_RO(access_mode);
0286 
0287 static ssize_t rnbd_clt_unmap_dev_show(struct kobject *kobj,
0288                     struct kobj_attribute *attr, char *page)
0289 {
0290     return sysfs_emit(page, "Usage: echo <normal|force> > %s\n",
0291               attr->attr.name);
0292 }
0293 
0294 static ssize_t rnbd_clt_unmap_dev_store(struct kobject *kobj,
0295                      struct kobj_attribute *attr,
0296                      const char *buf, size_t count)
0297 {
0298     struct rnbd_clt_dev *dev;
0299     char *opt, *options;
0300     bool force;
0301     int err;
0302 
0303     opt = kstrdup(buf, GFP_KERNEL);
0304     if (!opt)
0305         return -ENOMEM;
0306 
0307     options = strstrip(opt);
0308     dev = container_of(kobj, struct rnbd_clt_dev, kobj);
0309     if (sysfs_streq(options, "normal")) {
0310         force = false;
0311     } else if (sysfs_streq(options, "force")) {
0312         force = true;
0313     } else {
0314         rnbd_clt_err(dev,
0315                   "unmap_device: Invalid value: %s\n",
0316                   options);
0317         err = -EINVAL;
0318         goto out;
0319     }
0320 
0321     rnbd_clt_info(dev, "Unmapping device, option: %s.\n",
0322                force ? "force" : "normal");
0323 
0324     /*
0325      * We take explicit module reference only for one reason: do not
0326      * race with lockless rnbd_destroy_sessions().
0327      */
0328     if (!try_module_get(THIS_MODULE)) {
0329         err = -ENODEV;
0330         goto out;
0331     }
0332     err = rnbd_clt_unmap_device(dev, force, &attr->attr);
0333     if (err) {
0334         if (err != -EALREADY)
0335             rnbd_clt_err(dev, "unmap_device: %d\n",  err);
0336         goto module_put;
0337     }
0338 
0339     /*
0340      * Here device can be vanished!
0341      */
0342 
0343     err = count;
0344 
0345 module_put:
0346     module_put(THIS_MODULE);
0347 out:
0348     kfree(opt);
0349 
0350     return err;
0351 }
0352 
0353 static struct kobj_attribute rnbd_clt_unmap_device_attr =
0354     __ATTR(unmap_device, 0644, rnbd_clt_unmap_dev_show,
0355            rnbd_clt_unmap_dev_store);
0356 
0357 static ssize_t rnbd_clt_resize_dev_show(struct kobject *kobj,
0358                      struct kobj_attribute *attr,
0359                      char *page)
0360 {
0361     return sysfs_emit(page, "Usage: echo <new size in sectors> > %s\n",
0362               attr->attr.name);
0363 }
0364 
0365 static ssize_t rnbd_clt_resize_dev_store(struct kobject *kobj,
0366                       struct kobj_attribute *attr,
0367                       const char *buf, size_t count)
0368 {
0369     int ret;
0370     unsigned long sectors;
0371     struct rnbd_clt_dev *dev;
0372 
0373     dev = container_of(kobj, struct rnbd_clt_dev, kobj);
0374 
0375     ret = kstrtoul(buf, 0, &sectors);
0376     if (ret)
0377         return ret;
0378 
0379     ret = rnbd_clt_resize_disk(dev, sectors);
0380     if (ret)
0381         return ret;
0382 
0383     return count;
0384 }
0385 
0386 static struct kobj_attribute rnbd_clt_resize_dev_attr =
0387     __ATTR(resize, 0644, rnbd_clt_resize_dev_show,
0388            rnbd_clt_resize_dev_store);
0389 
0390 static ssize_t rnbd_clt_remap_dev_show(struct kobject *kobj,
0391                     struct kobj_attribute *attr, char *page)
0392 {
0393     return sysfs_emit(page, "Usage: echo <1> > %s\n", attr->attr.name);
0394 }
0395 
0396 static ssize_t rnbd_clt_remap_dev_store(struct kobject *kobj,
0397                      struct kobj_attribute *attr,
0398                      const char *buf, size_t count)
0399 {
0400     struct rnbd_clt_dev *dev;
0401     char *opt, *options;
0402     int err;
0403 
0404     opt = kstrdup(buf, GFP_KERNEL);
0405     if (!opt)
0406         return -ENOMEM;
0407 
0408     options = strstrip(opt);
0409     dev = container_of(kobj, struct rnbd_clt_dev, kobj);
0410     if (!sysfs_streq(options, "1")) {
0411         rnbd_clt_err(dev,
0412                   "remap_device: Invalid value: %s\n",
0413                   options);
0414         err = -EINVAL;
0415         goto out;
0416     }
0417     err = rnbd_clt_remap_device(dev);
0418     if (likely(!err))
0419         err = count;
0420 
0421 out:
0422     kfree(opt);
0423 
0424     return err;
0425 }
0426 
0427 static struct kobj_attribute rnbd_clt_remap_device_attr =
0428     __ATTR(remap_device, 0644, rnbd_clt_remap_dev_show,
0429            rnbd_clt_remap_dev_store);
0430 
0431 static ssize_t session_show(struct kobject *kobj, struct kobj_attribute *attr,
0432                 char *page)
0433 {
0434     struct rnbd_clt_dev *dev;
0435 
0436     dev = container_of(kobj, struct rnbd_clt_dev, kobj);
0437 
0438     return sysfs_emit(page, "%s\n", dev->sess->sessname);
0439 }
0440 
0441 static struct kobj_attribute rnbd_clt_session_attr =
0442     __ATTR_RO(session);
0443 
0444 static struct attribute *rnbd_dev_attrs[] = {
0445     &rnbd_clt_unmap_device_attr.attr,
0446     &rnbd_clt_resize_dev_attr.attr,
0447     &rnbd_clt_remap_device_attr.attr,
0448     &rnbd_clt_mapping_path_attr.attr,
0449     &rnbd_clt_state_attr.attr,
0450     &rnbd_clt_session_attr.attr,
0451     &rnbd_clt_access_mode.attr,
0452     &rnbd_clt_nr_poll_queues.attr,
0453     NULL,
0454 };
0455 ATTRIBUTE_GROUPS(rnbd_dev);
0456 
0457 void rnbd_clt_remove_dev_symlink(struct rnbd_clt_dev *dev)
0458 {
0459     /*
0460      * The module unload rnbd_client_exit path is racing with unmapping of
0461      * the last single device from the sysfs manually
0462      * i.e. rnbd_clt_unmap_dev_store() leading to a sysfs warning because
0463      * of sysfs link already was removed already.
0464      */
0465     if (dev->blk_symlink_name) {
0466         if (try_module_get(THIS_MODULE)) {
0467             sysfs_remove_link(rnbd_devs_kobj, dev->blk_symlink_name);
0468             module_put(THIS_MODULE);
0469         }
0470         /* It should be freed always. */
0471         kfree(dev->blk_symlink_name);
0472         dev->blk_symlink_name = NULL;
0473     }
0474 }
0475 
0476 static struct kobj_type rnbd_dev_ktype = {
0477     .sysfs_ops      = &kobj_sysfs_ops,
0478     .default_groups = rnbd_dev_groups,
0479 };
0480 
0481 static int rnbd_clt_add_dev_kobj(struct rnbd_clt_dev *dev)
0482 {
0483     int ret;
0484     struct kobject *gd_kobj = &disk_to_dev(dev->gd)->kobj;
0485 
0486     ret = kobject_init_and_add(&dev->kobj, &rnbd_dev_ktype, gd_kobj, "%s",
0487                    "rnbd");
0488     if (ret) {
0489         rnbd_clt_err(dev, "Failed to create device sysfs dir, err: %d\n",
0490                   ret);
0491         kobject_put(&dev->kobj);
0492     }
0493     kobject_uevent(gd_kobj, KOBJ_ONLINE);
0494 
0495     return ret;
0496 }
0497 
0498 static ssize_t rnbd_clt_map_device_show(struct kobject *kobj,
0499                      struct kobj_attribute *attr,
0500                      char *page)
0501 {
0502     return sysfs_emit(page,
0503               "Usage: echo \"[dest_port=server port number] sessname=<name of the rtrs session> path=<[srcaddr@]dstaddr> [path=<[srcaddr@]dstaddr>] device_path=<full path on remote side> [access_mode=<ro|rw|migration>] [nr_poll_queues=<number of queues>]\" > %s\n\naddr ::= [ ip:<ipv4> | ip:<ipv6> | gid:<gid> ]\n",
0504              attr->attr.name);
0505 }
0506 
0507 static int rnbd_clt_get_path_name(struct rnbd_clt_dev *dev, char *buf,
0508                    size_t len)
0509 {
0510     int ret;
0511     char pathname[NAME_MAX], *s;
0512 
0513     strscpy(pathname, dev->pathname, sizeof(pathname));
0514     while ((s = strchr(pathname, '/')))
0515         s[0] = '!';
0516 
0517     ret = snprintf(buf, len, "%s@%s", pathname, dev->sess->sessname);
0518     if (ret >= len)
0519         return -ENAMETOOLONG;
0520 
0521     return 0;
0522 }
0523 
0524 static int rnbd_clt_add_dev_symlink(struct rnbd_clt_dev *dev)
0525 {
0526     struct kobject *gd_kobj = &disk_to_dev(dev->gd)->kobj;
0527     int ret, len;
0528 
0529     len = strlen(dev->pathname) + strlen(dev->sess->sessname) + 2;
0530     dev->blk_symlink_name = kzalloc(len, GFP_KERNEL);
0531     if (!dev->blk_symlink_name) {
0532         rnbd_clt_err(dev, "Failed to allocate memory for blk_symlink_name\n");
0533         return -ENOMEM;
0534     }
0535 
0536     ret = rnbd_clt_get_path_name(dev, dev->blk_symlink_name,
0537                       len);
0538     if (ret) {
0539         rnbd_clt_err(dev, "Failed to get /sys/block symlink path, err: %d\n",
0540                   ret);
0541         goto out_err;
0542     }
0543 
0544     ret = sysfs_create_link(rnbd_devs_kobj, gd_kobj,
0545                 dev->blk_symlink_name);
0546     if (ret) {
0547         rnbd_clt_err(dev, "Creating /sys/block symlink failed, err: %d\n",
0548                   ret);
0549         goto out_err;
0550     }
0551 
0552     return 0;
0553 
0554 out_err:
0555     kfree(dev->blk_symlink_name);
0556     dev->blk_symlink_name = NULL ;
0557     return ret;
0558 }
0559 
0560 static ssize_t rnbd_clt_map_device_store(struct kobject *kobj,
0561                       struct kobj_attribute *attr,
0562                       const char *buf, size_t count)
0563 {
0564     struct rnbd_clt_dev *dev;
0565     struct rnbd_map_options opt;
0566     int ret;
0567     char pathname[NAME_MAX];
0568     char sessname[NAME_MAX];
0569     enum rnbd_access_mode access_mode = RNBD_ACCESS_RW;
0570     u16 port_nr = RTRS_PORT;
0571     u32 nr_poll_queues = 0;
0572 
0573     struct sockaddr_storage *addrs;
0574     struct rtrs_addr paths[6];
0575     size_t path_cnt;
0576 
0577     opt.sessname = sessname;
0578     opt.paths = paths;
0579     opt.path_cnt = &path_cnt;
0580     opt.pathname = pathname;
0581     opt.dest_port = &port_nr;
0582     opt.access_mode = &access_mode;
0583     opt.nr_poll_queues = &nr_poll_queues;
0584     addrs = kcalloc(ARRAY_SIZE(paths) * 2, sizeof(*addrs), GFP_KERNEL);
0585     if (!addrs)
0586         return -ENOMEM;
0587 
0588     for (path_cnt = 0; path_cnt < ARRAY_SIZE(paths); path_cnt++) {
0589         paths[path_cnt].src = &addrs[path_cnt * 2];
0590         paths[path_cnt].dst = &addrs[path_cnt * 2 + 1];
0591     }
0592 
0593     ret = rnbd_clt_parse_map_options(buf, ARRAY_SIZE(paths), &opt);
0594     if (ret)
0595         goto out;
0596 
0597     pr_info("Mapping device %s on session %s, (access_mode: %s, nr_poll_queues: %d)\n",
0598         pathname, sessname,
0599         rnbd_access_mode_str(access_mode),
0600         nr_poll_queues);
0601 
0602     dev = rnbd_clt_map_device(sessname, paths, path_cnt, port_nr, pathname,
0603                   access_mode, nr_poll_queues);
0604     if (IS_ERR(dev)) {
0605         ret = PTR_ERR(dev);
0606         goto out;
0607     }
0608 
0609     ret = rnbd_clt_add_dev_kobj(dev);
0610     if (ret)
0611         goto unmap_dev;
0612 
0613     ret = rnbd_clt_add_dev_symlink(dev);
0614     if (ret)
0615         goto unmap_dev;
0616 
0617     kfree(addrs);
0618     return count;
0619 
0620 unmap_dev:
0621     rnbd_clt_unmap_device(dev, true, NULL);
0622 out:
0623     kfree(addrs);
0624     return ret;
0625 }
0626 
0627 static struct kobj_attribute rnbd_clt_map_device_attr =
0628     __ATTR(map_device, 0644,
0629            rnbd_clt_map_device_show, rnbd_clt_map_device_store);
0630 
0631 static struct attribute *default_attrs[] = {
0632     &rnbd_clt_map_device_attr.attr,
0633     NULL,
0634 };
0635 
0636 static struct attribute_group default_attr_group = {
0637     .attrs = default_attrs,
0638 };
0639 
0640 static const struct attribute_group *default_attr_groups[] = {
0641     &default_attr_group,
0642     NULL,
0643 };
0644 
0645 int rnbd_clt_create_sysfs_files(void)
0646 {
0647     int err;
0648 
0649     rnbd_dev_class = class_create(THIS_MODULE, "rnbd-client");
0650     if (IS_ERR(rnbd_dev_class))
0651         return PTR_ERR(rnbd_dev_class);
0652 
0653     rnbd_dev = device_create_with_groups(rnbd_dev_class, NULL,
0654                           MKDEV(0, 0), NULL,
0655                           default_attr_groups, "ctl");
0656     if (IS_ERR(rnbd_dev)) {
0657         err = PTR_ERR(rnbd_dev);
0658         goto cls_destroy;
0659     }
0660     rnbd_devs_kobj = kobject_create_and_add("devices", &rnbd_dev->kobj);
0661     if (!rnbd_devs_kobj) {
0662         err = -ENOMEM;
0663         goto dev_destroy;
0664     }
0665 
0666     return 0;
0667 
0668 dev_destroy:
0669     device_destroy(rnbd_dev_class, MKDEV(0, 0));
0670 cls_destroy:
0671     class_destroy(rnbd_dev_class);
0672 
0673     return err;
0674 }
0675 
0676 void rnbd_clt_destroy_sysfs_files(void)
0677 {
0678     sysfs_remove_group(&rnbd_dev->kobj, &default_attr_group);
0679     kobject_del(rnbd_devs_kobj);
0680     kobject_put(rnbd_devs_kobj);
0681     device_destroy(rnbd_dev_class, MKDEV(0, 0));
0682     class_destroy(rnbd_dev_class);
0683 }