Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * NVMe over Fabrics loopback device.
0004  * Copyright (c) 2015-2016 HGST, a Western Digital Company.
0005  */
0006 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
0007 #include <linux/scatterlist.h>
0008 #include <linux/blk-mq.h>
0009 #include <linux/nvme.h>
0010 #include <linux/module.h>
0011 #include <linux/parser.h>
0012 #include "nvmet.h"
0013 #include "../host/nvme.h"
0014 #include "../host/fabrics.h"
0015 
0016 #define NVME_LOOP_MAX_SEGMENTS      256
0017 
0018 struct nvme_loop_iod {
0019     struct nvme_request nvme_req;
0020     struct nvme_command cmd;
0021     struct nvme_completion  cqe;
0022     struct nvmet_req    req;
0023     struct nvme_loop_queue  *queue;
0024     struct work_struct  work;
0025     struct sg_table     sg_table;
0026     struct scatterlist  first_sgl[];
0027 };
0028 
0029 struct nvme_loop_ctrl {
0030     struct nvme_loop_queue  *queues;
0031 
0032     struct blk_mq_tag_set   admin_tag_set;
0033 
0034     struct list_head    list;
0035     struct blk_mq_tag_set   tag_set;
0036     struct nvme_loop_iod    async_event_iod;
0037     struct nvme_ctrl    ctrl;
0038 
0039     struct nvmet_port   *port;
0040 };
0041 
0042 static inline struct nvme_loop_ctrl *to_loop_ctrl(struct nvme_ctrl *ctrl)
0043 {
0044     return container_of(ctrl, struct nvme_loop_ctrl, ctrl);
0045 }
0046 
0047 enum nvme_loop_queue_flags {
0048     NVME_LOOP_Q_LIVE    = 0,
0049 };
0050 
0051 struct nvme_loop_queue {
0052     struct nvmet_cq     nvme_cq;
0053     struct nvmet_sq     nvme_sq;
0054     struct nvme_loop_ctrl   *ctrl;
0055     unsigned long       flags;
0056 };
0057 
0058 static LIST_HEAD(nvme_loop_ports);
0059 static DEFINE_MUTEX(nvme_loop_ports_mutex);
0060 
0061 static LIST_HEAD(nvme_loop_ctrl_list);
0062 static DEFINE_MUTEX(nvme_loop_ctrl_mutex);
0063 
0064 static void nvme_loop_queue_response(struct nvmet_req *nvme_req);
0065 static void nvme_loop_delete_ctrl(struct nvmet_ctrl *ctrl);
0066 
0067 static const struct nvmet_fabrics_ops nvme_loop_ops;
0068 
0069 static inline int nvme_loop_queue_idx(struct nvme_loop_queue *queue)
0070 {
0071     return queue - queue->ctrl->queues;
0072 }
0073 
0074 static void nvme_loop_complete_rq(struct request *req)
0075 {
0076     struct nvme_loop_iod *iod = blk_mq_rq_to_pdu(req);
0077 
0078     sg_free_table_chained(&iod->sg_table, NVME_INLINE_SG_CNT);
0079     nvme_complete_rq(req);
0080 }
0081 
0082 static struct blk_mq_tags *nvme_loop_tagset(struct nvme_loop_queue *queue)
0083 {
0084     u32 queue_idx = nvme_loop_queue_idx(queue);
0085 
0086     if (queue_idx == 0)
0087         return queue->ctrl->admin_tag_set.tags[queue_idx];
0088     return queue->ctrl->tag_set.tags[queue_idx - 1];
0089 }
0090 
0091 static void nvme_loop_queue_response(struct nvmet_req *req)
0092 {
0093     struct nvme_loop_queue *queue =
0094         container_of(req->sq, struct nvme_loop_queue, nvme_sq);
0095     struct nvme_completion *cqe = req->cqe;
0096 
0097     /*
0098      * AEN requests are special as they don't time out and can
0099      * survive any kind of queue freeze and often don't respond to
0100      * aborts.  We don't even bother to allocate a struct request
0101      * for them but rather special case them here.
0102      */
0103     if (unlikely(nvme_is_aen_req(nvme_loop_queue_idx(queue),
0104                      cqe->command_id))) {
0105         nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status,
0106                 &cqe->result);
0107     } else {
0108         struct request *rq;
0109 
0110         rq = nvme_find_rq(nvme_loop_tagset(queue), cqe->command_id);
0111         if (!rq) {
0112             dev_err(queue->ctrl->ctrl.device,
0113                 "got bad command_id %#x on queue %d\n",
0114                 cqe->command_id, nvme_loop_queue_idx(queue));
0115             return;
0116         }
0117 
0118         if (!nvme_try_complete_req(rq, cqe->status, cqe->result))
0119             nvme_loop_complete_rq(rq);
0120     }
0121 }
0122 
0123 static void nvme_loop_execute_work(struct work_struct *work)
0124 {
0125     struct nvme_loop_iod *iod =
0126         container_of(work, struct nvme_loop_iod, work);
0127 
0128     iod->req.execute(&iod->req);
0129 }
0130 
0131 static blk_status_t nvme_loop_queue_rq(struct blk_mq_hw_ctx *hctx,
0132         const struct blk_mq_queue_data *bd)
0133 {
0134     struct nvme_ns *ns = hctx->queue->queuedata;
0135     struct nvme_loop_queue *queue = hctx->driver_data;
0136     struct request *req = bd->rq;
0137     struct nvme_loop_iod *iod = blk_mq_rq_to_pdu(req);
0138     bool queue_ready = test_bit(NVME_LOOP_Q_LIVE, &queue->flags);
0139     blk_status_t ret;
0140 
0141     if (!nvme_check_ready(&queue->ctrl->ctrl, req, queue_ready))
0142         return nvme_fail_nonready_command(&queue->ctrl->ctrl, req);
0143 
0144     ret = nvme_setup_cmd(ns, req);
0145     if (ret)
0146         return ret;
0147 
0148     blk_mq_start_request(req);
0149     iod->cmd.common.flags |= NVME_CMD_SGL_METABUF;
0150     iod->req.port = queue->ctrl->port;
0151     if (!nvmet_req_init(&iod->req, &queue->nvme_cq,
0152             &queue->nvme_sq, &nvme_loop_ops))
0153         return BLK_STS_OK;
0154 
0155     if (blk_rq_nr_phys_segments(req)) {
0156         iod->sg_table.sgl = iod->first_sgl;
0157         if (sg_alloc_table_chained(&iod->sg_table,
0158                 blk_rq_nr_phys_segments(req),
0159                 iod->sg_table.sgl, NVME_INLINE_SG_CNT)) {
0160             nvme_cleanup_cmd(req);
0161             return BLK_STS_RESOURCE;
0162         }
0163 
0164         iod->req.sg = iod->sg_table.sgl;
0165         iod->req.sg_cnt = blk_rq_map_sg(req->q, req, iod->sg_table.sgl);
0166         iod->req.transfer_len = blk_rq_payload_bytes(req);
0167     }
0168 
0169     queue_work(nvmet_wq, &iod->work);
0170     return BLK_STS_OK;
0171 }
0172 
0173 static void nvme_loop_submit_async_event(struct nvme_ctrl *arg)
0174 {
0175     struct nvme_loop_ctrl *ctrl = to_loop_ctrl(arg);
0176     struct nvme_loop_queue *queue = &ctrl->queues[0];
0177     struct nvme_loop_iod *iod = &ctrl->async_event_iod;
0178 
0179     memset(&iod->cmd, 0, sizeof(iod->cmd));
0180     iod->cmd.common.opcode = nvme_admin_async_event;
0181     iod->cmd.common.command_id = NVME_AQ_BLK_MQ_DEPTH;
0182     iod->cmd.common.flags |= NVME_CMD_SGL_METABUF;
0183 
0184     if (!nvmet_req_init(&iod->req, &queue->nvme_cq, &queue->nvme_sq,
0185             &nvme_loop_ops)) {
0186         dev_err(ctrl->ctrl.device, "failed async event work\n");
0187         return;
0188     }
0189 
0190     queue_work(nvmet_wq, &iod->work);
0191 }
0192 
0193 static int nvme_loop_init_iod(struct nvme_loop_ctrl *ctrl,
0194         struct nvme_loop_iod *iod, unsigned int queue_idx)
0195 {
0196     iod->req.cmd = &iod->cmd;
0197     iod->req.cqe = &iod->cqe;
0198     iod->queue = &ctrl->queues[queue_idx];
0199     INIT_WORK(&iod->work, nvme_loop_execute_work);
0200     return 0;
0201 }
0202 
0203 static int nvme_loop_init_request(struct blk_mq_tag_set *set,
0204         struct request *req, unsigned int hctx_idx,
0205         unsigned int numa_node)
0206 {
0207     struct nvme_loop_ctrl *ctrl = set->driver_data;
0208     struct nvme_loop_iod *iod = blk_mq_rq_to_pdu(req);
0209 
0210     nvme_req(req)->ctrl = &ctrl->ctrl;
0211     nvme_req(req)->cmd = &iod->cmd;
0212     return nvme_loop_init_iod(ctrl, blk_mq_rq_to_pdu(req),
0213             (set == &ctrl->tag_set) ? hctx_idx + 1 : 0);
0214 }
0215 
0216 static struct lock_class_key loop_hctx_fq_lock_key;
0217 
0218 static int nvme_loop_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
0219         unsigned int hctx_idx)
0220 {
0221     struct nvme_loop_ctrl *ctrl = data;
0222     struct nvme_loop_queue *queue = &ctrl->queues[hctx_idx + 1];
0223 
0224     BUG_ON(hctx_idx >= ctrl->ctrl.queue_count);
0225 
0226     /*
0227      * flush_end_io() can be called recursively for us, so use our own
0228      * lock class key for avoiding lockdep possible recursive locking,
0229      * then we can remove the dynamically allocated lock class for each
0230      * flush queue, that way may cause horrible boot delay.
0231      */
0232     blk_mq_hctx_set_fq_lock_class(hctx, &loop_hctx_fq_lock_key);
0233 
0234     hctx->driver_data = queue;
0235     return 0;
0236 }
0237 
0238 static int nvme_loop_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data,
0239         unsigned int hctx_idx)
0240 {
0241     struct nvme_loop_ctrl *ctrl = data;
0242     struct nvme_loop_queue *queue = &ctrl->queues[0];
0243 
0244     BUG_ON(hctx_idx != 0);
0245 
0246     hctx->driver_data = queue;
0247     return 0;
0248 }
0249 
0250 static const struct blk_mq_ops nvme_loop_mq_ops = {
0251     .queue_rq   = nvme_loop_queue_rq,
0252     .complete   = nvme_loop_complete_rq,
0253     .init_request   = nvme_loop_init_request,
0254     .init_hctx  = nvme_loop_init_hctx,
0255 };
0256 
0257 static const struct blk_mq_ops nvme_loop_admin_mq_ops = {
0258     .queue_rq   = nvme_loop_queue_rq,
0259     .complete   = nvme_loop_complete_rq,
0260     .init_request   = nvme_loop_init_request,
0261     .init_hctx  = nvme_loop_init_admin_hctx,
0262 };
0263 
0264 static void nvme_loop_destroy_admin_queue(struct nvme_loop_ctrl *ctrl)
0265 {
0266     if (!test_and_clear_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[0].flags))
0267         return;
0268     nvmet_sq_destroy(&ctrl->queues[0].nvme_sq);
0269     blk_mq_destroy_queue(ctrl->ctrl.admin_q);
0270     blk_mq_destroy_queue(ctrl->ctrl.fabrics_q);
0271     blk_mq_free_tag_set(&ctrl->admin_tag_set);
0272 }
0273 
0274 static void nvme_loop_free_ctrl(struct nvme_ctrl *nctrl)
0275 {
0276     struct nvme_loop_ctrl *ctrl = to_loop_ctrl(nctrl);
0277 
0278     if (list_empty(&ctrl->list))
0279         goto free_ctrl;
0280 
0281     mutex_lock(&nvme_loop_ctrl_mutex);
0282     list_del(&ctrl->list);
0283     mutex_unlock(&nvme_loop_ctrl_mutex);
0284 
0285     if (nctrl->tagset) {
0286         blk_mq_destroy_queue(ctrl->ctrl.connect_q);
0287         blk_mq_free_tag_set(&ctrl->tag_set);
0288     }
0289     kfree(ctrl->queues);
0290     nvmf_free_options(nctrl->opts);
0291 free_ctrl:
0292     kfree(ctrl);
0293 }
0294 
0295 static void nvme_loop_destroy_io_queues(struct nvme_loop_ctrl *ctrl)
0296 {
0297     int i;
0298 
0299     for (i = 1; i < ctrl->ctrl.queue_count; i++) {
0300         clear_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[i].flags);
0301         nvmet_sq_destroy(&ctrl->queues[i].nvme_sq);
0302     }
0303     ctrl->ctrl.queue_count = 1;
0304 }
0305 
0306 static int nvme_loop_init_io_queues(struct nvme_loop_ctrl *ctrl)
0307 {
0308     struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
0309     unsigned int nr_io_queues;
0310     int ret, i;
0311 
0312     nr_io_queues = min(opts->nr_io_queues, num_online_cpus());
0313     ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues);
0314     if (ret || !nr_io_queues)
0315         return ret;
0316 
0317     dev_info(ctrl->ctrl.device, "creating %d I/O queues.\n", nr_io_queues);
0318 
0319     for (i = 1; i <= nr_io_queues; i++) {
0320         ctrl->queues[i].ctrl = ctrl;
0321         ret = nvmet_sq_init(&ctrl->queues[i].nvme_sq);
0322         if (ret)
0323             goto out_destroy_queues;
0324 
0325         ctrl->ctrl.queue_count++;
0326     }
0327 
0328     return 0;
0329 
0330 out_destroy_queues:
0331     nvme_loop_destroy_io_queues(ctrl);
0332     return ret;
0333 }
0334 
0335 static int nvme_loop_connect_io_queues(struct nvme_loop_ctrl *ctrl)
0336 {
0337     int i, ret;
0338 
0339     for (i = 1; i < ctrl->ctrl.queue_count; i++) {
0340         ret = nvmf_connect_io_queue(&ctrl->ctrl, i);
0341         if (ret)
0342             return ret;
0343         set_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[i].flags);
0344     }
0345 
0346     return 0;
0347 }
0348 
0349 static int nvme_loop_configure_admin_queue(struct nvme_loop_ctrl *ctrl)
0350 {
0351     int error;
0352 
0353     memset(&ctrl->admin_tag_set, 0, sizeof(ctrl->admin_tag_set));
0354     ctrl->admin_tag_set.ops = &nvme_loop_admin_mq_ops;
0355     ctrl->admin_tag_set.queue_depth = NVME_AQ_MQ_TAG_DEPTH;
0356     ctrl->admin_tag_set.reserved_tags = NVMF_RESERVED_TAGS;
0357     ctrl->admin_tag_set.numa_node = ctrl->ctrl.numa_node;
0358     ctrl->admin_tag_set.cmd_size = sizeof(struct nvme_loop_iod) +
0359         NVME_INLINE_SG_CNT * sizeof(struct scatterlist);
0360     ctrl->admin_tag_set.driver_data = ctrl;
0361     ctrl->admin_tag_set.nr_hw_queues = 1;
0362     ctrl->admin_tag_set.timeout = NVME_ADMIN_TIMEOUT;
0363     ctrl->admin_tag_set.flags = BLK_MQ_F_NO_SCHED;
0364 
0365     ctrl->queues[0].ctrl = ctrl;
0366     error = nvmet_sq_init(&ctrl->queues[0].nvme_sq);
0367     if (error)
0368         return error;
0369     ctrl->ctrl.queue_count = 1;
0370 
0371     error = blk_mq_alloc_tag_set(&ctrl->admin_tag_set);
0372     if (error)
0373         goto out_free_sq;
0374     ctrl->ctrl.admin_tagset = &ctrl->admin_tag_set;
0375 
0376     ctrl->ctrl.fabrics_q = blk_mq_init_queue(&ctrl->admin_tag_set);
0377     if (IS_ERR(ctrl->ctrl.fabrics_q)) {
0378         error = PTR_ERR(ctrl->ctrl.fabrics_q);
0379         goto out_free_tagset;
0380     }
0381 
0382     ctrl->ctrl.admin_q = blk_mq_init_queue(&ctrl->admin_tag_set);
0383     if (IS_ERR(ctrl->ctrl.admin_q)) {
0384         error = PTR_ERR(ctrl->ctrl.admin_q);
0385         goto out_cleanup_fabrics_q;
0386     }
0387     /* reset stopped state for the fresh admin queue */
0388     clear_bit(NVME_CTRL_ADMIN_Q_STOPPED, &ctrl->ctrl.flags);
0389 
0390     error = nvmf_connect_admin_queue(&ctrl->ctrl);
0391     if (error)
0392         goto out_cleanup_queue;
0393 
0394     set_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[0].flags);
0395 
0396     error = nvme_enable_ctrl(&ctrl->ctrl);
0397     if (error)
0398         goto out_cleanup_queue;
0399 
0400     ctrl->ctrl.max_hw_sectors =
0401         (NVME_LOOP_MAX_SEGMENTS - 1) << (PAGE_SHIFT - 9);
0402 
0403     nvme_start_admin_queue(&ctrl->ctrl);
0404 
0405     error = nvme_init_ctrl_finish(&ctrl->ctrl);
0406     if (error)
0407         goto out_cleanup_queue;
0408 
0409     return 0;
0410 
0411 out_cleanup_queue:
0412     clear_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[0].flags);
0413     blk_mq_destroy_queue(ctrl->ctrl.admin_q);
0414 out_cleanup_fabrics_q:
0415     blk_mq_destroy_queue(ctrl->ctrl.fabrics_q);
0416 out_free_tagset:
0417     blk_mq_free_tag_set(&ctrl->admin_tag_set);
0418 out_free_sq:
0419     nvmet_sq_destroy(&ctrl->queues[0].nvme_sq);
0420     return error;
0421 }
0422 
0423 static void nvme_loop_shutdown_ctrl(struct nvme_loop_ctrl *ctrl)
0424 {
0425     if (ctrl->ctrl.queue_count > 1) {
0426         nvme_stop_queues(&ctrl->ctrl);
0427         nvme_cancel_tagset(&ctrl->ctrl);
0428         nvme_loop_destroy_io_queues(ctrl);
0429     }
0430 
0431     nvme_stop_admin_queue(&ctrl->ctrl);
0432     if (ctrl->ctrl.state == NVME_CTRL_LIVE)
0433         nvme_shutdown_ctrl(&ctrl->ctrl);
0434 
0435     nvme_cancel_admin_tagset(&ctrl->ctrl);
0436     nvme_loop_destroy_admin_queue(ctrl);
0437 }
0438 
0439 static void nvme_loop_delete_ctrl_host(struct nvme_ctrl *ctrl)
0440 {
0441     nvme_loop_shutdown_ctrl(to_loop_ctrl(ctrl));
0442 }
0443 
0444 static void nvme_loop_delete_ctrl(struct nvmet_ctrl *nctrl)
0445 {
0446     struct nvme_loop_ctrl *ctrl;
0447 
0448     mutex_lock(&nvme_loop_ctrl_mutex);
0449     list_for_each_entry(ctrl, &nvme_loop_ctrl_list, list) {
0450         if (ctrl->ctrl.cntlid == nctrl->cntlid)
0451             nvme_delete_ctrl(&ctrl->ctrl);
0452     }
0453     mutex_unlock(&nvme_loop_ctrl_mutex);
0454 }
0455 
0456 static void nvme_loop_reset_ctrl_work(struct work_struct *work)
0457 {
0458     struct nvme_loop_ctrl *ctrl =
0459         container_of(work, struct nvme_loop_ctrl, ctrl.reset_work);
0460     int ret;
0461 
0462     nvme_stop_ctrl(&ctrl->ctrl);
0463     nvme_loop_shutdown_ctrl(ctrl);
0464 
0465     if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) {
0466         if (ctrl->ctrl.state != NVME_CTRL_DELETING &&
0467             ctrl->ctrl.state != NVME_CTRL_DELETING_NOIO)
0468             /* state change failure for non-deleted ctrl? */
0469             WARN_ON_ONCE(1);
0470         return;
0471     }
0472 
0473     ret = nvme_loop_configure_admin_queue(ctrl);
0474     if (ret)
0475         goto out_disable;
0476 
0477     ret = nvme_loop_init_io_queues(ctrl);
0478     if (ret)
0479         goto out_destroy_admin;
0480 
0481     ret = nvme_loop_connect_io_queues(ctrl);
0482     if (ret)
0483         goto out_destroy_io;
0484 
0485     blk_mq_update_nr_hw_queues(&ctrl->tag_set,
0486             ctrl->ctrl.queue_count - 1);
0487 
0488     if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE))
0489         WARN_ON_ONCE(1);
0490 
0491     nvme_start_ctrl(&ctrl->ctrl);
0492 
0493     return;
0494 
0495 out_destroy_io:
0496     nvme_loop_destroy_io_queues(ctrl);
0497 out_destroy_admin:
0498     nvme_loop_destroy_admin_queue(ctrl);
0499 out_disable:
0500     dev_warn(ctrl->ctrl.device, "Removing after reset failure\n");
0501     nvme_uninit_ctrl(&ctrl->ctrl);
0502 }
0503 
0504 static const struct nvme_ctrl_ops nvme_loop_ctrl_ops = {
0505     .name           = "loop",
0506     .module         = THIS_MODULE,
0507     .flags          = NVME_F_FABRICS,
0508     .reg_read32     = nvmf_reg_read32,
0509     .reg_read64     = nvmf_reg_read64,
0510     .reg_write32        = nvmf_reg_write32,
0511     .free_ctrl      = nvme_loop_free_ctrl,
0512     .submit_async_event = nvme_loop_submit_async_event,
0513     .delete_ctrl        = nvme_loop_delete_ctrl_host,
0514     .get_address        = nvmf_get_address,
0515 };
0516 
0517 static int nvme_loop_create_io_queues(struct nvme_loop_ctrl *ctrl)
0518 {
0519     int ret;
0520 
0521     ret = nvme_loop_init_io_queues(ctrl);
0522     if (ret)
0523         return ret;
0524 
0525     memset(&ctrl->tag_set, 0, sizeof(ctrl->tag_set));
0526     ctrl->tag_set.ops = &nvme_loop_mq_ops;
0527     ctrl->tag_set.queue_depth = ctrl->ctrl.opts->queue_size;
0528     ctrl->tag_set.reserved_tags = NVMF_RESERVED_TAGS;
0529     ctrl->tag_set.numa_node = ctrl->ctrl.numa_node;
0530     ctrl->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;
0531     ctrl->tag_set.cmd_size = sizeof(struct nvme_loop_iod) +
0532         NVME_INLINE_SG_CNT * sizeof(struct scatterlist);
0533     ctrl->tag_set.driver_data = ctrl;
0534     ctrl->tag_set.nr_hw_queues = ctrl->ctrl.queue_count - 1;
0535     ctrl->tag_set.timeout = NVME_IO_TIMEOUT;
0536     ctrl->ctrl.tagset = &ctrl->tag_set;
0537 
0538     ret = blk_mq_alloc_tag_set(&ctrl->tag_set);
0539     if (ret)
0540         goto out_destroy_queues;
0541 
0542     ret = nvme_ctrl_init_connect_q(&(ctrl->ctrl));
0543     if (ret)
0544         goto out_free_tagset;
0545 
0546     ret = nvme_loop_connect_io_queues(ctrl);
0547     if (ret)
0548         goto out_cleanup_connect_q;
0549 
0550     return 0;
0551 
0552 out_cleanup_connect_q:
0553     blk_mq_destroy_queue(ctrl->ctrl.connect_q);
0554 out_free_tagset:
0555     blk_mq_free_tag_set(&ctrl->tag_set);
0556 out_destroy_queues:
0557     nvme_loop_destroy_io_queues(ctrl);
0558     return ret;
0559 }
0560 
0561 static struct nvmet_port *nvme_loop_find_port(struct nvme_ctrl *ctrl)
0562 {
0563     struct nvmet_port *p, *found = NULL;
0564 
0565     mutex_lock(&nvme_loop_ports_mutex);
0566     list_for_each_entry(p, &nvme_loop_ports, entry) {
0567         /* if no transport address is specified use the first port */
0568         if ((ctrl->opts->mask & NVMF_OPT_TRADDR) &&
0569             strcmp(ctrl->opts->traddr, p->disc_addr.traddr))
0570             continue;
0571         found = p;
0572         break;
0573     }
0574     mutex_unlock(&nvme_loop_ports_mutex);
0575     return found;
0576 }
0577 
0578 static struct nvme_ctrl *nvme_loop_create_ctrl(struct device *dev,
0579         struct nvmf_ctrl_options *opts)
0580 {
0581     struct nvme_loop_ctrl *ctrl;
0582     int ret;
0583 
0584     ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);
0585     if (!ctrl)
0586         return ERR_PTR(-ENOMEM);
0587     ctrl->ctrl.opts = opts;
0588     INIT_LIST_HEAD(&ctrl->list);
0589 
0590     INIT_WORK(&ctrl->ctrl.reset_work, nvme_loop_reset_ctrl_work);
0591 
0592     ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_loop_ctrl_ops,
0593                 0 /* no quirks, we're perfect! */);
0594     if (ret) {
0595         kfree(ctrl);
0596         goto out;
0597     }
0598 
0599     if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING))
0600         WARN_ON_ONCE(1);
0601 
0602     ret = -ENOMEM;
0603 
0604     ctrl->ctrl.sqsize = opts->queue_size - 1;
0605     ctrl->ctrl.kato = opts->kato;
0606     ctrl->port = nvme_loop_find_port(&ctrl->ctrl);
0607 
0608     ctrl->queues = kcalloc(opts->nr_io_queues + 1, sizeof(*ctrl->queues),
0609             GFP_KERNEL);
0610     if (!ctrl->queues)
0611         goto out_uninit_ctrl;
0612 
0613     ret = nvme_loop_configure_admin_queue(ctrl);
0614     if (ret)
0615         goto out_free_queues;
0616 
0617     if (opts->queue_size > ctrl->ctrl.maxcmd) {
0618         /* warn if maxcmd is lower than queue_size */
0619         dev_warn(ctrl->ctrl.device,
0620             "queue_size %zu > ctrl maxcmd %u, clamping down\n",
0621             opts->queue_size, ctrl->ctrl.maxcmd);
0622         opts->queue_size = ctrl->ctrl.maxcmd;
0623     }
0624 
0625     if (opts->nr_io_queues) {
0626         ret = nvme_loop_create_io_queues(ctrl);
0627         if (ret)
0628             goto out_remove_admin_queue;
0629     }
0630 
0631     nvme_loop_init_iod(ctrl, &ctrl->async_event_iod, 0);
0632 
0633     dev_info(ctrl->ctrl.device,
0634          "new ctrl: \"%s\"\n", ctrl->ctrl.opts->subsysnqn);
0635 
0636     if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE))
0637         WARN_ON_ONCE(1);
0638 
0639     mutex_lock(&nvme_loop_ctrl_mutex);
0640     list_add_tail(&ctrl->list, &nvme_loop_ctrl_list);
0641     mutex_unlock(&nvme_loop_ctrl_mutex);
0642 
0643     nvme_start_ctrl(&ctrl->ctrl);
0644 
0645     return &ctrl->ctrl;
0646 
0647 out_remove_admin_queue:
0648     nvme_loop_destroy_admin_queue(ctrl);
0649 out_free_queues:
0650     kfree(ctrl->queues);
0651 out_uninit_ctrl:
0652     nvme_uninit_ctrl(&ctrl->ctrl);
0653     nvme_put_ctrl(&ctrl->ctrl);
0654 out:
0655     if (ret > 0)
0656         ret = -EIO;
0657     return ERR_PTR(ret);
0658 }
0659 
0660 static int nvme_loop_add_port(struct nvmet_port *port)
0661 {
0662     mutex_lock(&nvme_loop_ports_mutex);
0663     list_add_tail(&port->entry, &nvme_loop_ports);
0664     mutex_unlock(&nvme_loop_ports_mutex);
0665     return 0;
0666 }
0667 
0668 static void nvme_loop_remove_port(struct nvmet_port *port)
0669 {
0670     mutex_lock(&nvme_loop_ports_mutex);
0671     list_del_init(&port->entry);
0672     mutex_unlock(&nvme_loop_ports_mutex);
0673 
0674     /*
0675      * Ensure any ctrls that are in the process of being
0676      * deleted are in fact deleted before we return
0677      * and free the port. This is to prevent active
0678      * ctrls from using a port after it's freed.
0679      */
0680     flush_workqueue(nvme_delete_wq);
0681 }
0682 
0683 static const struct nvmet_fabrics_ops nvme_loop_ops = {
0684     .owner      = THIS_MODULE,
0685     .type       = NVMF_TRTYPE_LOOP,
0686     .add_port   = nvme_loop_add_port,
0687     .remove_port    = nvme_loop_remove_port,
0688     .queue_response = nvme_loop_queue_response,
0689     .delete_ctrl    = nvme_loop_delete_ctrl,
0690 };
0691 
0692 static struct nvmf_transport_ops nvme_loop_transport = {
0693     .name       = "loop",
0694     .module     = THIS_MODULE,
0695     .create_ctrl    = nvme_loop_create_ctrl,
0696     .allowed_opts   = NVMF_OPT_TRADDR,
0697 };
0698 
0699 static int __init nvme_loop_init_module(void)
0700 {
0701     int ret;
0702 
0703     ret = nvmet_register_transport(&nvme_loop_ops);
0704     if (ret)
0705         return ret;
0706 
0707     ret = nvmf_register_transport(&nvme_loop_transport);
0708     if (ret)
0709         nvmet_unregister_transport(&nvme_loop_ops);
0710 
0711     return ret;
0712 }
0713 
0714 static void __exit nvme_loop_cleanup_module(void)
0715 {
0716     struct nvme_loop_ctrl *ctrl, *next;
0717 
0718     nvmf_unregister_transport(&nvme_loop_transport);
0719     nvmet_unregister_transport(&nvme_loop_ops);
0720 
0721     mutex_lock(&nvme_loop_ctrl_mutex);
0722     list_for_each_entry_safe(ctrl, next, &nvme_loop_ctrl_list, list)
0723         nvme_delete_ctrl(&ctrl->ctrl);
0724     mutex_unlock(&nvme_loop_ctrl_mutex);
0725 
0726     flush_workqueue(nvme_delete_wq);
0727 }
0728 
0729 module_init(nvme_loop_init_module);
0730 module_exit(nvme_loop_cleanup_module);
0731 
0732 MODULE_LICENSE("GPL v2");
0733 MODULE_ALIAS("nvmet-transport-254"); /* 254 == NVMF_TRTYPE_LOOP */
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!