sw/rdmavt/vt.c

0001 // SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause
0002 /*
0003  * Copyright(c) 2016 - 2018 Intel Corporation.
0004  */
0005
0006 #include <linux/module.h>
0007 #include <linux/kernel.h>
0008 #include <linux/dma-mapping.h>
0009 #include "vt.h"
0010 #include "cq.h"
0011 #include "trace.h"
0012
0013 #define RVT_UVERBS_ABI_VERSION 2
0014
0015 MODULE_LICENSE("Dual BSD/GPL");
0016 MODULE_DESCRIPTION("RDMA Verbs Transport Library");
0017
0018 static int rvt_init(void)
0019 {
0020     int ret = rvt_driver_cq_init();
0021
0022     if (ret)
0023         pr_err("Error in driver CQ init.\n");
0024
0025     return ret;
0026 }
0027 module_init(rvt_init);
0028
0029 static void rvt_cleanup(void)
0030 {
0031     rvt_cq_exit();
0032 }
0033 module_exit(rvt_cleanup);
0034
0035 /**
0036  * rvt_alloc_device - allocate rdi
0037  * @size: how big of a structure to allocate
0038  * @nports: number of ports to allocate array slots for
0039  *
0040  * Use IB core device alloc to allocate space for the rdi which is assumed to be
0041  * inside of the ib_device. Any extra space that drivers require should be
0042  * included in size.
0043  *
0044  * We also allocate a port array based on the number of ports.
0045  *
0046  * Return: pointer to allocated rdi
0047  */
0048 struct rvt_dev_info *rvt_alloc_device(size_t size, int nports)
0049 {
0050     struct rvt_dev_info *rdi;
0051
0052     rdi = container_of(_ib_alloc_device(size), struct rvt_dev_info, ibdev);
0053     if (!rdi)
0054         return rdi;
0055
0056     rdi->ports = kcalloc(nports, sizeof(*rdi->ports), GFP_KERNEL);
0057     if (!rdi->ports)
0058         ib_dealloc_device(&rdi->ibdev);
0059
0060     return rdi;
0061 }
0062 EXPORT_SYMBOL(rvt_alloc_device);
0063
0064 /**
0065  * rvt_dealloc_device - deallocate rdi
0066  * @rdi: structure to free
0067  *
0068  * Free a structure allocated with rvt_alloc_device()
0069  */
0070 void rvt_dealloc_device(struct rvt_dev_info *rdi)
0071 {
0072     kfree(rdi->ports);
0073     ib_dealloc_device(&rdi->ibdev);
0074 }
0075 EXPORT_SYMBOL(rvt_dealloc_device);
0076
0077 static int rvt_query_device(struct ib_device *ibdev,
0078                 struct ib_device_attr *props,
0079                 struct ib_udata *uhw)
0080 {
0081     struct rvt_dev_info *rdi = ib_to_rvt(ibdev);
0082
0083     if (uhw->inlen || uhw->outlen)
0084         return -EINVAL;
0085     /*
0086      * Return rvt_dev_info.dparms.props contents
0087      */
0088     *props = rdi->dparms.props;
0089     return 0;
0090 }
0091
0092 static int rvt_get_numa_node(struct ib_device *ibdev)
0093 {
0094     struct rvt_dev_info *rdi = ib_to_rvt(ibdev);
0095
0096     return rdi->dparms.node;
0097 }
0098
0099 static int rvt_modify_device(struct ib_device *device,
0100                  int device_modify_mask,
0101                  struct ib_device_modify *device_modify)
0102 {
0103     /*
0104      * There is currently no need to supply this based on qib and hfi1.
0105      * Future drivers may need to implement this though.
0106      */
0107
0108     return -EOPNOTSUPP;
0109 }
0110
0111 /**
0112  * rvt_query_port - Passes the query port call to the driver
0113  * @ibdev: Verbs IB dev
0114  * @port_num: port number, 1 based from ib core
0115  * @props: structure to hold returned properties
0116  *
0117  * Return: 0 on success
0118  */
0119 static int rvt_query_port(struct ib_device *ibdev, u32 port_num,
0120               struct ib_port_attr *props)
0121 {
0122     struct rvt_dev_info *rdi = ib_to_rvt(ibdev);
0123     struct rvt_ibport *rvp;
0124     u32 port_index = ibport_num_to_idx(ibdev, port_num);
0125
0126     rvp = rdi->ports[port_index];
0127     /* props being zeroed by the caller, avoid zeroing it here */
0128     props->sm_lid = rvp->sm_lid;
0129     props->sm_sl = rvp->sm_sl;
0130     props->port_cap_flags = rvp->port_cap_flags;
0131     props->max_msg_sz = 0x80000000;
0132     props->pkey_tbl_len = rvt_get_npkeys(rdi);
0133     props->bad_pkey_cntr = rvp->pkey_violations;
0134     props->qkey_viol_cntr = rvp->qkey_violations;
0135     props->subnet_timeout = rvp->subnet_timeout;
0136     props->init_type_reply = 0;
0137
0138     /* Populate the remaining ib_port_attr elements */
0139     return rdi->driver_f.query_port_state(rdi, port_num, props);
0140 }
0141
0142 /**
0143  * rvt_modify_port - modify port
0144  * @ibdev: Verbs IB dev
0145  * @port_num: Port number, 1 based from ib core
0146  * @port_modify_mask: How to change the port
0147  * @props: Structure to fill in
0148  *
0149  * Return: 0 on success
0150  */
0151 static int rvt_modify_port(struct ib_device *ibdev, u32 port_num,
0152                int port_modify_mask, struct ib_port_modify *props)
0153 {
0154     struct rvt_dev_info *rdi = ib_to_rvt(ibdev);
0155     struct rvt_ibport *rvp;
0156     int ret = 0;
0157     u32 port_index = ibport_num_to_idx(ibdev, port_num);
0158
0159     rvp = rdi->ports[port_index];
0160     if (port_modify_mask & IB_PORT_OPA_MASK_CHG) {
0161         rvp->port_cap3_flags |= props->set_port_cap_mask;
0162         rvp->port_cap3_flags &= ~props->clr_port_cap_mask;
0163     } else {
0164         rvp->port_cap_flags |= props->set_port_cap_mask;
0165         rvp->port_cap_flags &= ~props->clr_port_cap_mask;
0166     }
0167
0168     if (props->set_port_cap_mask || props->clr_port_cap_mask)
0169         rdi->driver_f.cap_mask_chg(rdi, port_num);
0170     if (port_modify_mask & IB_PORT_SHUTDOWN)
0171         ret = rdi->driver_f.shut_down_port(rdi, port_num);
0172     if (port_modify_mask & IB_PORT_RESET_QKEY_CNTR)
0173         rvp->qkey_violations = 0;
0174
0175     return ret;
0176 }
0177
0178 /**
0179  * rvt_query_pkey - Return a pkey from the table at a given index
0180  * @ibdev: Verbs IB dev
0181  * @port_num: Port number, 1 based from ib core
0182  * @index: Index into pkey table
0183  * @pkey: returned pkey from the port pkey table
0184  *
0185  * Return: 0 on failure pkey otherwise
0186  */
0187 static int rvt_query_pkey(struct ib_device *ibdev, u32 port_num, u16 index,
0188               u16 *pkey)
0189 {
0190     /*
0191      * Driver will be responsible for keeping rvt_dev_info.pkey_table up to
0192      * date. This function will just return that value. There is no need to
0193      * lock, if a stale value is read and sent to the user so be it there is
0194      * no way to protect against that anyway.
0195      */
0196     struct rvt_dev_info *rdi = ib_to_rvt(ibdev);
0197     u32 port_index;
0198
0199     port_index = ibport_num_to_idx(ibdev, port_num);
0200
0201     if (index >= rvt_get_npkeys(rdi))
0202         return -EINVAL;
0203
0204     *pkey = rvt_get_pkey(rdi, port_index, index);
0205     return 0;
0206 }
0207
0208 /**
0209  * rvt_query_gid - Return a gid from the table
0210  * @ibdev: Verbs IB dev
0211  * @port_num: Port number, 1 based from ib core
0212  * @guid_index: Index in table
0213  * @gid: Gid to return
0214  *
0215  * Return: 0 on success
0216  */
0217 static int rvt_query_gid(struct ib_device *ibdev, u32 port_num,
0218              int guid_index, union ib_gid *gid)
0219 {
0220     struct rvt_dev_info *rdi;
0221     struct rvt_ibport *rvp;
0222     u32 port_index;
0223
0224     /*
0225      * Driver is responsible for updating the guid table. Which will be used
0226      * to craft the return value. This will work similar to how query_pkey()
0227      * is being done.
0228      */
0229     port_index = ibport_num_to_idx(ibdev, port_num);
0230
0231     rdi = ib_to_rvt(ibdev);
0232     rvp = rdi->ports[port_index];
0233
0234     gid->global.subnet_prefix = rvp->gid_prefix;
0235
0236     return rdi->driver_f.get_guid_be(rdi, rvp, guid_index,
0237                      &gid->global.interface_id);
0238 }
0239
0240 /**
0241  * rvt_alloc_ucontext - Allocate a user context
0242  * @uctx: Verbs context
0243  * @udata: User data allocated
0244  */
0245 static int rvt_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
0246 {
0247     return 0;
0248 }
0249
0250 /**
0251  * rvt_dealloc_ucontext - Free a user context
0252  * @context: Unused
0253  */
0254 static void rvt_dealloc_ucontext(struct ib_ucontext *context)
0255 {
0256     return;
0257 }
0258
0259 static int rvt_get_port_immutable(struct ib_device *ibdev, u32 port_num,
0260                   struct ib_port_immutable *immutable)
0261 {
0262     struct rvt_dev_info *rdi = ib_to_rvt(ibdev);
0263     struct ib_port_attr attr;
0264     int err;
0265
0266     immutable->core_cap_flags = rdi->dparms.core_cap_flags;
0267
0268     err = ib_query_port(ibdev, port_num, &attr);
0269     if (err)
0270         return err;
0271
0272     immutable->pkey_tbl_len = attr.pkey_tbl_len;
0273     immutable->gid_tbl_len = attr.gid_tbl_len;
0274     immutable->max_mad_size = rdi->dparms.max_mad_size;
0275
0276     return 0;
0277 }
0278
0279 enum {
0280     MISC,
0281     QUERY_DEVICE,
0282     MODIFY_DEVICE,
0283     QUERY_PORT,
0284     MODIFY_PORT,
0285     QUERY_PKEY,
0286     QUERY_GID,
0287     ALLOC_UCONTEXT,
0288     DEALLOC_UCONTEXT,
0289     GET_PORT_IMMUTABLE,
0290     CREATE_QP,
0291     MODIFY_QP,
0292     DESTROY_QP,
0293     QUERY_QP,
0294     POST_SEND,
0295     POST_RECV,
0296     POST_SRQ_RECV,
0297     CREATE_AH,
0298     DESTROY_AH,
0299     MODIFY_AH,
0300     QUERY_AH,
0301     CREATE_SRQ,
0302     MODIFY_SRQ,
0303     DESTROY_SRQ,
0304     QUERY_SRQ,
0305     ATTACH_MCAST,
0306     DETACH_MCAST,
0307     GET_DMA_MR,
0308     REG_USER_MR,
0309     DEREG_MR,
0310     ALLOC_MR,
0311     MAP_MR_SG,
0312     ALLOC_FMR,
0313     MAP_PHYS_FMR,
0314     UNMAP_FMR,
0315     DEALLOC_FMR,
0316     MMAP,
0317     CREATE_CQ,
0318     DESTROY_CQ,
0319     POLL_CQ,
0320     REQ_NOTFIY_CQ,
0321     RESIZE_CQ,
0322     ALLOC_PD,
0323     DEALLOC_PD,
0324     _VERB_IDX_MAX /* Must always be last! */
0325 };
0326
0327 static const struct ib_device_ops rvt_dev_ops = {
0328     .uverbs_abi_ver = RVT_UVERBS_ABI_VERSION,
0329
0330     .alloc_mr = rvt_alloc_mr,
0331     .alloc_pd = rvt_alloc_pd,
0332     .alloc_ucontext = rvt_alloc_ucontext,
0333     .attach_mcast = rvt_attach_mcast,
0334     .create_ah = rvt_create_ah,
0335     .create_cq = rvt_create_cq,
0336     .create_qp = rvt_create_qp,
0337     .create_srq = rvt_create_srq,
0338     .create_user_ah = rvt_create_ah,
0339     .dealloc_pd = rvt_dealloc_pd,
0340     .dealloc_ucontext = rvt_dealloc_ucontext,
0341     .dereg_mr = rvt_dereg_mr,
0342     .destroy_ah = rvt_destroy_ah,
0343     .destroy_cq = rvt_destroy_cq,
0344     .destroy_qp = rvt_destroy_qp,
0345     .destroy_srq = rvt_destroy_srq,
0346     .detach_mcast = rvt_detach_mcast,
0347     .get_dma_mr = rvt_get_dma_mr,
0348     .get_numa_node = rvt_get_numa_node,
0349     .get_port_immutable = rvt_get_port_immutable,
0350     .map_mr_sg = rvt_map_mr_sg,
0351     .mmap = rvt_mmap,
0352     .modify_ah = rvt_modify_ah,
0353     .modify_device = rvt_modify_device,
0354     .modify_port = rvt_modify_port,
0355     .modify_qp = rvt_modify_qp,
0356     .modify_srq = rvt_modify_srq,
0357     .poll_cq = rvt_poll_cq,
0358     .post_recv = rvt_post_recv,
0359     .post_send = rvt_post_send,
0360     .post_srq_recv = rvt_post_srq_recv,
0361     .query_ah = rvt_query_ah,
0362     .query_device = rvt_query_device,
0363     .query_gid = rvt_query_gid,
0364     .query_pkey = rvt_query_pkey,
0365     .query_port = rvt_query_port,
0366     .query_qp = rvt_query_qp,
0367     .query_srq = rvt_query_srq,
0368     .reg_user_mr = rvt_reg_user_mr,
0369     .req_notify_cq = rvt_req_notify_cq,
0370     .resize_cq = rvt_resize_cq,
0371
0372     INIT_RDMA_OBJ_SIZE(ib_ah, rvt_ah, ibah),
0373     INIT_RDMA_OBJ_SIZE(ib_cq, rvt_cq, ibcq),
0374     INIT_RDMA_OBJ_SIZE(ib_pd, rvt_pd, ibpd),
0375     INIT_RDMA_OBJ_SIZE(ib_qp, rvt_qp, ibqp),
0376     INIT_RDMA_OBJ_SIZE(ib_srq, rvt_srq, ibsrq),
0377     INIT_RDMA_OBJ_SIZE(ib_ucontext, rvt_ucontext, ibucontext),
0378 };
0379
0380 static noinline int check_support(struct rvt_dev_info *rdi, int verb)
0381 {
0382     switch (verb) {
0383     case MISC:
0384         /*
0385          * These functions are not part of verbs specifically but are
0386          * required for rdmavt to function.
0387          */
0388         if ((!rdi->ibdev.ops.port_groups) ||
0389             (!rdi->driver_f.get_pci_dev))
0390             return -EINVAL;
0391         break;
0392
0393     case MODIFY_DEVICE:
0394         /*
0395          * rdmavt does not support modify device currently drivers must
0396          * provide.
0397          */
0398         if (!rdi->ibdev.ops.modify_device)
0399             return -EOPNOTSUPP;
0400         break;
0401
0402     case QUERY_PORT:
0403         if (!rdi->ibdev.ops.query_port)
0404             if (!rdi->driver_f.query_port_state)
0405                 return -EINVAL;
0406         break;
0407
0408     case MODIFY_PORT:
0409         if (!rdi->ibdev.ops.modify_port)
0410             if (!rdi->driver_f.cap_mask_chg ||
0411                 !rdi->driver_f.shut_down_port)
0412                 return -EINVAL;
0413         break;
0414
0415     case QUERY_GID:
0416         if (!rdi->ibdev.ops.query_gid)
0417             if (!rdi->driver_f.get_guid_be)
0418                 return -EINVAL;
0419         break;
0420
0421     case CREATE_QP:
0422         if (!rdi->ibdev.ops.create_qp)
0423             if (!rdi->driver_f.qp_priv_alloc ||
0424                 !rdi->driver_f.qp_priv_free ||
0425                 !rdi->driver_f.notify_qp_reset ||
0426                 !rdi->driver_f.flush_qp_waiters ||
0427                 !rdi->driver_f.stop_send_queue ||
0428                 !rdi->driver_f.quiesce_qp)
0429                 return -EINVAL;
0430         break;
0431
0432     case MODIFY_QP:
0433         if (!rdi->ibdev.ops.modify_qp)
0434             if (!rdi->driver_f.notify_qp_reset ||
0435                 !rdi->driver_f.schedule_send ||
0436                 !rdi->driver_f.get_pmtu_from_attr ||
0437                 !rdi->driver_f.flush_qp_waiters ||
0438                 !rdi->driver_f.stop_send_queue ||
0439                 !rdi->driver_f.quiesce_qp ||
0440                 !rdi->driver_f.notify_error_qp ||
0441                 !rdi->driver_f.mtu_from_qp ||
0442                 !rdi->driver_f.mtu_to_path_mtu)
0443                 return -EINVAL;
0444         break;
0445
0446     case DESTROY_QP:
0447         if (!rdi->ibdev.ops.destroy_qp)
0448             if (!rdi->driver_f.qp_priv_free ||
0449                 !rdi->driver_f.notify_qp_reset ||
0450                 !rdi->driver_f.flush_qp_waiters ||
0451                 !rdi->driver_f.stop_send_queue ||
0452                 !rdi->driver_f.quiesce_qp)
0453                 return -EINVAL;
0454         break;
0455
0456     case POST_SEND:
0457         if (!rdi->ibdev.ops.post_send)
0458             if (!rdi->driver_f.schedule_send ||
0459                 !rdi->driver_f.do_send ||
0460                 !rdi->post_parms)
0461                 return -EINVAL;
0462         break;
0463
0464     }
0465
0466     return 0;
0467 }
0468
0469 /**
0470  * rvt_register_device - register a driver
0471  * @rdi: main dev structure for all of rdmavt operations
0472  *
0473  * It is up to drivers to allocate the rdi and fill in the appropriate
0474  * information.
0475  *
0476  * Return: 0 on success otherwise an errno.
0477  */
0478 int rvt_register_device(struct rvt_dev_info *rdi)
0479 {
0480     int ret = 0, i;
0481
0482     if (!rdi)
0483         return -EINVAL;
0484
0485     /*
0486      * Check to ensure drivers have setup the required helpers for the verbs
0487      * they want rdmavt to handle
0488      */
0489     for (i = 0; i < _VERB_IDX_MAX; i++)
0490         if (check_support(rdi, i)) {
0491             pr_err("Driver support req not met at %d\n", i);
0492             return -EINVAL;
0493         }
0494
0495     ib_set_device_ops(&rdi->ibdev, &rvt_dev_ops);
0496
0497     /* Once we get past here we can use rvt_pr macros and tracepoints */
0498     trace_rvt_dbg(rdi, "Driver attempting registration");
0499     rvt_mmap_init(rdi);
0500
0501     /* Queue Pairs */
0502     ret = rvt_driver_qp_init(rdi);
0503     if (ret) {
0504         pr_err("Error in driver QP init.\n");
0505         return -EINVAL;
0506     }
0507
0508     /* Address Handle */
0509     spin_lock_init(&rdi->n_ahs_lock);
0510     rdi->n_ahs_allocated = 0;
0511
0512     /* Shared Receive Queue */
0513     rvt_driver_srq_init(rdi);
0514
0515     /* Multicast */
0516     rvt_driver_mcast_init(rdi);
0517
0518     /* Mem Region */
0519     ret = rvt_driver_mr_init(rdi);
0520     if (ret) {
0521         pr_err("Error in driver MR init.\n");
0522         goto bail_no_mr;
0523     }
0524
0525     /* Memory Working Set Size */
0526     ret = rvt_wss_init(rdi);
0527     if (ret) {
0528         rvt_pr_err(rdi, "Error in WSS init.\n");
0529         goto bail_mr;
0530     }
0531
0532     /* Completion queues */
0533     spin_lock_init(&rdi->n_cqs_lock);
0534
0535     /* Protection Domain */
0536     spin_lock_init(&rdi->n_pds_lock);
0537     rdi->n_pds_allocated = 0;
0538
0539     /*
0540      * There are some things which could be set by underlying drivers but
0541      * really should be up to rdmavt to set. For instance drivers can't know
0542      * exactly which functions rdmavt supports, nor do they know the ABI
0543      * version, so we do all of this sort of stuff here.
0544      */
0545     rdi->ibdev.uverbs_cmd_mask |=
0546         (1ull << IB_USER_VERBS_CMD_POLL_CQ)             |
0547         (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ)       |
0548         (1ull << IB_USER_VERBS_CMD_POST_SEND)           |
0549         (1ull << IB_USER_VERBS_CMD_POST_RECV)           |
0550         (1ull << IB_USER_VERBS_CMD_POST_SRQ_RECV);
0551     rdi->ibdev.node_type = RDMA_NODE_IB_CA;
0552     if (!rdi->ibdev.num_comp_vectors)
0553         rdi->ibdev.num_comp_vectors = 1;
0554
0555     /* We are now good to announce we exist */
0556     ret = ib_register_device(&rdi->ibdev, dev_name(&rdi->ibdev.dev), NULL);
0557     if (ret) {
0558         rvt_pr_err(rdi, "Failed to register driver with ib core.\n");
0559         goto bail_wss;
0560     }
0561
0562     rvt_create_mad_agents(rdi);
0563
0564     rvt_pr_info(rdi, "Registration with rdmavt done.\n");
0565     return ret;
0566
0567 bail_wss:
0568     rvt_wss_exit(rdi);
0569 bail_mr:
0570     rvt_mr_exit(rdi);
0571
0572 bail_no_mr:
0573     rvt_qp_exit(rdi);
0574
0575     return ret;
0576 }
0577 EXPORT_SYMBOL(rvt_register_device);
0578
0579 /**
0580  * rvt_unregister_device - remove a driver
0581  * @rdi: rvt dev struct
0582  */
0583 void rvt_unregister_device(struct rvt_dev_info *rdi)
0584 {
0585     trace_rvt_dbg(rdi, "Driver is unregistering.");
0586     if (!rdi)
0587         return;
0588
0589     rvt_free_mad_agents(rdi);
0590
0591     ib_unregister_device(&rdi->ibdev);
0592     rvt_wss_exit(rdi);
0593     rvt_mr_exit(rdi);
0594     rvt_qp_exit(rdi);
0595 }
0596 EXPORT_SYMBOL(rvt_unregister_device);
0597
0598 /**
0599  * rvt_init_port - init internal data for driver port
0600  * @rdi: rvt_dev_info struct
0601  * @port: rvt port
0602  * @port_index: 0 based index of ports, different from IB core port num
0603  * @pkey_table: pkey_table for @port
0604  *
0605  * Keep track of a list of ports. No need to have a detach port.
0606  * They persist until the driver goes away.
0607  *
0608  * Return: always 0
0609  */
0610 int rvt_init_port(struct rvt_dev_info *rdi, struct rvt_ibport *port,
0611           int port_index, u16 *pkey_table)
0612 {
0613
0614     rdi->ports[port_index] = port;
0615     rdi->ports[port_index]->pkey_table = pkey_table;
0616
0617     return 0;
0618 }
0619 EXPORT_SYMBOL(rvt_init_port);