hw/mlx5/mem.c

0001 /*
0002  * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
0003  *
0004  * This software is available to you under a choice of one of two
0005  * licenses.  You may choose to be licensed under the terms of the GNU
0006  * General Public License (GPL) Version 2, available from the file
0007  * COPYING in the main directory of this source tree, or the
0008  * OpenIB.org BSD license below:
0009  *
0010  *     Redistribution and use in source and binary forms, with or
0011  *     without modification, are permitted provided that the following
0012  *     conditions are met:
0013  *
0014  *      - Redistributions of source code must retain the above
0015  *        copyright notice, this list of conditions and the following
0016  *        disclaimer.
0017  *
0018  *      - Redistributions in binary form must reproduce the above
0019  *        copyright notice, this list of conditions and the following
0020  *        disclaimer in the documentation and/or other materials
0021  *        provided with the distribution.
0022  *
0023  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
0024  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
0025  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
0026  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
0027  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
0028  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
0029  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
0030  * SOFTWARE.
0031  */
0032
0033 #include <rdma/ib_umem.h>
0034 #include <rdma/ib_umem_odp.h>
0035 #include "mlx5_ib.h"
0036 #include <linux/jiffies.h>
0037
0038 /*
0039  * Fill in a physical address list. ib_umem_num_dma_blocks() entries will be
0040  * filled in the pas array.
0041  */
0042 void mlx5_ib_populate_pas(struct ib_umem *umem, size_t page_size, __be64 *pas,
0043               u64 access_flags)
0044 {
0045     struct ib_block_iter biter;
0046
0047     rdma_umem_for_each_dma_block (umem, &biter, page_size) {
0048         *pas = cpu_to_be64(rdma_block_iter_dma_address(&biter) |
0049                    access_flags);
0050         pas++;
0051     }
0052 }
0053
0054 /*
0055  * Compute the page shift and page_offset for mailboxes that use a quantized
0056  * page_offset. The granulatity of the page offset scales according to page
0057  * size.
0058  */
0059 unsigned long __mlx5_umem_find_best_quantized_pgoff(
0060     struct ib_umem *umem, unsigned long pgsz_bitmap,
0061     unsigned int page_offset_bits, u64 pgoff_bitmask, unsigned int scale,
0062     unsigned int *page_offset_quantized)
0063 {
0064     const u64 page_offset_mask = (1UL << page_offset_bits) - 1;
0065     unsigned long page_size;
0066     u64 page_offset;
0067
0068     page_size = ib_umem_find_best_pgoff(umem, pgsz_bitmap, pgoff_bitmask);
0069     if (!page_size)
0070         return 0;
0071
0072     /*
0073      * page size is the largest possible page size.
0074      *
0075      * Reduce the page_size, and thus the page_offset and quanta, until the
0076      * page_offset fits into the mailbox field. Once page_size < scale this
0077      * loop is guaranteed to terminate.
0078      */
0079     page_offset = ib_umem_dma_offset(umem, page_size);
0080     while (page_offset & ~(u64)(page_offset_mask * (page_size / scale))) {
0081         page_size /= 2;
0082         page_offset = ib_umem_dma_offset(umem, page_size);
0083     }
0084
0085     /*
0086      * The address is not aligned, or otherwise cannot be represented by the
0087      * page_offset.
0088      */
0089     if (!(pgsz_bitmap & page_size))
0090         return 0;
0091
0092     *page_offset_quantized =
0093         (unsigned long)page_offset / (page_size / scale);
0094     if (WARN_ON(*page_offset_quantized > page_offset_mask))
0095         return 0;
0096     return page_size;
0097 }
0098
0099 #define WR_ID_BF 0xBF
0100 #define WR_ID_END 0xBAD
0101 #define TEST_WC_NUM_WQES 255
0102 #define TEST_WC_POLLING_MAX_TIME_JIFFIES msecs_to_jiffies(100)
0103 static int post_send_nop(struct mlx5_ib_dev *dev, struct ib_qp *ibqp, u64 wr_id,
0104              bool signaled)
0105 {
0106     struct mlx5_ib_qp *qp = to_mqp(ibqp);
0107     struct mlx5_wqe_ctrl_seg *ctrl;
0108     struct mlx5_bf *bf = &qp->bf;
0109     __be32 mmio_wqe[16] = {};
0110     unsigned long flags;
0111     unsigned int idx;
0112     int i;
0113
0114     if (unlikely(dev->mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR))
0115         return -EIO;
0116
0117     spin_lock_irqsave(&qp->sq.lock, flags);
0118
0119     idx = qp->sq.cur_post & (qp->sq.wqe_cnt - 1);
0120     ctrl = mlx5_frag_buf_get_wqe(&qp->sq.fbc, idx);
0121
0122     memset(ctrl, 0, sizeof(struct mlx5_wqe_ctrl_seg));
0123     ctrl->fm_ce_se = signaled ? MLX5_WQE_CTRL_CQ_UPDATE : 0;
0124     ctrl->opmod_idx_opcode =
0125         cpu_to_be32(((u32)(qp->sq.cur_post) << 8) | MLX5_OPCODE_NOP);
0126     ctrl->qpn_ds = cpu_to_be32((sizeof(struct mlx5_wqe_ctrl_seg) / 16) |
0127                    (qp->trans_qp.base.mqp.qpn << 8));
0128
0129     qp->sq.wrid[idx] = wr_id;
0130     qp->sq.w_list[idx].opcode = MLX5_OPCODE_NOP;
0131     qp->sq.wqe_head[idx] = qp->sq.head + 1;
0132     qp->sq.cur_post += DIV_ROUND_UP(sizeof(struct mlx5_wqe_ctrl_seg),
0133                     MLX5_SEND_WQE_BB);
0134     qp->sq.w_list[idx].next = qp->sq.cur_post;
0135     qp->sq.head++;
0136
0137     memcpy(mmio_wqe, ctrl, sizeof(*ctrl));
0138     ((struct mlx5_wqe_ctrl_seg *)&mmio_wqe)->fm_ce_se |=
0139         MLX5_WQE_CTRL_CQ_UPDATE;
0140
0141     /* Make sure that descriptors are written before
0142      * updating doorbell record and ringing the doorbell
0143      */
0144     wmb();
0145
0146     qp->db.db[MLX5_SND_DBR] = cpu_to_be32(qp->sq.cur_post);
0147
0148     /* Make sure doorbell record is visible to the HCA before
0149      * we hit doorbell
0150      */
0151     wmb();
0152     for (i = 0; i < 8; i++)
0153         mlx5_write64(&mmio_wqe[i * 2],
0154                  bf->bfreg->map + bf->offset + i * 8);
0155
0156     bf->offset ^= bf->buf_size;
0157
0158     spin_unlock_irqrestore(&qp->sq.lock, flags);
0159
0160     return 0;
0161 }
0162
0163 static int test_wc_poll_cq_result(struct mlx5_ib_dev *dev, struct ib_cq *cq)
0164 {
0165     int ret;
0166     struct ib_wc wc = {};
0167     unsigned long end = jiffies + TEST_WC_POLLING_MAX_TIME_JIFFIES;
0168
0169     do {
0170         ret = ib_poll_cq(cq, 1, &wc);
0171         if (ret < 0 || wc.status)
0172             return ret < 0 ? ret : -EINVAL;
0173         if (ret)
0174             break;
0175     } while (!time_after(jiffies, end));
0176
0177     if (!ret)
0178         return -ETIMEDOUT;
0179
0180     if (wc.wr_id != WR_ID_BF)
0181         ret = 0;
0182
0183     return ret;
0184 }
0185
0186 static int test_wc_do_send(struct mlx5_ib_dev *dev, struct ib_qp *qp)
0187 {
0188     int err, i;
0189
0190     for (i = 0; i < TEST_WC_NUM_WQES; i++) {
0191         err = post_send_nop(dev, qp, WR_ID_BF, false);
0192         if (err)
0193             return err;
0194     }
0195
0196     return post_send_nop(dev, qp, WR_ID_END, true);
0197 }
0198
0199 int mlx5_ib_test_wc(struct mlx5_ib_dev *dev)
0200 {
0201     struct ib_cq_init_attr cq_attr = { .cqe = TEST_WC_NUM_WQES + 1 };
0202     int port_type_cap = MLX5_CAP_GEN(dev->mdev, port_type);
0203     struct ib_qp_init_attr qp_init_attr = {
0204         .cap = { .max_send_wr = TEST_WC_NUM_WQES },
0205         .qp_type = IB_QPT_UD,
0206         .sq_sig_type = IB_SIGNAL_REQ_WR,
0207         .create_flags = MLX5_IB_QP_CREATE_WC_TEST,
0208     };
0209     struct ib_qp_attr qp_attr = { .port_num = 1 };
0210     struct ib_device *ibdev = &dev->ib_dev;
0211     struct ib_qp *qp;
0212     struct ib_cq *cq;
0213     struct ib_pd *pd;
0214     int ret;
0215
0216     if (!MLX5_CAP_GEN(dev->mdev, bf))
0217         return 0;
0218
0219     if (!dev->mdev->roce.roce_en &&
0220         port_type_cap == MLX5_CAP_PORT_TYPE_ETH) {
0221         if (mlx5_core_is_pf(dev->mdev))
0222             dev->wc_support = arch_can_pci_mmap_wc();
0223         return 0;
0224     }
0225
0226     ret = mlx5_alloc_bfreg(dev->mdev, &dev->wc_bfreg, true, false);
0227     if (ret)
0228         goto print_err;
0229
0230     if (!dev->wc_bfreg.wc)
0231         goto out1;
0232
0233     pd = ib_alloc_pd(ibdev, 0);
0234     if (IS_ERR(pd)) {
0235         ret = PTR_ERR(pd);
0236         goto out1;
0237     }
0238
0239     cq = ib_create_cq(ibdev, NULL, NULL, NULL, &cq_attr);
0240     if (IS_ERR(cq)) {
0241         ret = PTR_ERR(cq);
0242         goto out2;
0243     }
0244
0245     qp_init_attr.recv_cq = cq;
0246     qp_init_attr.send_cq = cq;
0247     qp = ib_create_qp(pd, &qp_init_attr);
0248     if (IS_ERR(qp)) {
0249         ret = PTR_ERR(qp);
0250         goto out3;
0251     }
0252
0253     qp_attr.qp_state = IB_QPS_INIT;
0254     ret = ib_modify_qp(qp, &qp_attr,
0255                IB_QP_STATE | IB_QP_PORT | IB_QP_PKEY_INDEX |
0256                    IB_QP_QKEY);
0257     if (ret)
0258         goto out4;
0259
0260     qp_attr.qp_state = IB_QPS_RTR;
0261     ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE);
0262     if (ret)
0263         goto out4;
0264
0265     qp_attr.qp_state = IB_QPS_RTS;
0266     ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE | IB_QP_SQ_PSN);
0267     if (ret)
0268         goto out4;
0269
0270     ret = test_wc_do_send(dev, qp);
0271     if (ret < 0)
0272         goto out4;
0273
0274     ret = test_wc_poll_cq_result(dev, cq);
0275     if (ret > 0) {
0276         dev->wc_support = true;
0277         ret = 0;
0278     }
0279
0280 out4:
0281     ib_destroy_qp(qp);
0282 out3:
0283     ib_destroy_cq(cq);
0284 out2:
0285     ib_dealloc_pd(pd);
0286 out1:
0287     mlx5_free_bfreg(dev->mdev, &dev->wc_bfreg);
0288 print_err:
0289     if (ret)
0290         mlx5_ib_err(
0291             dev,
0292             "Error %d while trying to test write-combining support\n",
0293             ret);
0294     return ret;
0295 }