mlx5/core/wq.c

0001 /*
0002  * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
0003  *
0004  * This software is available to you under a choice of one of two
0005  * licenses.  You may choose to be licensed under the terms of the GNU
0006  * General Public License (GPL) Version 2, available from the file
0007  * COPYING in the main directory of this source tree, or the
0008  * OpenIB.org BSD license below:
0009  *
0010  *     Redistribution and use in source and binary forms, with or
0011  *     without modification, are permitted provided that the following
0012  *     conditions are met:
0013  *
0014  *      - Redistributions of source code must retain the above
0015  *        copyright notice, this list of conditions and the following
0016  *        disclaimer.
0017  *
0018  *      - Redistributions in binary form must reproduce the above
0019  *        copyright notice, this list of conditions and the following
0020  *        disclaimer in the documentation and/or other materials
0021  *        provided with the distribution.
0022  *
0023  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
0024  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
0025  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
0026  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
0027  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
0028  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
0029  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
0030  * SOFTWARE.
0031  */
0032
0033 #include <linux/mlx5/driver.h>
0034 #include "wq.h"
0035 #include "mlx5_core.h"
0036
0037 int mlx5_wq_cyc_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param,
0038                void *wqc, struct mlx5_wq_cyc *wq,
0039                struct mlx5_wq_ctrl *wq_ctrl)
0040 {
0041     u8 log_wq_stride = MLX5_GET(wq, wqc, log_wq_stride);
0042     u8 log_wq_sz     = MLX5_GET(wq, wqc, log_wq_sz);
0043     struct mlx5_frag_buf_ctrl *fbc = &wq->fbc;
0044     int err;
0045
0046     err = mlx5_db_alloc_node(mdev, &wq_ctrl->db, param->db_numa_node);
0047     if (err) {
0048         mlx5_core_warn(mdev, "mlx5_db_alloc_node() failed, %d\n", err);
0049         return err;
0050     }
0051
0052     wq->db  = wq_ctrl->db.db;
0053
0054     err = mlx5_frag_buf_alloc_node(mdev, wq_get_byte_sz(log_wq_sz, log_wq_stride),
0055                        &wq_ctrl->buf, param->buf_numa_node);
0056     if (err) {
0057         mlx5_core_warn(mdev, "mlx5_frag_buf_alloc_node() failed, %d\n", err);
0058         goto err_db_free;
0059     }
0060
0061     mlx5_init_fbc(wq_ctrl->buf.frags, log_wq_stride, log_wq_sz, fbc);
0062     wq->sz = mlx5_wq_cyc_get_size(wq);
0063
0064     wq_ctrl->mdev = mdev;
0065
0066     return 0;
0067
0068 err_db_free:
0069     mlx5_db_free(mdev, &wq_ctrl->db);
0070
0071     return err;
0072 }
0073
0074 void mlx5_wq_cyc_wqe_dump(struct mlx5_wq_cyc *wq, u16 ix, u8 nstrides)
0075 {
0076     size_t len;
0077     void *wqe;
0078
0079     if (!net_ratelimit())
0080         return;
0081
0082     nstrides = max_t(u8, nstrides, 1);
0083
0084     len = nstrides << wq->fbc.log_stride;
0085     wqe = mlx5_wq_cyc_get_wqe(wq, ix);
0086
0087     pr_info("WQE DUMP: WQ size %d WQ cur size %d, WQE index 0x%x, len: %zu\n",
0088         mlx5_wq_cyc_get_size(wq), wq->cur_sz, ix, len);
0089     print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET, 16, 1, wqe, len, false);
0090 }
0091
0092 void mlx5_wq_cyc_reset(struct mlx5_wq_cyc *wq)
0093 {
0094     wq->wqe_ctr = 0;
0095     wq->cur_sz = 0;
0096     mlx5_wq_cyc_update_db_record(wq);
0097 }
0098
0099 int mlx5_wq_qp_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param,
0100               void *qpc, struct mlx5_wq_qp *wq,
0101               struct mlx5_wq_ctrl *wq_ctrl)
0102 {
0103     u8 log_rq_stride = MLX5_GET(qpc, qpc, log_rq_stride) + 4;
0104     u8 log_rq_sz     = MLX5_GET(qpc, qpc, log_rq_size);
0105     u8 log_sq_stride = ilog2(MLX5_SEND_WQE_BB);
0106     u8 log_sq_sz     = MLX5_GET(qpc, qpc, log_sq_size);
0107
0108     u32 rq_byte_size;
0109     int err;
0110
0111
0112
0113     err = mlx5_db_alloc_node(mdev, &wq_ctrl->db, param->db_numa_node);
0114     if (err) {
0115         mlx5_core_warn(mdev, "mlx5_db_alloc_node() failed, %d\n", err);
0116         return err;
0117     }
0118
0119     err = mlx5_frag_buf_alloc_node(mdev,
0120                        wq_get_byte_sz(log_rq_sz, log_rq_stride) +
0121                        wq_get_byte_sz(log_sq_sz, log_sq_stride),
0122                        &wq_ctrl->buf, param->buf_numa_node);
0123     if (err) {
0124         mlx5_core_warn(mdev, "mlx5_frag_buf_alloc_node() failed, %d\n", err);
0125         goto err_db_free;
0126     }
0127
0128     mlx5_init_fbc(wq_ctrl->buf.frags, log_rq_stride, log_rq_sz, &wq->rq.fbc);
0129
0130     rq_byte_size = wq_get_byte_sz(log_rq_sz, log_rq_stride);
0131
0132     if (rq_byte_size < PAGE_SIZE) {
0133         /* SQ starts within the same page of the RQ */
0134         u16 sq_strides_offset = rq_byte_size / MLX5_SEND_WQE_BB;
0135
0136         mlx5_init_fbc_offset(wq_ctrl->buf.frags,
0137                      log_sq_stride, log_sq_sz, sq_strides_offset,
0138                      &wq->sq.fbc);
0139     } else {
0140         u16 rq_npages = rq_byte_size >> PAGE_SHIFT;
0141
0142         mlx5_init_fbc(wq_ctrl->buf.frags + rq_npages,
0143                   log_sq_stride, log_sq_sz, &wq->sq.fbc);
0144     }
0145
0146     wq->rq.db  = &wq_ctrl->db.db[MLX5_RCV_DBR];
0147     wq->sq.db  = &wq_ctrl->db.db[MLX5_SND_DBR];
0148
0149     wq_ctrl->mdev = mdev;
0150
0151     return 0;
0152
0153 err_db_free:
0154     mlx5_db_free(mdev, &wq_ctrl->db);
0155
0156     return err;
0157 }
0158
0159 int mlx5_cqwq_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param,
0160              void *cqc, struct mlx5_cqwq *wq,
0161              struct mlx5_wq_ctrl *wq_ctrl)
0162 {
0163     /* CQE_STRIDE_128 and CQE_STRIDE_128_PAD both mean 128B stride */
0164     u8 log_wq_stride = MLX5_GET(cqc, cqc, cqe_sz) == CQE_STRIDE_64 ? 6 : 7;
0165     u8 log_wq_sz     = MLX5_GET(cqc, cqc, log_cq_size);
0166     int err;
0167
0168     err = mlx5_db_alloc_node(mdev, &wq_ctrl->db, param->db_numa_node);
0169     if (err) {
0170         mlx5_core_warn(mdev, "mlx5_db_alloc_node() failed, %d\n", err);
0171         return err;
0172     }
0173
0174     wq->db  = wq_ctrl->db.db;
0175
0176     err = mlx5_frag_buf_alloc_node(mdev, wq_get_byte_sz(log_wq_sz, log_wq_stride),
0177                        &wq_ctrl->buf,
0178                        param->buf_numa_node);
0179     if (err) {
0180         mlx5_core_warn(mdev, "mlx5_frag_buf_alloc_node() failed, %d\n",
0181                    err);
0182         goto err_db_free;
0183     }
0184
0185     mlx5_init_fbc(wq_ctrl->buf.frags, log_wq_stride, log_wq_sz, &wq->fbc);
0186
0187     wq_ctrl->mdev = mdev;
0188
0189     return 0;
0190
0191 err_db_free:
0192     mlx5_db_free(mdev, &wq_ctrl->db);
0193
0194     return err;
0195 }
0196
0197 static void mlx5_wq_ll_init_list(struct mlx5_wq_ll *wq)
0198 {
0199     struct mlx5_wqe_srq_next_seg *next_seg;
0200     int i;
0201
0202     for (i = 0; i < wq->fbc.sz_m1; i++) {
0203         next_seg = mlx5_wq_ll_get_wqe(wq, i);
0204         next_seg->next_wqe_index = cpu_to_be16(i + 1);
0205     }
0206     next_seg = mlx5_wq_ll_get_wqe(wq, i);
0207     wq->tail_next = &next_seg->next_wqe_index;
0208 }
0209
0210 int mlx5_wq_ll_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param,
0211               void *wqc, struct mlx5_wq_ll *wq,
0212               struct mlx5_wq_ctrl *wq_ctrl)
0213 {
0214     u8 log_wq_stride = MLX5_GET(wq, wqc, log_wq_stride);
0215     u8 log_wq_sz     = MLX5_GET(wq, wqc, log_wq_sz);
0216     struct mlx5_frag_buf_ctrl *fbc = &wq->fbc;
0217     int err;
0218
0219     err = mlx5_db_alloc_node(mdev, &wq_ctrl->db, param->db_numa_node);
0220     if (err) {
0221         mlx5_core_warn(mdev, "mlx5_db_alloc_node() failed, %d\n", err);
0222         return err;
0223     }
0224
0225     wq->db  = wq_ctrl->db.db;
0226
0227     err = mlx5_frag_buf_alloc_node(mdev, wq_get_byte_sz(log_wq_sz, log_wq_stride),
0228                        &wq_ctrl->buf, param->buf_numa_node);
0229     if (err) {
0230         mlx5_core_warn(mdev, "mlx5_frag_buf_alloc_node() failed, %d\n", err);
0231         goto err_db_free;
0232     }
0233
0234     mlx5_init_fbc(wq_ctrl->buf.frags, log_wq_stride, log_wq_sz, fbc);
0235
0236     mlx5_wq_ll_init_list(wq);
0237     wq_ctrl->mdev = mdev;
0238
0239     return 0;
0240
0241 err_db_free:
0242     mlx5_db_free(mdev, &wq_ctrl->db);
0243
0244     return err;
0245 }
0246
0247 void mlx5_wq_ll_reset(struct mlx5_wq_ll *wq)
0248 {
0249     wq->head = 0;
0250     wq->wqe_ctr = 0;
0251     wq->cur_sz = 0;
0252     mlx5_wq_ll_init_list(wq);
0253     mlx5_wq_ll_update_db_record(wq);
0254 }
0255
0256 void mlx5_wq_destroy(struct mlx5_wq_ctrl *wq_ctrl)
0257 {
0258     mlx5_frag_buf_free(wq_ctrl->mdev, &wq_ctrl->buf);
0259     mlx5_db_free(wq_ctrl->mdev, &wq_ctrl->db);
0260 }
0261