Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
0002 /*
0003  * Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved.
0004  */
0005 
0006 #include "mlx5_ib.h"
0007 #include <linux/mlx5/eswitch.h>
0008 #include "counters.h"
0009 #include "ib_rep.h"
0010 #include "qp.h"
0011 
0012 struct mlx5_ib_counter {
0013     const char *name;
0014     size_t offset;
0015     u32 type;
0016 };
0017 
0018 #define INIT_Q_COUNTER(_name)       \
0019     { .name = #_name, .offset = MLX5_BYTE_OFF(query_q_counter_out, _name)}
0020 
0021 static const struct mlx5_ib_counter basic_q_cnts[] = {
0022     INIT_Q_COUNTER(rx_write_requests),
0023     INIT_Q_COUNTER(rx_read_requests),
0024     INIT_Q_COUNTER(rx_atomic_requests),
0025     INIT_Q_COUNTER(out_of_buffer),
0026 };
0027 
0028 static const struct mlx5_ib_counter out_of_seq_q_cnts[] = {
0029     INIT_Q_COUNTER(out_of_sequence),
0030 };
0031 
0032 static const struct mlx5_ib_counter retrans_q_cnts[] = {
0033     INIT_Q_COUNTER(duplicate_request),
0034     INIT_Q_COUNTER(rnr_nak_retry_err),
0035     INIT_Q_COUNTER(packet_seq_err),
0036     INIT_Q_COUNTER(implied_nak_seq_err),
0037     INIT_Q_COUNTER(local_ack_timeout_err),
0038 };
0039 
0040 #define INIT_CONG_COUNTER(_name)        \
0041     { .name = #_name, .offset = \
0042         MLX5_BYTE_OFF(query_cong_statistics_out, _name ## _high)}
0043 
0044 static const struct mlx5_ib_counter cong_cnts[] = {
0045     INIT_CONG_COUNTER(rp_cnp_ignored),
0046     INIT_CONG_COUNTER(rp_cnp_handled),
0047     INIT_CONG_COUNTER(np_ecn_marked_roce_packets),
0048     INIT_CONG_COUNTER(np_cnp_sent),
0049 };
0050 
0051 static const struct mlx5_ib_counter extended_err_cnts[] = {
0052     INIT_Q_COUNTER(resp_local_length_error),
0053     INIT_Q_COUNTER(resp_cqe_error),
0054     INIT_Q_COUNTER(req_cqe_error),
0055     INIT_Q_COUNTER(req_remote_invalid_request),
0056     INIT_Q_COUNTER(req_remote_access_errors),
0057     INIT_Q_COUNTER(resp_remote_access_errors),
0058     INIT_Q_COUNTER(resp_cqe_flush_error),
0059     INIT_Q_COUNTER(req_cqe_flush_error),
0060 };
0061 
0062 static const struct mlx5_ib_counter roce_accl_cnts[] = {
0063     INIT_Q_COUNTER(roce_adp_retrans),
0064     INIT_Q_COUNTER(roce_adp_retrans_to),
0065     INIT_Q_COUNTER(roce_slow_restart),
0066     INIT_Q_COUNTER(roce_slow_restart_cnps),
0067     INIT_Q_COUNTER(roce_slow_restart_trans),
0068 };
0069 
0070 #define INIT_EXT_PPCNT_COUNTER(_name)       \
0071     { .name = #_name, .offset = \
0072     MLX5_BYTE_OFF(ppcnt_reg, \
0073               counter_set.eth_extended_cntrs_grp_data_layout._name##_high)}
0074 
0075 static const struct mlx5_ib_counter ext_ppcnt_cnts[] = {
0076     INIT_EXT_PPCNT_COUNTER(rx_icrc_encapsulated),
0077 };
0078 
0079 #define INIT_OP_COUNTER(_name, _type)       \
0080     { .name = #_name, .type = MLX5_IB_OPCOUNTER_##_type}
0081 
0082 static const struct mlx5_ib_counter basic_op_cnts[] = {
0083     INIT_OP_COUNTER(cc_rx_ce_pkts, CC_RX_CE_PKTS),
0084 };
0085 
0086 static const struct mlx5_ib_counter rdmarx_cnp_op_cnts[] = {
0087     INIT_OP_COUNTER(cc_rx_cnp_pkts, CC_RX_CNP_PKTS),
0088 };
0089 
0090 static const struct mlx5_ib_counter rdmatx_cnp_op_cnts[] = {
0091     INIT_OP_COUNTER(cc_tx_cnp_pkts, CC_TX_CNP_PKTS),
0092 };
0093 
0094 static int mlx5_ib_read_counters(struct ib_counters *counters,
0095                  struct ib_counters_read_attr *read_attr,
0096                  struct uverbs_attr_bundle *attrs)
0097 {
0098     struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
0099     struct mlx5_read_counters_attr mread_attr = {};
0100     struct mlx5_ib_flow_counters_desc *desc;
0101     int ret, i;
0102 
0103     mutex_lock(&mcounters->mcntrs_mutex);
0104     if (mcounters->cntrs_max_index > read_attr->ncounters) {
0105         ret = -EINVAL;
0106         goto err_bound;
0107     }
0108 
0109     mread_attr.out = kcalloc(mcounters->counters_num, sizeof(u64),
0110                  GFP_KERNEL);
0111     if (!mread_attr.out) {
0112         ret = -ENOMEM;
0113         goto err_bound;
0114     }
0115 
0116     mread_attr.hw_cntrs_hndl = mcounters->hw_cntrs_hndl;
0117     mread_attr.flags = read_attr->flags;
0118     ret = mcounters->read_counters(counters->device, &mread_attr);
0119     if (ret)
0120         goto err_read;
0121 
0122     /* do the pass over the counters data array to assign according to the
0123      * descriptions and indexing pairs
0124      */
0125     desc = mcounters->counters_data;
0126     for (i = 0; i < mcounters->ncounters; i++)
0127         read_attr->counters_buff[desc[i].index] += mread_attr.out[desc[i].description];
0128 
0129 err_read:
0130     kfree(mread_attr.out);
0131 err_bound:
0132     mutex_unlock(&mcounters->mcntrs_mutex);
0133     return ret;
0134 }
0135 
0136 static int mlx5_ib_destroy_counters(struct ib_counters *counters)
0137 {
0138     struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
0139 
0140     mlx5_ib_counters_clear_description(counters);
0141     if (mcounters->hw_cntrs_hndl)
0142         mlx5_fc_destroy(to_mdev(counters->device)->mdev,
0143                 mcounters->hw_cntrs_hndl);
0144     return 0;
0145 }
0146 
0147 static int mlx5_ib_create_counters(struct ib_counters *counters,
0148                    struct uverbs_attr_bundle *attrs)
0149 {
0150     struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
0151 
0152     mutex_init(&mcounters->mcntrs_mutex);
0153     return 0;
0154 }
0155 
0156 
0157 static const struct mlx5_ib_counters *get_counters(struct mlx5_ib_dev *dev,
0158                            u32 port_num)
0159 {
0160     return is_mdev_switchdev_mode(dev->mdev) ? &dev->port[0].cnts :
0161                            &dev->port[port_num].cnts;
0162 }
0163 
0164 /**
0165  * mlx5_ib_get_counters_id - Returns counters id to use for device+port
0166  * @dev:    Pointer to mlx5 IB device
0167  * @port_num:   Zero based port number
0168  *
0169  * mlx5_ib_get_counters_id() Returns counters set id to use for given
0170  * device port combination in switchdev and non switchdev mode of the
0171  * parent device.
0172  */
0173 u16 mlx5_ib_get_counters_id(struct mlx5_ib_dev *dev, u32 port_num)
0174 {
0175     const struct mlx5_ib_counters *cnts = get_counters(dev, port_num);
0176 
0177     return cnts->set_id;
0178 }
0179 
0180 static struct rdma_hw_stats *do_alloc_stats(const struct mlx5_ib_counters *cnts)
0181 {
0182     struct rdma_hw_stats *stats;
0183     u32 num_hw_counters;
0184     int i;
0185 
0186     num_hw_counters = cnts->num_q_counters + cnts->num_cong_counters +
0187               cnts->num_ext_ppcnt_counters;
0188     stats = rdma_alloc_hw_stats_struct(cnts->descs,
0189                        num_hw_counters +
0190                        cnts->num_op_counters,
0191                        RDMA_HW_STATS_DEFAULT_LIFESPAN);
0192     if (!stats)
0193         return NULL;
0194 
0195     for (i = 0; i < cnts->num_op_counters; i++)
0196         set_bit(num_hw_counters + i, stats->is_disabled);
0197 
0198     return stats;
0199 }
0200 
0201 static struct rdma_hw_stats *
0202 mlx5_ib_alloc_hw_device_stats(struct ib_device *ibdev)
0203 {
0204     struct mlx5_ib_dev *dev = to_mdev(ibdev);
0205     const struct mlx5_ib_counters *cnts = &dev->port[0].cnts;
0206 
0207     return do_alloc_stats(cnts);
0208 }
0209 
0210 static struct rdma_hw_stats *
0211 mlx5_ib_alloc_hw_port_stats(struct ib_device *ibdev, u32 port_num)
0212 {
0213     struct mlx5_ib_dev *dev = to_mdev(ibdev);
0214     const struct mlx5_ib_counters *cnts = &dev->port[port_num - 1].cnts;
0215 
0216     return do_alloc_stats(cnts);
0217 }
0218 
0219 static int mlx5_ib_query_q_counters(struct mlx5_core_dev *mdev,
0220                     const struct mlx5_ib_counters *cnts,
0221                     struct rdma_hw_stats *stats,
0222                     u16 set_id)
0223 {
0224     u32 out[MLX5_ST_SZ_DW(query_q_counter_out)] = {};
0225     u32 in[MLX5_ST_SZ_DW(query_q_counter_in)] = {};
0226     __be32 val;
0227     int ret, i;
0228 
0229     MLX5_SET(query_q_counter_in, in, opcode, MLX5_CMD_OP_QUERY_Q_COUNTER);
0230     MLX5_SET(query_q_counter_in, in, counter_set_id, set_id);
0231     ret = mlx5_cmd_exec_inout(mdev, query_q_counter, in, out);
0232     if (ret)
0233         return ret;
0234 
0235     for (i = 0; i < cnts->num_q_counters; i++) {
0236         val = *(__be32 *)((void *)out + cnts->offsets[i]);
0237         stats->value[i] = (u64)be32_to_cpu(val);
0238     }
0239 
0240     return 0;
0241 }
0242 
0243 static int mlx5_ib_query_ext_ppcnt_counters(struct mlx5_ib_dev *dev,
0244                         const struct mlx5_ib_counters *cnts,
0245                         struct rdma_hw_stats *stats)
0246 {
0247     int offset = cnts->num_q_counters + cnts->num_cong_counters;
0248     u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {};
0249     int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
0250     int ret, i;
0251     void *out;
0252 
0253     out = kvzalloc(sz, GFP_KERNEL);
0254     if (!out)
0255         return -ENOMEM;
0256 
0257     MLX5_SET(ppcnt_reg, in, local_port, 1);
0258     MLX5_SET(ppcnt_reg, in, grp, MLX5_ETHERNET_EXTENDED_COUNTERS_GROUP);
0259     ret = mlx5_core_access_reg(dev->mdev, in, sz, out, sz, MLX5_REG_PPCNT,
0260                    0, 0);
0261     if (ret)
0262         goto free;
0263 
0264     for (i = 0; i < cnts->num_ext_ppcnt_counters; i++)
0265         stats->value[i + offset] =
0266             be64_to_cpup((__be64 *)(out +
0267                     cnts->offsets[i + offset]));
0268 free:
0269     kvfree(out);
0270     return ret;
0271 }
0272 
0273 static int do_get_hw_stats(struct ib_device *ibdev,
0274                struct rdma_hw_stats *stats,
0275                u32 port_num, int index)
0276 {
0277     struct mlx5_ib_dev *dev = to_mdev(ibdev);
0278     const struct mlx5_ib_counters *cnts = get_counters(dev, port_num - 1);
0279     struct mlx5_core_dev *mdev;
0280     int ret, num_counters;
0281     u32 mdev_port_num;
0282 
0283     if (!stats)
0284         return -EINVAL;
0285 
0286     num_counters = cnts->num_q_counters +
0287                cnts->num_cong_counters +
0288                cnts->num_ext_ppcnt_counters;
0289 
0290     /* q_counters are per IB device, query the master mdev */
0291     ret = mlx5_ib_query_q_counters(dev->mdev, cnts, stats, cnts->set_id);
0292     if (ret)
0293         return ret;
0294 
0295     if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) {
0296         ret =  mlx5_ib_query_ext_ppcnt_counters(dev, cnts, stats);
0297         if (ret)
0298             return ret;
0299     }
0300 
0301     if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
0302         mdev = mlx5_ib_get_native_port_mdev(dev, port_num,
0303                             &mdev_port_num);
0304         if (!mdev) {
0305             /* If port is not affiliated yet, its in down state
0306              * which doesn't have any counters yet, so it would be
0307              * zero. So no need to read from the HCA.
0308              */
0309             goto done;
0310         }
0311         ret = mlx5_lag_query_cong_counters(dev->mdev,
0312                            stats->value +
0313                            cnts->num_q_counters,
0314                            cnts->num_cong_counters,
0315                            cnts->offsets +
0316                            cnts->num_q_counters);
0317 
0318         mlx5_ib_put_native_port_mdev(dev, port_num);
0319         if (ret)
0320             return ret;
0321     }
0322 
0323 done:
0324     return num_counters;
0325 }
0326 
0327 static int do_get_op_stat(struct ib_device *ibdev,
0328               struct rdma_hw_stats *stats,
0329               u32 port_num, int index)
0330 {
0331     struct mlx5_ib_dev *dev = to_mdev(ibdev);
0332     const struct mlx5_ib_counters *cnts;
0333     const struct mlx5_ib_op_fc *opfcs;
0334     u64 packets = 0, bytes;
0335     u32 type;
0336     int ret;
0337 
0338     cnts = get_counters(dev, port_num - 1);
0339     opfcs = cnts->opfcs;
0340     type = *(u32 *)cnts->descs[index].priv;
0341     if (type >= MLX5_IB_OPCOUNTER_MAX)
0342         return -EINVAL;
0343 
0344     if (!opfcs[type].fc)
0345         goto out;
0346 
0347     ret = mlx5_fc_query(dev->mdev, opfcs[type].fc,
0348                 &packets, &bytes);
0349     if (ret)
0350         return ret;
0351 
0352 out:
0353     stats->value[index] = packets;
0354     return index;
0355 }
0356 
0357 static int do_get_op_stats(struct ib_device *ibdev,
0358                struct rdma_hw_stats *stats,
0359                u32 port_num)
0360 {
0361     struct mlx5_ib_dev *dev = to_mdev(ibdev);
0362     const struct mlx5_ib_counters *cnts;
0363     int index, ret, num_hw_counters;
0364 
0365     cnts = get_counters(dev, port_num - 1);
0366     num_hw_counters = cnts->num_q_counters + cnts->num_cong_counters +
0367               cnts->num_ext_ppcnt_counters;
0368     for (index = num_hw_counters;
0369          index < (num_hw_counters + cnts->num_op_counters); index++) {
0370         ret = do_get_op_stat(ibdev, stats, port_num, index);
0371         if (ret != index)
0372             return ret;
0373     }
0374 
0375     return cnts->num_op_counters;
0376 }
0377 
0378 static int mlx5_ib_get_hw_stats(struct ib_device *ibdev,
0379                 struct rdma_hw_stats *stats,
0380                 u32 port_num, int index)
0381 {
0382     int num_counters, num_hw_counters, num_op_counters;
0383     struct mlx5_ib_dev *dev = to_mdev(ibdev);
0384     const struct mlx5_ib_counters *cnts;
0385 
0386     cnts = get_counters(dev, port_num - 1);
0387     num_hw_counters = cnts->num_q_counters + cnts->num_cong_counters +
0388         cnts->num_ext_ppcnt_counters;
0389     num_counters = num_hw_counters + cnts->num_op_counters;
0390 
0391     if (index < 0 || index > num_counters)
0392         return -EINVAL;
0393     else if (index > 0 && index < num_hw_counters)
0394         return do_get_hw_stats(ibdev, stats, port_num, index);
0395     else if (index >= num_hw_counters && index < num_counters)
0396         return do_get_op_stat(ibdev, stats, port_num, index);
0397 
0398     num_hw_counters = do_get_hw_stats(ibdev, stats, port_num, index);
0399     if (num_hw_counters < 0)
0400         return num_hw_counters;
0401 
0402     num_op_counters = do_get_op_stats(ibdev, stats, port_num);
0403     if (num_op_counters < 0)
0404         return num_op_counters;
0405 
0406     return num_hw_counters + num_op_counters;
0407 }
0408 
0409 static struct rdma_hw_stats *
0410 mlx5_ib_counter_alloc_stats(struct rdma_counter *counter)
0411 {
0412     struct mlx5_ib_dev *dev = to_mdev(counter->device);
0413     const struct mlx5_ib_counters *cnts =
0414         get_counters(dev, counter->port - 1);
0415 
0416     return do_alloc_stats(cnts);
0417 }
0418 
0419 static int mlx5_ib_counter_update_stats(struct rdma_counter *counter)
0420 {
0421     struct mlx5_ib_dev *dev = to_mdev(counter->device);
0422     const struct mlx5_ib_counters *cnts =
0423         get_counters(dev, counter->port - 1);
0424 
0425     return mlx5_ib_query_q_counters(dev->mdev, cnts,
0426                     counter->stats, counter->id);
0427 }
0428 
0429 static int mlx5_ib_counter_dealloc(struct rdma_counter *counter)
0430 {
0431     struct mlx5_ib_dev *dev = to_mdev(counter->device);
0432     u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {};
0433 
0434     if (!counter->id)
0435         return 0;
0436 
0437     MLX5_SET(dealloc_q_counter_in, in, opcode,
0438          MLX5_CMD_OP_DEALLOC_Q_COUNTER);
0439     MLX5_SET(dealloc_q_counter_in, in, counter_set_id, counter->id);
0440     return mlx5_cmd_exec_in(dev->mdev, dealloc_q_counter, in);
0441 }
0442 
0443 static int mlx5_ib_counter_bind_qp(struct rdma_counter *counter,
0444                    struct ib_qp *qp)
0445 {
0446     struct mlx5_ib_dev *dev = to_mdev(qp->device);
0447     int err;
0448 
0449     if (!counter->id) {
0450         u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {};
0451         u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {};
0452 
0453         MLX5_SET(alloc_q_counter_in, in, opcode,
0454              MLX5_CMD_OP_ALLOC_Q_COUNTER);
0455         MLX5_SET(alloc_q_counter_in, in, uid, MLX5_SHARED_RESOURCE_UID);
0456         err = mlx5_cmd_exec_inout(dev->mdev, alloc_q_counter, in, out);
0457         if (err)
0458             return err;
0459         counter->id =
0460             MLX5_GET(alloc_q_counter_out, out, counter_set_id);
0461     }
0462 
0463     err = mlx5_ib_qp_set_counter(qp, counter);
0464     if (err)
0465         goto fail_set_counter;
0466 
0467     return 0;
0468 
0469 fail_set_counter:
0470     mlx5_ib_counter_dealloc(counter);
0471     counter->id = 0;
0472 
0473     return err;
0474 }
0475 
0476 static int mlx5_ib_counter_unbind_qp(struct ib_qp *qp)
0477 {
0478     return mlx5_ib_qp_set_counter(qp, NULL);
0479 }
0480 
0481 static void mlx5_ib_fill_counters(struct mlx5_ib_dev *dev,
0482                   struct rdma_stat_desc *descs, size_t *offsets)
0483 {
0484     int i;
0485     int j = 0;
0486 
0487     for (i = 0; i < ARRAY_SIZE(basic_q_cnts); i++, j++) {
0488         descs[j].name = basic_q_cnts[i].name;
0489         offsets[j] = basic_q_cnts[i].offset;
0490     }
0491 
0492     if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt)) {
0493         for (i = 0; i < ARRAY_SIZE(out_of_seq_q_cnts); i++, j++) {
0494             descs[j].name = out_of_seq_q_cnts[i].name;
0495             offsets[j] = out_of_seq_q_cnts[i].offset;
0496         }
0497     }
0498 
0499     if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters)) {
0500         for (i = 0; i < ARRAY_SIZE(retrans_q_cnts); i++, j++) {
0501             descs[j].name = retrans_q_cnts[i].name;
0502             offsets[j] = retrans_q_cnts[i].offset;
0503         }
0504     }
0505 
0506     if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters)) {
0507         for (i = 0; i < ARRAY_SIZE(extended_err_cnts); i++, j++) {
0508             descs[j].name = extended_err_cnts[i].name;
0509             offsets[j] = extended_err_cnts[i].offset;
0510         }
0511     }
0512 
0513     if (MLX5_CAP_GEN(dev->mdev, roce_accl)) {
0514         for (i = 0; i < ARRAY_SIZE(roce_accl_cnts); i++, j++) {
0515             descs[j].name = roce_accl_cnts[i].name;
0516             offsets[j] = roce_accl_cnts[i].offset;
0517         }
0518     }
0519 
0520     if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
0521         for (i = 0; i < ARRAY_SIZE(cong_cnts); i++, j++) {
0522             descs[j].name = cong_cnts[i].name;
0523             offsets[j] = cong_cnts[i].offset;
0524         }
0525     }
0526 
0527     if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) {
0528         for (i = 0; i < ARRAY_SIZE(ext_ppcnt_cnts); i++, j++) {
0529             descs[j].name = ext_ppcnt_cnts[i].name;
0530             offsets[j] = ext_ppcnt_cnts[i].offset;
0531         }
0532     }
0533 
0534     for (i = 0; i < ARRAY_SIZE(basic_op_cnts); i++, j++) {
0535         descs[j].name = basic_op_cnts[i].name;
0536         descs[j].flags |= IB_STAT_FLAG_OPTIONAL;
0537         descs[j].priv = &basic_op_cnts[i].type;
0538     }
0539 
0540     if (MLX5_CAP_FLOWTABLE(dev->mdev,
0541                    ft_field_support_2_nic_receive_rdma.bth_opcode)) {
0542         for (i = 0; i < ARRAY_SIZE(rdmarx_cnp_op_cnts); i++, j++) {
0543             descs[j].name = rdmarx_cnp_op_cnts[i].name;
0544             descs[j].flags |= IB_STAT_FLAG_OPTIONAL;
0545             descs[j].priv = &rdmarx_cnp_op_cnts[i].type;
0546         }
0547     }
0548 
0549     if (MLX5_CAP_FLOWTABLE(dev->mdev,
0550                    ft_field_support_2_nic_transmit_rdma.bth_opcode)) {
0551         for (i = 0; i < ARRAY_SIZE(rdmatx_cnp_op_cnts); i++, j++) {
0552             descs[j].name = rdmatx_cnp_op_cnts[i].name;
0553             descs[j].flags |= IB_STAT_FLAG_OPTIONAL;
0554             descs[j].priv = &rdmatx_cnp_op_cnts[i].type;
0555         }
0556     }
0557 }
0558 
0559 
0560 static int __mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev,
0561                     struct mlx5_ib_counters *cnts)
0562 {
0563     u32 num_counters, num_op_counters;
0564 
0565     num_counters = ARRAY_SIZE(basic_q_cnts);
0566 
0567     if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt))
0568         num_counters += ARRAY_SIZE(out_of_seq_q_cnts);
0569 
0570     if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters))
0571         num_counters += ARRAY_SIZE(retrans_q_cnts);
0572 
0573     if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters))
0574         num_counters += ARRAY_SIZE(extended_err_cnts);
0575 
0576     if (MLX5_CAP_GEN(dev->mdev, roce_accl))
0577         num_counters += ARRAY_SIZE(roce_accl_cnts);
0578 
0579     cnts->num_q_counters = num_counters;
0580 
0581     if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
0582         cnts->num_cong_counters = ARRAY_SIZE(cong_cnts);
0583         num_counters += ARRAY_SIZE(cong_cnts);
0584     }
0585     if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) {
0586         cnts->num_ext_ppcnt_counters = ARRAY_SIZE(ext_ppcnt_cnts);
0587         num_counters += ARRAY_SIZE(ext_ppcnt_cnts);
0588     }
0589 
0590     num_op_counters = ARRAY_SIZE(basic_op_cnts);
0591 
0592     if (MLX5_CAP_FLOWTABLE(dev->mdev,
0593                    ft_field_support_2_nic_receive_rdma.bth_opcode))
0594         num_op_counters += ARRAY_SIZE(rdmarx_cnp_op_cnts);
0595 
0596     if (MLX5_CAP_FLOWTABLE(dev->mdev,
0597                    ft_field_support_2_nic_transmit_rdma.bth_opcode))
0598         num_op_counters += ARRAY_SIZE(rdmatx_cnp_op_cnts);
0599 
0600     cnts->num_op_counters = num_op_counters;
0601     num_counters += num_op_counters;
0602     cnts->descs = kcalloc(num_counters,
0603                   sizeof(struct rdma_stat_desc), GFP_KERNEL);
0604     if (!cnts->descs)
0605         return -ENOMEM;
0606 
0607     cnts->offsets = kcalloc(num_counters,
0608                 sizeof(*cnts->offsets), GFP_KERNEL);
0609     if (!cnts->offsets)
0610         goto err;
0611 
0612     return 0;
0613 
0614 err:
0615     kfree(cnts->descs);
0616     cnts->descs = NULL;
0617     return -ENOMEM;
0618 }
0619 
0620 static void mlx5_ib_dealloc_counters(struct mlx5_ib_dev *dev)
0621 {
0622     u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {};
0623     int num_cnt_ports;
0624     int i, j;
0625 
0626     num_cnt_ports = is_mdev_switchdev_mode(dev->mdev) ? 1 : dev->num_ports;
0627 
0628     MLX5_SET(dealloc_q_counter_in, in, opcode,
0629          MLX5_CMD_OP_DEALLOC_Q_COUNTER);
0630 
0631     for (i = 0; i < num_cnt_ports; i++) {
0632         if (dev->port[i].cnts.set_id) {
0633             MLX5_SET(dealloc_q_counter_in, in, counter_set_id,
0634                  dev->port[i].cnts.set_id);
0635             mlx5_cmd_exec_in(dev->mdev, dealloc_q_counter, in);
0636         }
0637         kfree(dev->port[i].cnts.descs);
0638         kfree(dev->port[i].cnts.offsets);
0639 
0640         for (j = 0; j < MLX5_IB_OPCOUNTER_MAX; j++) {
0641             if (!dev->port[i].cnts.opfcs[j].fc)
0642                 continue;
0643 
0644             if (IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS))
0645                 mlx5_ib_fs_remove_op_fc(dev,
0646                     &dev->port[i].cnts.opfcs[j], j);
0647             mlx5_fc_destroy(dev->mdev,
0648                     dev->port[i].cnts.opfcs[j].fc);
0649             dev->port[i].cnts.opfcs[j].fc = NULL;
0650         }
0651     }
0652 }
0653 
0654 static int mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev)
0655 {
0656     u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {};
0657     u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {};
0658     int num_cnt_ports;
0659     int err = 0;
0660     int i;
0661     bool is_shared;
0662 
0663     MLX5_SET(alloc_q_counter_in, in, opcode, MLX5_CMD_OP_ALLOC_Q_COUNTER);
0664     is_shared = MLX5_CAP_GEN(dev->mdev, log_max_uctx) != 0;
0665     num_cnt_ports = is_mdev_switchdev_mode(dev->mdev) ? 1 : dev->num_ports;
0666 
0667     for (i = 0; i < num_cnt_ports; i++) {
0668         err = __mlx5_ib_alloc_counters(dev, &dev->port[i].cnts);
0669         if (err)
0670             goto err_alloc;
0671 
0672         mlx5_ib_fill_counters(dev, dev->port[i].cnts.descs,
0673                       dev->port[i].cnts.offsets);
0674 
0675         MLX5_SET(alloc_q_counter_in, in, uid,
0676              is_shared ? MLX5_SHARED_RESOURCE_UID : 0);
0677 
0678         err = mlx5_cmd_exec_inout(dev->mdev, alloc_q_counter, in, out);
0679         if (err) {
0680             mlx5_ib_warn(dev,
0681                      "couldn't allocate queue counter for port %d, err %d\n",
0682                      i + 1, err);
0683             goto err_alloc;
0684         }
0685 
0686         dev->port[i].cnts.set_id =
0687             MLX5_GET(alloc_q_counter_out, out, counter_set_id);
0688     }
0689     return 0;
0690 
0691 err_alloc:
0692     mlx5_ib_dealloc_counters(dev);
0693     return err;
0694 }
0695 
0696 static int read_flow_counters(struct ib_device *ibdev,
0697                   struct mlx5_read_counters_attr *read_attr)
0698 {
0699     struct mlx5_fc *fc = read_attr->hw_cntrs_hndl;
0700     struct mlx5_ib_dev *dev = to_mdev(ibdev);
0701 
0702     return mlx5_fc_query(dev->mdev, fc,
0703                  &read_attr->out[IB_COUNTER_PACKETS],
0704                  &read_attr->out[IB_COUNTER_BYTES]);
0705 }
0706 
0707 /* flow counters currently expose two counters packets and bytes */
0708 #define FLOW_COUNTERS_NUM 2
0709 static int counters_set_description(
0710     struct ib_counters *counters, enum mlx5_ib_counters_type counters_type,
0711     struct mlx5_ib_flow_counters_desc *desc_data, u32 ncounters)
0712 {
0713     struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
0714     u32 cntrs_max_index = 0;
0715     int i;
0716 
0717     if (counters_type != MLX5_IB_COUNTERS_FLOW)
0718         return -EINVAL;
0719 
0720     /* init the fields for the object */
0721     mcounters->type = counters_type;
0722     mcounters->read_counters = read_flow_counters;
0723     mcounters->counters_num = FLOW_COUNTERS_NUM;
0724     mcounters->ncounters = ncounters;
0725     /* each counter entry have both description and index pair */
0726     for (i = 0; i < ncounters; i++) {
0727         if (desc_data[i].description > IB_COUNTER_BYTES)
0728             return -EINVAL;
0729 
0730         if (cntrs_max_index <= desc_data[i].index)
0731             cntrs_max_index = desc_data[i].index + 1;
0732     }
0733 
0734     mutex_lock(&mcounters->mcntrs_mutex);
0735     mcounters->counters_data = desc_data;
0736     mcounters->cntrs_max_index = cntrs_max_index;
0737     mutex_unlock(&mcounters->mcntrs_mutex);
0738 
0739     return 0;
0740 }
0741 
0742 #define MAX_COUNTERS_NUM (USHRT_MAX / (sizeof(u32) * 2))
0743 int mlx5_ib_flow_counters_set_data(struct ib_counters *ibcounters,
0744                    struct mlx5_ib_create_flow *ucmd)
0745 {
0746     struct mlx5_ib_mcounters *mcounters = to_mcounters(ibcounters);
0747     struct mlx5_ib_flow_counters_data *cntrs_data = NULL;
0748     struct mlx5_ib_flow_counters_desc *desc_data = NULL;
0749     bool hw_hndl = false;
0750     int ret = 0;
0751 
0752     if (ucmd && ucmd->ncounters_data != 0) {
0753         cntrs_data = ucmd->data;
0754         if (cntrs_data->ncounters > MAX_COUNTERS_NUM)
0755             return -EINVAL;
0756 
0757         desc_data = kcalloc(cntrs_data->ncounters,
0758                     sizeof(*desc_data),
0759                     GFP_KERNEL);
0760         if (!desc_data)
0761             return  -ENOMEM;
0762 
0763         if (copy_from_user(desc_data,
0764                    u64_to_user_ptr(cntrs_data->counters_data),
0765                    sizeof(*desc_data) * cntrs_data->ncounters)) {
0766             ret = -EFAULT;
0767             goto free;
0768         }
0769     }
0770 
0771     if (!mcounters->hw_cntrs_hndl) {
0772         mcounters->hw_cntrs_hndl = mlx5_fc_create(
0773             to_mdev(ibcounters->device)->mdev, false);
0774         if (IS_ERR(mcounters->hw_cntrs_hndl)) {
0775             ret = PTR_ERR(mcounters->hw_cntrs_hndl);
0776             goto free;
0777         }
0778         hw_hndl = true;
0779     }
0780 
0781     if (desc_data) {
0782         /* counters already bound to at least one flow */
0783         if (mcounters->cntrs_max_index) {
0784             ret = -EINVAL;
0785             goto free_hndl;
0786         }
0787 
0788         ret = counters_set_description(ibcounters,
0789                            MLX5_IB_COUNTERS_FLOW,
0790                            desc_data,
0791                            cntrs_data->ncounters);
0792         if (ret)
0793             goto free_hndl;
0794 
0795     } else if (!mcounters->cntrs_max_index) {
0796         /* counters not bound yet, must have udata passed */
0797         ret = -EINVAL;
0798         goto free_hndl;
0799     }
0800 
0801     return 0;
0802 
0803 free_hndl:
0804     if (hw_hndl) {
0805         mlx5_fc_destroy(to_mdev(ibcounters->device)->mdev,
0806                 mcounters->hw_cntrs_hndl);
0807         mcounters->hw_cntrs_hndl = NULL;
0808     }
0809 free:
0810     kfree(desc_data);
0811     return ret;
0812 }
0813 
0814 void mlx5_ib_counters_clear_description(struct ib_counters *counters)
0815 {
0816     struct mlx5_ib_mcounters *mcounters;
0817 
0818     if (!counters || atomic_read(&counters->usecnt) != 1)
0819         return;
0820 
0821     mcounters = to_mcounters(counters);
0822 
0823     mutex_lock(&mcounters->mcntrs_mutex);
0824     kfree(mcounters->counters_data);
0825     mcounters->counters_data = NULL;
0826     mcounters->cntrs_max_index = 0;
0827     mutex_unlock(&mcounters->mcntrs_mutex);
0828 }
0829 
0830 static int mlx5_ib_modify_stat(struct ib_device *device, u32 port,
0831                    unsigned int index, bool enable)
0832 {
0833     struct mlx5_ib_dev *dev = to_mdev(device);
0834     struct mlx5_ib_counters *cnts;
0835     struct mlx5_ib_op_fc *opfc;
0836     u32 num_hw_counters, type;
0837     int ret;
0838 
0839     cnts = &dev->port[port - 1].cnts;
0840     num_hw_counters = cnts->num_q_counters + cnts->num_cong_counters +
0841         cnts->num_ext_ppcnt_counters;
0842     if (index < num_hw_counters ||
0843         index >= (num_hw_counters + cnts->num_op_counters))
0844         return -EINVAL;
0845 
0846     if (!(cnts->descs[index].flags & IB_STAT_FLAG_OPTIONAL))
0847         return -EINVAL;
0848 
0849     type = *(u32 *)cnts->descs[index].priv;
0850     if (type >= MLX5_IB_OPCOUNTER_MAX)
0851         return -EINVAL;
0852 
0853     opfc = &cnts->opfcs[type];
0854 
0855     if (enable) {
0856         if (opfc->fc)
0857             return -EEXIST;
0858 
0859         opfc->fc = mlx5_fc_create(dev->mdev, false);
0860         if (IS_ERR(opfc->fc))
0861             return PTR_ERR(opfc->fc);
0862 
0863         ret = mlx5_ib_fs_add_op_fc(dev, port, opfc, type);
0864         if (ret) {
0865             mlx5_fc_destroy(dev->mdev, opfc->fc);
0866             opfc->fc = NULL;
0867         }
0868         return ret;
0869     }
0870 
0871     if (!opfc->fc)
0872         return -EINVAL;
0873 
0874     mlx5_ib_fs_remove_op_fc(dev, opfc, type);
0875     mlx5_fc_destroy(dev->mdev, opfc->fc);
0876     opfc->fc = NULL;
0877     return 0;
0878 }
0879 
0880 static const struct ib_device_ops hw_stats_ops = {
0881     .alloc_hw_port_stats = mlx5_ib_alloc_hw_port_stats,
0882     .get_hw_stats = mlx5_ib_get_hw_stats,
0883     .counter_bind_qp = mlx5_ib_counter_bind_qp,
0884     .counter_unbind_qp = mlx5_ib_counter_unbind_qp,
0885     .counter_dealloc = mlx5_ib_counter_dealloc,
0886     .counter_alloc_stats = mlx5_ib_counter_alloc_stats,
0887     .counter_update_stats = mlx5_ib_counter_update_stats,
0888     .modify_hw_stat = IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS) ?
0889               mlx5_ib_modify_stat : NULL,
0890 };
0891 
0892 static const struct ib_device_ops hw_switchdev_stats_ops = {
0893     .alloc_hw_device_stats = mlx5_ib_alloc_hw_device_stats,
0894     .get_hw_stats = mlx5_ib_get_hw_stats,
0895     .counter_bind_qp = mlx5_ib_counter_bind_qp,
0896     .counter_unbind_qp = mlx5_ib_counter_unbind_qp,
0897     .counter_dealloc = mlx5_ib_counter_dealloc,
0898     .counter_alloc_stats = mlx5_ib_counter_alloc_stats,
0899     .counter_update_stats = mlx5_ib_counter_update_stats,
0900 };
0901 
0902 static const struct ib_device_ops counters_ops = {
0903     .create_counters = mlx5_ib_create_counters,
0904     .destroy_counters = mlx5_ib_destroy_counters,
0905     .read_counters = mlx5_ib_read_counters,
0906 
0907     INIT_RDMA_OBJ_SIZE(ib_counters, mlx5_ib_mcounters, ibcntrs),
0908 };
0909 
0910 int mlx5_ib_counters_init(struct mlx5_ib_dev *dev)
0911 {
0912     ib_set_device_ops(&dev->ib_dev, &counters_ops);
0913 
0914     if (!MLX5_CAP_GEN(dev->mdev, max_qp_cnt))
0915         return 0;
0916 
0917     if (is_mdev_switchdev_mode(dev->mdev))
0918         ib_set_device_ops(&dev->ib_dev, &hw_switchdev_stats_ops);
0919     else
0920         ib_set_device_ops(&dev->ib_dev, &hw_stats_ops);
0921     return mlx5_ib_alloc_counters(dev);
0922 }
0923 
0924 void mlx5_ib_counters_cleanup(struct mlx5_ib_dev *dev)
0925 {
0926     if (!MLX5_CAP_GEN(dev->mdev, max_qp_cnt))
0927         return;
0928 
0929     mlx5_ib_dealloc_counters(dev);
0930 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!