Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
0002 // Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES.
0003 
0004 #include "rss.h"
0005 
0006 #define mlx5e_rss_warn(__dev, format, ...)          \
0007     dev_warn((__dev)->device, "%s:%d:(pid %d): " format,    \
0008          __func__, __LINE__, current->pid,      \
0009          ##__VA_ARGS__)
0010 
0011 static const struct mlx5e_rss_params_traffic_type rss_default_config[MLX5E_NUM_INDIR_TIRS] = {
0012     [MLX5_TT_IPV4_TCP] = {
0013         .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4,
0014         .l4_prot_type = MLX5_L4_PROT_TYPE_TCP,
0015         .rx_hash_fields = MLX5_HASH_IP_L4PORTS,
0016     },
0017     [MLX5_TT_IPV6_TCP] = {
0018         .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6,
0019         .l4_prot_type = MLX5_L4_PROT_TYPE_TCP,
0020         .rx_hash_fields = MLX5_HASH_IP_L4PORTS,
0021     },
0022     [MLX5_TT_IPV4_UDP] = {
0023         .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4,
0024         .l4_prot_type = MLX5_L4_PROT_TYPE_UDP,
0025         .rx_hash_fields = MLX5_HASH_IP_L4PORTS,
0026     },
0027     [MLX5_TT_IPV6_UDP] = {
0028         .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6,
0029         .l4_prot_type = MLX5_L4_PROT_TYPE_UDP,
0030         .rx_hash_fields = MLX5_HASH_IP_L4PORTS,
0031     },
0032     [MLX5_TT_IPV4_IPSEC_AH] = {
0033         .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4,
0034         .l4_prot_type = 0,
0035         .rx_hash_fields = MLX5_HASH_IP_IPSEC_SPI,
0036     },
0037     [MLX5_TT_IPV6_IPSEC_AH] = {
0038         .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6,
0039         .l4_prot_type = 0,
0040         .rx_hash_fields = MLX5_HASH_IP_IPSEC_SPI,
0041     },
0042     [MLX5_TT_IPV4_IPSEC_ESP] = {
0043         .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4,
0044         .l4_prot_type = 0,
0045         .rx_hash_fields = MLX5_HASH_IP_IPSEC_SPI,
0046     },
0047     [MLX5_TT_IPV6_IPSEC_ESP] = {
0048         .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6,
0049         .l4_prot_type = 0,
0050         .rx_hash_fields = MLX5_HASH_IP_IPSEC_SPI,
0051     },
0052     [MLX5_TT_IPV4] = {
0053         .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4,
0054         .l4_prot_type = 0,
0055         .rx_hash_fields = MLX5_HASH_IP,
0056     },
0057     [MLX5_TT_IPV6] = {
0058         .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6,
0059         .l4_prot_type = 0,
0060         .rx_hash_fields = MLX5_HASH_IP,
0061     },
0062 };
0063 
0064 struct mlx5e_rss_params_traffic_type
0065 mlx5e_rss_get_default_tt_config(enum mlx5_traffic_types tt)
0066 {
0067     return rss_default_config[tt];
0068 }
0069 
0070 struct mlx5e_rss {
0071     struct mlx5e_rss_params_hash hash;
0072     struct mlx5e_rss_params_indir indir;
0073     u32 rx_hash_fields[MLX5E_NUM_INDIR_TIRS];
0074     struct mlx5e_tir *tir[MLX5E_NUM_INDIR_TIRS];
0075     struct mlx5e_tir *inner_tir[MLX5E_NUM_INDIR_TIRS];
0076     struct mlx5e_rqt rqt;
0077     struct mlx5_core_dev *mdev;
0078     u32 drop_rqn;
0079     bool inner_ft_support;
0080     bool enabled;
0081     refcount_t refcnt;
0082 };
0083 
0084 struct mlx5e_rss *mlx5e_rss_alloc(void)
0085 {
0086     return kvzalloc(sizeof(struct mlx5e_rss), GFP_KERNEL);
0087 }
0088 
0089 void mlx5e_rss_free(struct mlx5e_rss *rss)
0090 {
0091     kvfree(rss);
0092 }
0093 
0094 static void mlx5e_rss_params_init(struct mlx5e_rss *rss)
0095 {
0096     enum mlx5_traffic_types tt;
0097 
0098     rss->hash.hfunc = ETH_RSS_HASH_TOP;
0099     netdev_rss_key_fill(rss->hash.toeplitz_hash_key,
0100                 sizeof(rss->hash.toeplitz_hash_key));
0101     for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++)
0102         rss->rx_hash_fields[tt] =
0103             mlx5e_rss_get_default_tt_config(tt).rx_hash_fields;
0104 }
0105 
0106 static struct mlx5e_tir **rss_get_tirp(struct mlx5e_rss *rss, enum mlx5_traffic_types tt,
0107                        bool inner)
0108 {
0109     return inner ? &rss->inner_tir[tt] : &rss->tir[tt];
0110 }
0111 
0112 static struct mlx5e_tir *rss_get_tir(struct mlx5e_rss *rss, enum mlx5_traffic_types tt,
0113                      bool inner)
0114 {
0115     return *rss_get_tirp(rss, tt, inner);
0116 }
0117 
0118 static struct mlx5e_rss_params_traffic_type
0119 mlx5e_rss_get_tt_config(struct mlx5e_rss *rss, enum mlx5_traffic_types tt)
0120 {
0121     struct mlx5e_rss_params_traffic_type rss_tt;
0122 
0123     rss_tt = mlx5e_rss_get_default_tt_config(tt);
0124     rss_tt.rx_hash_fields = rss->rx_hash_fields[tt];
0125     return rss_tt;
0126 }
0127 
0128 static int mlx5e_rss_create_tir(struct mlx5e_rss *rss,
0129                 enum mlx5_traffic_types tt,
0130                 const struct mlx5e_packet_merge_param *init_pkt_merge_param,
0131                 bool inner)
0132 {
0133     struct mlx5e_rss_params_traffic_type rss_tt;
0134     struct mlx5e_tir_builder *builder;
0135     struct mlx5e_tir **tir_p;
0136     struct mlx5e_tir *tir;
0137     u32 rqtn;
0138     int err;
0139 
0140     if (inner && !rss->inner_ft_support) {
0141         mlx5e_rss_warn(rss->mdev,
0142                    "Cannot create inner indirect TIR[%d], RSS inner FT is not supported.\n",
0143                    tt);
0144         return -EINVAL;
0145     }
0146 
0147     tir_p = rss_get_tirp(rss, tt, inner);
0148     if (*tir_p)
0149         return -EINVAL;
0150 
0151     tir = kvzalloc(sizeof(*tir), GFP_KERNEL);
0152     if (!tir)
0153         return -ENOMEM;
0154 
0155     builder = mlx5e_tir_builder_alloc(false);
0156     if (!builder) {
0157         err = -ENOMEM;
0158         goto free_tir;
0159     }
0160 
0161     rqtn = mlx5e_rqt_get_rqtn(&rss->rqt);
0162     mlx5e_tir_builder_build_rqt(builder, rss->mdev->mlx5e_res.hw_objs.td.tdn,
0163                     rqtn, rss->inner_ft_support);
0164     mlx5e_tir_builder_build_packet_merge(builder, init_pkt_merge_param);
0165     rss_tt = mlx5e_rss_get_tt_config(rss, tt);
0166     mlx5e_tir_builder_build_rss(builder, &rss->hash, &rss_tt, inner);
0167 
0168     err = mlx5e_tir_init(tir, builder, rss->mdev, true);
0169     mlx5e_tir_builder_free(builder);
0170     if (err) {
0171         mlx5e_rss_warn(rss->mdev, "Failed to create %sindirect TIR: err = %d, tt = %d\n",
0172                    inner ? "inner " : "", err, tt);
0173         goto free_tir;
0174     }
0175 
0176     *tir_p = tir;
0177     return 0;
0178 
0179 free_tir:
0180     kvfree(tir);
0181     return err;
0182 }
0183 
0184 static void mlx5e_rss_destroy_tir(struct mlx5e_rss *rss, enum mlx5_traffic_types tt,
0185                   bool inner)
0186 {
0187     struct mlx5e_tir **tir_p;
0188     struct mlx5e_tir *tir;
0189 
0190     tir_p = rss_get_tirp(rss, tt, inner);
0191     if (!*tir_p)
0192         return;
0193 
0194     tir = *tir_p;
0195     mlx5e_tir_destroy(tir);
0196     kvfree(tir);
0197     *tir_p = NULL;
0198 }
0199 
0200 static int mlx5e_rss_create_tirs(struct mlx5e_rss *rss,
0201                  const struct mlx5e_packet_merge_param *init_pkt_merge_param,
0202                  bool inner)
0203 {
0204     enum mlx5_traffic_types tt, max_tt;
0205     int err;
0206 
0207     for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) {
0208         err = mlx5e_rss_create_tir(rss, tt, init_pkt_merge_param, inner);
0209         if (err)
0210             goto err_destroy_tirs;
0211     }
0212 
0213     return 0;
0214 
0215 err_destroy_tirs:
0216     max_tt = tt;
0217     for (tt = 0; tt < max_tt; tt++)
0218         mlx5e_rss_destroy_tir(rss, tt, inner);
0219     return err;
0220 }
0221 
0222 static void mlx5e_rss_destroy_tirs(struct mlx5e_rss *rss, bool inner)
0223 {
0224     enum mlx5_traffic_types tt;
0225 
0226     for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++)
0227         mlx5e_rss_destroy_tir(rss, tt, inner);
0228 }
0229 
0230 static int mlx5e_rss_update_tir(struct mlx5e_rss *rss, enum mlx5_traffic_types tt,
0231                 bool inner)
0232 {
0233     struct mlx5e_rss_params_traffic_type rss_tt;
0234     struct mlx5e_tir_builder *builder;
0235     struct mlx5e_tir *tir;
0236     int err;
0237 
0238     tir = rss_get_tir(rss, tt, inner);
0239     if (!tir)
0240         return 0;
0241 
0242     builder = mlx5e_tir_builder_alloc(true);
0243     if (!builder)
0244         return -ENOMEM;
0245 
0246     rss_tt = mlx5e_rss_get_tt_config(rss, tt);
0247 
0248     mlx5e_tir_builder_build_rss(builder, &rss->hash, &rss_tt, inner);
0249     err = mlx5e_tir_modify(tir, builder);
0250 
0251     mlx5e_tir_builder_free(builder);
0252     return err;
0253 }
0254 
0255 static int mlx5e_rss_update_tirs(struct mlx5e_rss *rss)
0256 {
0257     enum mlx5_traffic_types tt;
0258     int err, retval;
0259 
0260     retval = 0;
0261 
0262     for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) {
0263         err = mlx5e_rss_update_tir(rss, tt, false);
0264         if (err) {
0265             retval = retval ? : err;
0266             mlx5e_rss_warn(rss->mdev,
0267                        "Failed to update RSS hash of indirect TIR for traffic type %d: err = %d\n",
0268                        tt, err);
0269         }
0270 
0271         if (!rss->inner_ft_support)
0272             continue;
0273 
0274         err = mlx5e_rss_update_tir(rss, tt, true);
0275         if (err) {
0276             retval = retval ? : err;
0277             mlx5e_rss_warn(rss->mdev,
0278                        "Failed to update RSS hash of inner indirect TIR for traffic type %d: err = %d\n",
0279                        tt, err);
0280         }
0281     }
0282     return retval;
0283 }
0284 
0285 int mlx5e_rss_init_no_tirs(struct mlx5e_rss *rss, struct mlx5_core_dev *mdev,
0286                bool inner_ft_support, u32 drop_rqn)
0287 {
0288     rss->mdev = mdev;
0289     rss->inner_ft_support = inner_ft_support;
0290     rss->drop_rqn = drop_rqn;
0291 
0292     mlx5e_rss_params_init(rss);
0293     refcount_set(&rss->refcnt, 1);
0294 
0295     return mlx5e_rqt_init_direct(&rss->rqt, mdev, true, drop_rqn);
0296 }
0297 
0298 int mlx5e_rss_init(struct mlx5e_rss *rss, struct mlx5_core_dev *mdev,
0299            bool inner_ft_support, u32 drop_rqn,
0300            const struct mlx5e_packet_merge_param *init_pkt_merge_param)
0301 {
0302     int err;
0303 
0304     err = mlx5e_rss_init_no_tirs(rss, mdev, inner_ft_support, drop_rqn);
0305     if (err)
0306         goto err_out;
0307 
0308     err = mlx5e_rss_create_tirs(rss, init_pkt_merge_param, false);
0309     if (err)
0310         goto err_destroy_rqt;
0311 
0312     if (inner_ft_support) {
0313         err = mlx5e_rss_create_tirs(rss, init_pkt_merge_param, true);
0314         if (err)
0315             goto err_destroy_tirs;
0316     }
0317 
0318     return 0;
0319 
0320 err_destroy_tirs:
0321     mlx5e_rss_destroy_tirs(rss, false);
0322 err_destroy_rqt:
0323     mlx5e_rqt_destroy(&rss->rqt);
0324 err_out:
0325     return err;
0326 }
0327 
0328 int mlx5e_rss_cleanup(struct mlx5e_rss *rss)
0329 {
0330     if (!refcount_dec_if_one(&rss->refcnt))
0331         return -EBUSY;
0332 
0333     mlx5e_rss_destroy_tirs(rss, false);
0334 
0335     if (rss->inner_ft_support)
0336         mlx5e_rss_destroy_tirs(rss, true);
0337 
0338     mlx5e_rqt_destroy(&rss->rqt);
0339 
0340     return 0;
0341 }
0342 
0343 void mlx5e_rss_refcnt_inc(struct mlx5e_rss *rss)
0344 {
0345     refcount_inc(&rss->refcnt);
0346 }
0347 
0348 void mlx5e_rss_refcnt_dec(struct mlx5e_rss *rss)
0349 {
0350     refcount_dec(&rss->refcnt);
0351 }
0352 
0353 unsigned int mlx5e_rss_refcnt_read(struct mlx5e_rss *rss)
0354 {
0355     return refcount_read(&rss->refcnt);
0356 }
0357 
0358 u32 mlx5e_rss_get_tirn(struct mlx5e_rss *rss, enum mlx5_traffic_types tt,
0359                bool inner)
0360 {
0361     struct mlx5e_tir *tir;
0362 
0363     WARN_ON(inner && !rss->inner_ft_support);
0364     tir = rss_get_tir(rss, tt, inner);
0365     WARN_ON(!tir);
0366 
0367     return mlx5e_tir_get_tirn(tir);
0368 }
0369 
0370 /* Fill the "tirn" output parameter.
0371  * Create the requested TIR if it's its first usage.
0372  */
0373 int mlx5e_rss_obtain_tirn(struct mlx5e_rss *rss,
0374               enum mlx5_traffic_types tt,
0375               const struct mlx5e_packet_merge_param *init_pkt_merge_param,
0376               bool inner, u32 *tirn)
0377 {
0378     struct mlx5e_tir *tir;
0379 
0380     tir = rss_get_tir(rss, tt, inner);
0381     if (!tir) { /* TIR doesn't exist, create one */
0382         int err;
0383 
0384         err = mlx5e_rss_create_tir(rss, tt, init_pkt_merge_param, inner);
0385         if (err)
0386             return err;
0387         tir = rss_get_tir(rss, tt, inner);
0388     }
0389 
0390     *tirn = mlx5e_tir_get_tirn(tir);
0391     return 0;
0392 }
0393 
0394 static int mlx5e_rss_apply(struct mlx5e_rss *rss, u32 *rqns, unsigned int num_rqns)
0395 {
0396     int err;
0397 
0398     err = mlx5e_rqt_redirect_indir(&rss->rqt, rqns, num_rqns, rss->hash.hfunc, &rss->indir);
0399     if (err)
0400         mlx5e_rss_warn(rss->mdev, "Failed to redirect RQT %#x to channels: err = %d\n",
0401                    mlx5e_rqt_get_rqtn(&rss->rqt), err);
0402     return err;
0403 }
0404 
0405 void mlx5e_rss_enable(struct mlx5e_rss *rss, u32 *rqns, unsigned int num_rqns)
0406 {
0407     rss->enabled = true;
0408     mlx5e_rss_apply(rss, rqns, num_rqns);
0409 }
0410 
0411 void mlx5e_rss_disable(struct mlx5e_rss *rss)
0412 {
0413     int err;
0414 
0415     rss->enabled = false;
0416     err = mlx5e_rqt_redirect_direct(&rss->rqt, rss->drop_rqn);
0417     if (err)
0418         mlx5e_rss_warn(rss->mdev, "Failed to redirect RQT %#x to drop RQ %#x: err = %d\n",
0419                    mlx5e_rqt_get_rqtn(&rss->rqt), rss->drop_rqn, err);
0420 }
0421 
0422 int mlx5e_rss_packet_merge_set_param(struct mlx5e_rss *rss,
0423                      struct mlx5e_packet_merge_param *pkt_merge_param)
0424 {
0425     struct mlx5e_tir_builder *builder;
0426     enum mlx5_traffic_types tt;
0427     int err, final_err;
0428 
0429     builder = mlx5e_tir_builder_alloc(true);
0430     if (!builder)
0431         return -ENOMEM;
0432 
0433     mlx5e_tir_builder_build_packet_merge(builder, pkt_merge_param);
0434 
0435     final_err = 0;
0436 
0437     for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) {
0438         struct mlx5e_tir *tir;
0439 
0440         tir = rss_get_tir(rss, tt, false);
0441         if (!tir)
0442             goto inner_tir;
0443         err = mlx5e_tir_modify(tir, builder);
0444         if (err) {
0445             mlx5e_rss_warn(rss->mdev, "Failed to update packet merge state of indirect TIR %#x for traffic type %d: err = %d\n",
0446                        mlx5e_tir_get_tirn(tir), tt, err);
0447             if (!final_err)
0448                 final_err = err;
0449         }
0450 
0451 inner_tir:
0452         if (!rss->inner_ft_support)
0453             continue;
0454 
0455         tir = rss_get_tir(rss, tt, true);
0456         if (!tir)
0457             continue;
0458         err = mlx5e_tir_modify(tir, builder);
0459         if (err) {
0460             mlx5e_rss_warn(rss->mdev, "Failed to update packet merge state of inner indirect TIR %#x for traffic type %d: err = %d\n",
0461                        mlx5e_tir_get_tirn(tir), tt, err);
0462             if (!final_err)
0463                 final_err = err;
0464         }
0465     }
0466 
0467     mlx5e_tir_builder_free(builder);
0468     return final_err;
0469 }
0470 
0471 int mlx5e_rss_get_rxfh(struct mlx5e_rss *rss, u32 *indir, u8 *key, u8 *hfunc)
0472 {
0473     unsigned int i;
0474 
0475     if (indir)
0476         for (i = 0; i < MLX5E_INDIR_RQT_SIZE; i++)
0477             indir[i] = rss->indir.table[i];
0478 
0479     if (key)
0480         memcpy(key, rss->hash.toeplitz_hash_key,
0481                sizeof(rss->hash.toeplitz_hash_key));
0482 
0483     if (hfunc)
0484         *hfunc = rss->hash.hfunc;
0485 
0486     return 0;
0487 }
0488 
0489 int mlx5e_rss_set_rxfh(struct mlx5e_rss *rss, const u32 *indir,
0490                const u8 *key, const u8 *hfunc,
0491                u32 *rqns, unsigned int num_rqns)
0492 {
0493     bool changed_indir = false;
0494     bool changed_hash = false;
0495     struct mlx5e_rss *old_rss;
0496     int err = 0;
0497 
0498     old_rss = mlx5e_rss_alloc();
0499     if (!old_rss)
0500         return -ENOMEM;
0501 
0502     *old_rss = *rss;
0503 
0504     if (hfunc && *hfunc != rss->hash.hfunc) {
0505         switch (*hfunc) {
0506         case ETH_RSS_HASH_XOR:
0507         case ETH_RSS_HASH_TOP:
0508             break;
0509         default:
0510             err = -EINVAL;
0511             goto out;
0512         }
0513         changed_hash = true;
0514         changed_indir = true;
0515         rss->hash.hfunc = *hfunc;
0516     }
0517 
0518     if (key) {
0519         if (rss->hash.hfunc == ETH_RSS_HASH_TOP)
0520             changed_hash = true;
0521         memcpy(rss->hash.toeplitz_hash_key, key,
0522                sizeof(rss->hash.toeplitz_hash_key));
0523     }
0524 
0525     if (indir) {
0526         unsigned int i;
0527 
0528         changed_indir = true;
0529 
0530         for (i = 0; i < MLX5E_INDIR_RQT_SIZE; i++)
0531             rss->indir.table[i] = indir[i];
0532     }
0533 
0534     if (changed_indir && rss->enabled) {
0535         err = mlx5e_rss_apply(rss, rqns, num_rqns);
0536         if (err) {
0537             *rss = *old_rss;
0538             goto out;
0539         }
0540     }
0541 
0542     if (changed_hash)
0543         mlx5e_rss_update_tirs(rss);
0544 
0545 out:
0546     mlx5e_rss_free(old_rss);
0547     return err;
0548 }
0549 
0550 struct mlx5e_rss_params_hash mlx5e_rss_get_hash(struct mlx5e_rss *rss)
0551 {
0552     return rss->hash;
0553 }
0554 
0555 u8 mlx5e_rss_get_hash_fields(struct mlx5e_rss *rss, enum mlx5_traffic_types tt)
0556 {
0557     return rss->rx_hash_fields[tt];
0558 }
0559 
0560 int mlx5e_rss_set_hash_fields(struct mlx5e_rss *rss, enum mlx5_traffic_types tt,
0561                   u8 rx_hash_fields)
0562 {
0563     u8 old_rx_hash_fields;
0564     int err;
0565 
0566     old_rx_hash_fields = rss->rx_hash_fields[tt];
0567 
0568     if (old_rx_hash_fields == rx_hash_fields)
0569         return 0;
0570 
0571     rss->rx_hash_fields[tt] = rx_hash_fields;
0572 
0573     err = mlx5e_rss_update_tir(rss, tt, false);
0574     if (err) {
0575         rss->rx_hash_fields[tt] = old_rx_hash_fields;
0576         mlx5e_rss_warn(rss->mdev,
0577                    "Failed to update RSS hash fields of indirect TIR for traffic type %d: err = %d\n",
0578                    tt, err);
0579         return err;
0580     }
0581 
0582     if (!(rss->inner_ft_support))
0583         return 0;
0584 
0585     err = mlx5e_rss_update_tir(rss, tt, true);
0586     if (err) {
0587         /* Partial update happened. Try to revert - it may fail too, but
0588          * there is nothing more we can do.
0589          */
0590         rss->rx_hash_fields[tt] = old_rx_hash_fields;
0591         mlx5e_rss_warn(rss->mdev,
0592                    "Failed to update RSS hash fields of inner indirect TIR for traffic type %d: err = %d\n",
0593                    tt, err);
0594         if (mlx5e_rss_update_tir(rss, tt, false))
0595             mlx5e_rss_warn(rss->mdev,
0596                        "Partial update of RSS hash fields happened: failed to revert indirect TIR for traffic type %d to the old values\n",
0597                        tt);
0598     }
0599 
0600     return err;
0601 }
0602 
0603 void mlx5e_rss_set_indir_uniform(struct mlx5e_rss *rss, unsigned int nch)
0604 {
0605     mlx5e_rss_params_indir_init_uniform(&rss->indir, nch);
0606 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!