Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
0002 /* Copyright (c) 2018 Mellanox Technologies. All rights reserved */
0003 
0004 #include <linux/bitmap.h>
0005 #include <linux/errno.h>
0006 #include <linux/genalloc.h>
0007 #include <linux/gfp.h>
0008 #include <linux/kernel.h>
0009 #include <linux/list.h>
0010 #include <linux/mutex.h>
0011 #include <linux/objagg.h>
0012 #include <linux/rtnetlink.h>
0013 #include <linux/slab.h>
0014 
0015 #include "core.h"
0016 #include "reg.h"
0017 #include "spectrum.h"
0018 #include "spectrum_acl_tcam.h"
0019 
0020 /* gen_pool_alloc() returns 0 when allocation fails, so use an offset */
0021 #define MLXSW_SP_ACL_ERP_GENALLOC_OFFSET 0x100
0022 #define MLXSW_SP_ACL_ERP_MAX_PER_REGION 16
0023 
0024 struct mlxsw_sp_acl_erp_core {
0025     unsigned int erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_MAX + 1];
0026     struct gen_pool *erp_tables;
0027     struct mlxsw_sp *mlxsw_sp;
0028     struct mlxsw_sp_acl_bf *bf;
0029     unsigned int num_erp_banks;
0030 };
0031 
0032 struct mlxsw_sp_acl_erp_key {
0033     char mask[MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN];
0034 #define __MASK_LEN 0x38
0035 #define __MASK_IDX(i) (__MASK_LEN - (i) - 1)
0036     bool ctcam;
0037 };
0038 
0039 struct mlxsw_sp_acl_erp {
0040     struct mlxsw_sp_acl_erp_key key;
0041     u8 id;
0042     u8 index;
0043     DECLARE_BITMAP(mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN);
0044     struct list_head list;
0045     struct mlxsw_sp_acl_erp_table *erp_table;
0046 };
0047 
0048 struct mlxsw_sp_acl_erp_master_mask {
0049     DECLARE_BITMAP(bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN);
0050     unsigned int count[MLXSW_SP_ACL_TCAM_MASK_LEN];
0051 };
0052 
0053 struct mlxsw_sp_acl_erp_table {
0054     struct mlxsw_sp_acl_erp_master_mask master_mask;
0055     DECLARE_BITMAP(erp_id_bitmap, MLXSW_SP_ACL_ERP_MAX_PER_REGION);
0056     DECLARE_BITMAP(erp_index_bitmap, MLXSW_SP_ACL_ERP_MAX_PER_REGION);
0057     struct list_head atcam_erps_list;
0058     struct mlxsw_sp_acl_erp_core *erp_core;
0059     struct mlxsw_sp_acl_atcam_region *aregion;
0060     const struct mlxsw_sp_acl_erp_table_ops *ops;
0061     unsigned long base_index;
0062     unsigned int num_atcam_erps;
0063     unsigned int num_max_atcam_erps;
0064     unsigned int num_ctcam_erps;
0065     unsigned int num_deltas;
0066     struct objagg *objagg;
0067     struct mutex objagg_lock; /* guards objagg manipulation */
0068 };
0069 
0070 struct mlxsw_sp_acl_erp_table_ops {
0071     struct mlxsw_sp_acl_erp *
0072         (*erp_create)(struct mlxsw_sp_acl_erp_table *erp_table,
0073                   struct mlxsw_sp_acl_erp_key *key);
0074     void (*erp_destroy)(struct mlxsw_sp_acl_erp_table *erp_table,
0075                 struct mlxsw_sp_acl_erp *erp);
0076 };
0077 
0078 static struct mlxsw_sp_acl_erp *
0079 mlxsw_sp_acl_erp_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
0080                  struct mlxsw_sp_acl_erp_key *key);
0081 static void
0082 mlxsw_sp_acl_erp_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
0083                   struct mlxsw_sp_acl_erp *erp);
0084 static struct mlxsw_sp_acl_erp *
0085 mlxsw_sp_acl_erp_second_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
0086                     struct mlxsw_sp_acl_erp_key *key);
0087 static void
0088 mlxsw_sp_acl_erp_second_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
0089                      struct mlxsw_sp_acl_erp *erp);
0090 static struct mlxsw_sp_acl_erp *
0091 mlxsw_sp_acl_erp_first_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
0092                    struct mlxsw_sp_acl_erp_key *key);
0093 static void
0094 mlxsw_sp_acl_erp_first_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
0095                     struct mlxsw_sp_acl_erp *erp);
0096 static void
0097 mlxsw_sp_acl_erp_no_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
0098                  struct mlxsw_sp_acl_erp *erp);
0099 
0100 static const struct mlxsw_sp_acl_erp_table_ops erp_multiple_masks_ops = {
0101     .erp_create = mlxsw_sp_acl_erp_mask_create,
0102     .erp_destroy = mlxsw_sp_acl_erp_mask_destroy,
0103 };
0104 
0105 static const struct mlxsw_sp_acl_erp_table_ops erp_two_masks_ops = {
0106     .erp_create = mlxsw_sp_acl_erp_mask_create,
0107     .erp_destroy = mlxsw_sp_acl_erp_second_mask_destroy,
0108 };
0109 
0110 static const struct mlxsw_sp_acl_erp_table_ops erp_single_mask_ops = {
0111     .erp_create = mlxsw_sp_acl_erp_second_mask_create,
0112     .erp_destroy = mlxsw_sp_acl_erp_first_mask_destroy,
0113 };
0114 
0115 static const struct mlxsw_sp_acl_erp_table_ops erp_no_mask_ops = {
0116     .erp_create = mlxsw_sp_acl_erp_first_mask_create,
0117     .erp_destroy = mlxsw_sp_acl_erp_no_mask_destroy,
0118 };
0119 
0120 static bool
0121 mlxsw_sp_acl_erp_table_is_used(const struct mlxsw_sp_acl_erp_table *erp_table)
0122 {
0123     return erp_table->ops != &erp_single_mask_ops &&
0124            erp_table->ops != &erp_no_mask_ops;
0125 }
0126 
0127 static unsigned int
0128 mlxsw_sp_acl_erp_bank_get(const struct mlxsw_sp_acl_erp *erp)
0129 {
0130     return erp->index % erp->erp_table->erp_core->num_erp_banks;
0131 }
0132 
0133 static unsigned int
0134 mlxsw_sp_acl_erp_table_entry_size(const struct mlxsw_sp_acl_erp_table *erp_table)
0135 {
0136     struct mlxsw_sp_acl_atcam_region *aregion = erp_table->aregion;
0137     struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core;
0138 
0139     return erp_core->erpt_entries_size[aregion->type];
0140 }
0141 
0142 static int mlxsw_sp_acl_erp_id_get(struct mlxsw_sp_acl_erp_table *erp_table,
0143                    u8 *p_id)
0144 {
0145     u8 id;
0146 
0147     id = find_first_zero_bit(erp_table->erp_id_bitmap,
0148                  MLXSW_SP_ACL_ERP_MAX_PER_REGION);
0149     if (id < MLXSW_SP_ACL_ERP_MAX_PER_REGION) {
0150         __set_bit(id, erp_table->erp_id_bitmap);
0151         *p_id = id;
0152         return 0;
0153     }
0154 
0155     return -ENOBUFS;
0156 }
0157 
0158 static void mlxsw_sp_acl_erp_id_put(struct mlxsw_sp_acl_erp_table *erp_table,
0159                     u8 id)
0160 {
0161     __clear_bit(id, erp_table->erp_id_bitmap);
0162 }
0163 
0164 static void
0165 mlxsw_sp_acl_erp_master_mask_bit_set(unsigned long bit,
0166                      struct mlxsw_sp_acl_erp_master_mask *mask)
0167 {
0168     if (mask->count[bit]++ == 0)
0169         __set_bit(bit, mask->bitmap);
0170 }
0171 
0172 static void
0173 mlxsw_sp_acl_erp_master_mask_bit_clear(unsigned long bit,
0174                        struct mlxsw_sp_acl_erp_master_mask *mask)
0175 {
0176     if (--mask->count[bit] == 0)
0177         __clear_bit(bit, mask->bitmap);
0178 }
0179 
0180 static int
0181 mlxsw_sp_acl_erp_master_mask_update(struct mlxsw_sp_acl_erp_table *erp_table)
0182 {
0183     struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region;
0184     struct mlxsw_sp *mlxsw_sp = region->mlxsw_sp;
0185     char percr_pl[MLXSW_REG_PERCR_LEN];
0186     char *master_mask;
0187 
0188     mlxsw_reg_percr_pack(percr_pl, region->id);
0189     master_mask = mlxsw_reg_percr_master_mask_data(percr_pl);
0190     bitmap_to_arr32((u32 *) master_mask, erp_table->master_mask.bitmap,
0191             MLXSW_SP_ACL_TCAM_MASK_LEN);
0192 
0193     return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(percr), percr_pl);
0194 }
0195 
0196 static int
0197 mlxsw_sp_acl_erp_master_mask_set(struct mlxsw_sp_acl_erp_table *erp_table,
0198                  struct mlxsw_sp_acl_erp_key *key)
0199 {
0200     DECLARE_BITMAP(mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN);
0201     unsigned long bit;
0202     int err;
0203 
0204     bitmap_from_arr32(mask_bitmap, (u32 *) key->mask,
0205               MLXSW_SP_ACL_TCAM_MASK_LEN);
0206     for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN)
0207         mlxsw_sp_acl_erp_master_mask_bit_set(bit,
0208                              &erp_table->master_mask);
0209 
0210     err = mlxsw_sp_acl_erp_master_mask_update(erp_table);
0211     if (err)
0212         goto err_master_mask_update;
0213 
0214     return 0;
0215 
0216 err_master_mask_update:
0217     for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN)
0218         mlxsw_sp_acl_erp_master_mask_bit_clear(bit,
0219                                &erp_table->master_mask);
0220     return err;
0221 }
0222 
0223 static int
0224 mlxsw_sp_acl_erp_master_mask_clear(struct mlxsw_sp_acl_erp_table *erp_table,
0225                    struct mlxsw_sp_acl_erp_key *key)
0226 {
0227     DECLARE_BITMAP(mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN);
0228     unsigned long bit;
0229     int err;
0230 
0231     bitmap_from_arr32(mask_bitmap, (u32 *) key->mask,
0232               MLXSW_SP_ACL_TCAM_MASK_LEN);
0233     for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN)
0234         mlxsw_sp_acl_erp_master_mask_bit_clear(bit,
0235                                &erp_table->master_mask);
0236 
0237     err = mlxsw_sp_acl_erp_master_mask_update(erp_table);
0238     if (err)
0239         goto err_master_mask_update;
0240 
0241     return 0;
0242 
0243 err_master_mask_update:
0244     for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN)
0245         mlxsw_sp_acl_erp_master_mask_bit_set(bit,
0246                              &erp_table->master_mask);
0247     return err;
0248 }
0249 
0250 static struct mlxsw_sp_acl_erp *
0251 mlxsw_sp_acl_erp_generic_create(struct mlxsw_sp_acl_erp_table *erp_table,
0252                 struct mlxsw_sp_acl_erp_key *key)
0253 {
0254     struct mlxsw_sp_acl_erp *erp;
0255     int err;
0256 
0257     erp = kzalloc(sizeof(*erp), GFP_KERNEL);
0258     if (!erp)
0259         return ERR_PTR(-ENOMEM);
0260 
0261     err = mlxsw_sp_acl_erp_id_get(erp_table, &erp->id);
0262     if (err)
0263         goto err_erp_id_get;
0264 
0265     memcpy(&erp->key, key, sizeof(*key));
0266     list_add(&erp->list, &erp_table->atcam_erps_list);
0267     erp_table->num_atcam_erps++;
0268     erp->erp_table = erp_table;
0269 
0270     err = mlxsw_sp_acl_erp_master_mask_set(erp_table, &erp->key);
0271     if (err)
0272         goto err_master_mask_set;
0273 
0274     return erp;
0275 
0276 err_master_mask_set:
0277     erp_table->num_atcam_erps--;
0278     list_del(&erp->list);
0279     mlxsw_sp_acl_erp_id_put(erp_table, erp->id);
0280 err_erp_id_get:
0281     kfree(erp);
0282     return ERR_PTR(err);
0283 }
0284 
0285 static void
0286 mlxsw_sp_acl_erp_generic_destroy(struct mlxsw_sp_acl_erp *erp)
0287 {
0288     struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table;
0289 
0290     mlxsw_sp_acl_erp_master_mask_clear(erp_table, &erp->key);
0291     erp_table->num_atcam_erps--;
0292     list_del(&erp->list);
0293     mlxsw_sp_acl_erp_id_put(erp_table, erp->id);
0294     kfree(erp);
0295 }
0296 
0297 static int
0298 mlxsw_sp_acl_erp_table_alloc(struct mlxsw_sp_acl_erp_core *erp_core,
0299                  unsigned int num_erps,
0300                  enum mlxsw_sp_acl_atcam_region_type region_type,
0301                  unsigned long *p_index)
0302 {
0303     unsigned int num_rows, entry_size;
0304 
0305     /* We only allow allocations of entire rows */
0306     if (num_erps % erp_core->num_erp_banks != 0)
0307         return -EINVAL;
0308 
0309     entry_size = erp_core->erpt_entries_size[region_type];
0310     num_rows = num_erps / erp_core->num_erp_banks;
0311 
0312     *p_index = gen_pool_alloc(erp_core->erp_tables, num_rows * entry_size);
0313     if (*p_index == 0)
0314         return -ENOBUFS;
0315     *p_index -= MLXSW_SP_ACL_ERP_GENALLOC_OFFSET;
0316 
0317     return 0;
0318 }
0319 
0320 static void
0321 mlxsw_sp_acl_erp_table_free(struct mlxsw_sp_acl_erp_core *erp_core,
0322                 unsigned int num_erps,
0323                 enum mlxsw_sp_acl_atcam_region_type region_type,
0324                 unsigned long index)
0325 {
0326     unsigned long base_index;
0327     unsigned int entry_size;
0328     size_t size;
0329 
0330     entry_size = erp_core->erpt_entries_size[region_type];
0331     base_index = index + MLXSW_SP_ACL_ERP_GENALLOC_OFFSET;
0332     size = num_erps / erp_core->num_erp_banks * entry_size;
0333     gen_pool_free(erp_core->erp_tables, base_index, size);
0334 }
0335 
0336 static struct mlxsw_sp_acl_erp *
0337 mlxsw_sp_acl_erp_table_master_rp(struct mlxsw_sp_acl_erp_table *erp_table)
0338 {
0339     if (!list_is_singular(&erp_table->atcam_erps_list))
0340         return NULL;
0341 
0342     return list_first_entry(&erp_table->atcam_erps_list,
0343                 struct mlxsw_sp_acl_erp, list);
0344 }
0345 
0346 static int mlxsw_sp_acl_erp_index_get(struct mlxsw_sp_acl_erp_table *erp_table,
0347                       u8 *p_index)
0348 {
0349     u8 index;
0350 
0351     index = find_first_zero_bit(erp_table->erp_index_bitmap,
0352                     erp_table->num_max_atcam_erps);
0353     if (index < erp_table->num_max_atcam_erps) {
0354         __set_bit(index, erp_table->erp_index_bitmap);
0355         *p_index = index;
0356         return 0;
0357     }
0358 
0359     return -ENOBUFS;
0360 }
0361 
0362 static void mlxsw_sp_acl_erp_index_put(struct mlxsw_sp_acl_erp_table *erp_table,
0363                        u8 index)
0364 {
0365     __clear_bit(index, erp_table->erp_index_bitmap);
0366 }
0367 
0368 static void
0369 mlxsw_sp_acl_erp_table_locate(const struct mlxsw_sp_acl_erp_table *erp_table,
0370                   const struct mlxsw_sp_acl_erp *erp,
0371                   u8 *p_erpt_bank, u8 *p_erpt_index)
0372 {
0373     unsigned int entry_size = mlxsw_sp_acl_erp_table_entry_size(erp_table);
0374     struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core;
0375     unsigned int row;
0376 
0377     *p_erpt_bank = erp->index % erp_core->num_erp_banks;
0378     row = erp->index / erp_core->num_erp_banks;
0379     *p_erpt_index = erp_table->base_index + row * entry_size;
0380 }
0381 
0382 static int
0383 mlxsw_sp_acl_erp_table_erp_add(struct mlxsw_sp_acl_erp_table *erp_table,
0384                    struct mlxsw_sp_acl_erp *erp)
0385 {
0386     struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp;
0387     enum mlxsw_reg_perpt_key_size key_size;
0388     char perpt_pl[MLXSW_REG_PERPT_LEN];
0389     u8 erpt_bank, erpt_index;
0390 
0391     mlxsw_sp_acl_erp_table_locate(erp_table, erp, &erpt_bank, &erpt_index);
0392     key_size = (enum mlxsw_reg_perpt_key_size) erp_table->aregion->type;
0393     mlxsw_reg_perpt_pack(perpt_pl, erpt_bank, erpt_index, key_size, erp->id,
0394                  0, erp_table->base_index, erp->index,
0395                  erp->key.mask);
0396     mlxsw_reg_perpt_erp_vector_pack(perpt_pl, erp_table->erp_index_bitmap,
0397                     MLXSW_SP_ACL_ERP_MAX_PER_REGION);
0398     mlxsw_reg_perpt_erp_vector_set(perpt_pl, erp->index, true);
0399     return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(perpt), perpt_pl);
0400 }
0401 
0402 static void mlxsw_sp_acl_erp_table_erp_del(struct mlxsw_sp_acl_erp *erp)
0403 {
0404     char empty_mask[MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN] = { 0 };
0405     struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table;
0406     struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp;
0407     enum mlxsw_reg_perpt_key_size key_size;
0408     char perpt_pl[MLXSW_REG_PERPT_LEN];
0409     u8 erpt_bank, erpt_index;
0410 
0411     mlxsw_sp_acl_erp_table_locate(erp_table, erp, &erpt_bank, &erpt_index);
0412     key_size = (enum mlxsw_reg_perpt_key_size) erp_table->aregion->type;
0413     mlxsw_reg_perpt_pack(perpt_pl, erpt_bank, erpt_index, key_size, erp->id,
0414                  0, erp_table->base_index, erp->index, empty_mask);
0415     mlxsw_reg_perpt_erp_vector_pack(perpt_pl, erp_table->erp_index_bitmap,
0416                     MLXSW_SP_ACL_ERP_MAX_PER_REGION);
0417     mlxsw_reg_perpt_erp_vector_set(perpt_pl, erp->index, false);
0418     mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(perpt), perpt_pl);
0419 }
0420 
0421 static int
0422 mlxsw_sp_acl_erp_table_enable(struct mlxsw_sp_acl_erp_table *erp_table,
0423                   bool ctcam_le)
0424 {
0425     struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region;
0426     struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp;
0427     char pererp_pl[MLXSW_REG_PERERP_LEN];
0428 
0429     mlxsw_reg_pererp_pack(pererp_pl, region->id, ctcam_le, true, 0,
0430                   erp_table->base_index, 0);
0431     mlxsw_reg_pererp_erp_vector_pack(pererp_pl, erp_table->erp_index_bitmap,
0432                      MLXSW_SP_ACL_ERP_MAX_PER_REGION);
0433 
0434     return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl);
0435 }
0436 
0437 static void
0438 mlxsw_sp_acl_erp_table_disable(struct mlxsw_sp_acl_erp_table *erp_table)
0439 {
0440     struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region;
0441     struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp;
0442     char pererp_pl[MLXSW_REG_PERERP_LEN];
0443     struct mlxsw_sp_acl_erp *master_rp;
0444 
0445     master_rp = mlxsw_sp_acl_erp_table_master_rp(erp_table);
0446     /* It is possible we do not have a master RP when we disable the
0447      * table when there are no rules in the A-TCAM and the last C-TCAM
0448      * rule is deleted
0449      */
0450     mlxsw_reg_pererp_pack(pererp_pl, region->id, false, false, 0, 0,
0451                   master_rp ? master_rp->id : 0);
0452     mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl);
0453 }
0454 
0455 static int
0456 mlxsw_sp_acl_erp_table_relocate(struct mlxsw_sp_acl_erp_table *erp_table)
0457 {
0458     struct mlxsw_sp_acl_erp *erp;
0459     int err;
0460 
0461     list_for_each_entry(erp, &erp_table->atcam_erps_list, list) {
0462         err = mlxsw_sp_acl_erp_table_erp_add(erp_table, erp);
0463         if (err)
0464             goto err_table_erp_add;
0465     }
0466 
0467     return 0;
0468 
0469 err_table_erp_add:
0470     list_for_each_entry_continue_reverse(erp, &erp_table->atcam_erps_list,
0471                          list)
0472         mlxsw_sp_acl_erp_table_erp_del(erp);
0473     return err;
0474 }
0475 
0476 static int
0477 mlxsw_sp_acl_erp_table_expand(struct mlxsw_sp_acl_erp_table *erp_table)
0478 {
0479     unsigned int num_erps, old_num_erps = erp_table->num_max_atcam_erps;
0480     struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core;
0481     unsigned long old_base_index = erp_table->base_index;
0482     bool ctcam_le = erp_table->num_ctcam_erps > 0;
0483     int err;
0484 
0485     if (erp_table->num_atcam_erps < erp_table->num_max_atcam_erps)
0486         return 0;
0487 
0488     if (erp_table->num_max_atcam_erps == MLXSW_SP_ACL_ERP_MAX_PER_REGION)
0489         return -ENOBUFS;
0490 
0491     num_erps = old_num_erps + erp_core->num_erp_banks;
0492     err = mlxsw_sp_acl_erp_table_alloc(erp_core, num_erps,
0493                        erp_table->aregion->type,
0494                        &erp_table->base_index);
0495     if (err)
0496         return err;
0497     erp_table->num_max_atcam_erps = num_erps;
0498 
0499     err = mlxsw_sp_acl_erp_table_relocate(erp_table);
0500     if (err)
0501         goto err_table_relocate;
0502 
0503     err = mlxsw_sp_acl_erp_table_enable(erp_table, ctcam_le);
0504     if (err)
0505         goto err_table_enable;
0506 
0507     mlxsw_sp_acl_erp_table_free(erp_core, old_num_erps,
0508                     erp_table->aregion->type, old_base_index);
0509 
0510     return 0;
0511 
0512 err_table_enable:
0513 err_table_relocate:
0514     erp_table->num_max_atcam_erps = old_num_erps;
0515     mlxsw_sp_acl_erp_table_free(erp_core, num_erps,
0516                     erp_table->aregion->type,
0517                     erp_table->base_index);
0518     erp_table->base_index = old_base_index;
0519     return err;
0520 }
0521 
0522 static int
0523 mlxsw_acl_erp_table_bf_add(struct mlxsw_sp_acl_erp_table *erp_table,
0524                struct mlxsw_sp_acl_erp *erp)
0525 {
0526     struct mlxsw_sp_acl_atcam_region *aregion = erp_table->aregion;
0527     unsigned int erp_bank = mlxsw_sp_acl_erp_bank_get(erp);
0528     struct mlxsw_sp_acl_atcam_entry *aentry;
0529     int err;
0530 
0531     list_for_each_entry(aentry, &aregion->entries_list, list) {
0532         err = mlxsw_sp_acl_bf_entry_add(aregion->region->mlxsw_sp,
0533                         erp_table->erp_core->bf,
0534                         aregion, erp_bank, aentry);
0535         if (err)
0536             goto bf_entry_add_err;
0537     }
0538 
0539     return 0;
0540 
0541 bf_entry_add_err:
0542     list_for_each_entry_continue_reverse(aentry, &aregion->entries_list,
0543                          list)
0544         mlxsw_sp_acl_bf_entry_del(aregion->region->mlxsw_sp,
0545                       erp_table->erp_core->bf,
0546                       aregion, erp_bank, aentry);
0547     return err;
0548 }
0549 
0550 static void
0551 mlxsw_acl_erp_table_bf_del(struct mlxsw_sp_acl_erp_table *erp_table,
0552                struct mlxsw_sp_acl_erp *erp)
0553 {
0554     struct mlxsw_sp_acl_atcam_region *aregion = erp_table->aregion;
0555     unsigned int erp_bank = mlxsw_sp_acl_erp_bank_get(erp);
0556     struct mlxsw_sp_acl_atcam_entry *aentry;
0557 
0558     list_for_each_entry_reverse(aentry, &aregion->entries_list, list)
0559         mlxsw_sp_acl_bf_entry_del(aregion->region->mlxsw_sp,
0560                       erp_table->erp_core->bf,
0561                       aregion, erp_bank, aentry);
0562 }
0563 
0564 static int
0565 mlxsw_sp_acl_erp_region_table_trans(struct mlxsw_sp_acl_erp_table *erp_table)
0566 {
0567     struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core;
0568     struct mlxsw_sp_acl_erp *master_rp;
0569     int err;
0570 
0571     /* Initially, allocate a single eRP row. Expand later as needed */
0572     err = mlxsw_sp_acl_erp_table_alloc(erp_core, erp_core->num_erp_banks,
0573                        erp_table->aregion->type,
0574                        &erp_table->base_index);
0575     if (err)
0576         return err;
0577     erp_table->num_max_atcam_erps = erp_core->num_erp_banks;
0578 
0579     /* Transition the sole RP currently configured (the master RP)
0580      * to the eRP table
0581      */
0582     master_rp = mlxsw_sp_acl_erp_table_master_rp(erp_table);
0583     if (!master_rp) {
0584         err = -EINVAL;
0585         goto err_table_master_rp;
0586     }
0587 
0588     /* Make sure the master RP is using a valid index, as
0589      * only a single eRP row is currently allocated.
0590      */
0591     master_rp->index = 0;
0592     __set_bit(master_rp->index, erp_table->erp_index_bitmap);
0593 
0594     err = mlxsw_sp_acl_erp_table_erp_add(erp_table, master_rp);
0595     if (err)
0596         goto err_table_master_rp_add;
0597 
0598     /* Update Bloom filter before enabling eRP table, as rules
0599      * on the master RP were not set to Bloom filter up to this
0600      * point.
0601      */
0602     err = mlxsw_acl_erp_table_bf_add(erp_table, master_rp);
0603     if (err)
0604         goto err_table_bf_add;
0605 
0606     err = mlxsw_sp_acl_erp_table_enable(erp_table, false);
0607     if (err)
0608         goto err_table_enable;
0609 
0610     return 0;
0611 
0612 err_table_enable:
0613     mlxsw_acl_erp_table_bf_del(erp_table, master_rp);
0614 err_table_bf_add:
0615     mlxsw_sp_acl_erp_table_erp_del(master_rp);
0616 err_table_master_rp_add:
0617     __clear_bit(master_rp->index, erp_table->erp_index_bitmap);
0618 err_table_master_rp:
0619     mlxsw_sp_acl_erp_table_free(erp_core, erp_table->num_max_atcam_erps,
0620                     erp_table->aregion->type,
0621                     erp_table->base_index);
0622     return err;
0623 }
0624 
0625 static void
0626 mlxsw_sp_acl_erp_region_master_mask_trans(struct mlxsw_sp_acl_erp_table *erp_table)
0627 {
0628     struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core;
0629     struct mlxsw_sp_acl_erp *master_rp;
0630 
0631     mlxsw_sp_acl_erp_table_disable(erp_table);
0632     master_rp = mlxsw_sp_acl_erp_table_master_rp(erp_table);
0633     if (!master_rp)
0634         return;
0635     mlxsw_acl_erp_table_bf_del(erp_table, master_rp);
0636     mlxsw_sp_acl_erp_table_erp_del(master_rp);
0637     __clear_bit(master_rp->index, erp_table->erp_index_bitmap);
0638     mlxsw_sp_acl_erp_table_free(erp_core, erp_table->num_max_atcam_erps,
0639                     erp_table->aregion->type,
0640                     erp_table->base_index);
0641 }
0642 
0643 static int
0644 mlxsw_sp_acl_erp_region_erp_add(struct mlxsw_sp_acl_erp_table *erp_table,
0645                 struct mlxsw_sp_acl_erp *erp)
0646 {
0647     struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region;
0648     struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp;
0649     bool ctcam_le = erp_table->num_ctcam_erps > 0;
0650     char pererp_pl[MLXSW_REG_PERERP_LEN];
0651 
0652     mlxsw_reg_pererp_pack(pererp_pl, region->id, ctcam_le, true, 0,
0653                   erp_table->base_index, 0);
0654     mlxsw_reg_pererp_erp_vector_pack(pererp_pl, erp_table->erp_index_bitmap,
0655                      MLXSW_SP_ACL_ERP_MAX_PER_REGION);
0656     mlxsw_reg_pererp_erpt_vector_set(pererp_pl, erp->index, true);
0657 
0658     return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl);
0659 }
0660 
0661 static void mlxsw_sp_acl_erp_region_erp_del(struct mlxsw_sp_acl_erp *erp)
0662 {
0663     struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table;
0664     struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region;
0665     struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp;
0666     bool ctcam_le = erp_table->num_ctcam_erps > 0;
0667     char pererp_pl[MLXSW_REG_PERERP_LEN];
0668 
0669     mlxsw_reg_pererp_pack(pererp_pl, region->id, ctcam_le, true, 0,
0670                   erp_table->base_index, 0);
0671     mlxsw_reg_pererp_erp_vector_pack(pererp_pl, erp_table->erp_index_bitmap,
0672                      MLXSW_SP_ACL_ERP_MAX_PER_REGION);
0673     mlxsw_reg_pererp_erpt_vector_set(pererp_pl, erp->index, false);
0674 
0675     mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl);
0676 }
0677 
0678 static int
0679 mlxsw_sp_acl_erp_region_ctcam_enable(struct mlxsw_sp_acl_erp_table *erp_table)
0680 {
0681     /* No need to re-enable lookup in the C-TCAM */
0682     if (erp_table->num_ctcam_erps > 1)
0683         return 0;
0684 
0685     return mlxsw_sp_acl_erp_table_enable(erp_table, true);
0686 }
0687 
0688 static void
0689 mlxsw_sp_acl_erp_region_ctcam_disable(struct mlxsw_sp_acl_erp_table *erp_table)
0690 {
0691     /* Only disable C-TCAM lookup when last C-TCAM eRP is deleted */
0692     if (erp_table->num_ctcam_erps > 1)
0693         return;
0694 
0695     mlxsw_sp_acl_erp_table_enable(erp_table, false);
0696 }
0697 
0698 static int
0699 __mlxsw_sp_acl_erp_table_other_inc(struct mlxsw_sp_acl_erp_table *erp_table,
0700                    unsigned int *inc_num)
0701 {
0702     int err;
0703 
0704     /* If there are C-TCAM eRP or deltas in use we need to transition
0705      * the region to use eRP table, if it is not already done
0706      */
0707     if (!mlxsw_sp_acl_erp_table_is_used(erp_table)) {
0708         err = mlxsw_sp_acl_erp_region_table_trans(erp_table);
0709         if (err)
0710             return err;
0711     }
0712 
0713     /* When C-TCAM or deltas are used, the eRP table must be used */
0714     if (erp_table->ops != &erp_multiple_masks_ops)
0715         erp_table->ops = &erp_multiple_masks_ops;
0716 
0717     (*inc_num)++;
0718 
0719     return 0;
0720 }
0721 
0722 static int mlxsw_sp_acl_erp_ctcam_inc(struct mlxsw_sp_acl_erp_table *erp_table)
0723 {
0724     return __mlxsw_sp_acl_erp_table_other_inc(erp_table,
0725                           &erp_table->num_ctcam_erps);
0726 }
0727 
0728 static int mlxsw_sp_acl_erp_delta_inc(struct mlxsw_sp_acl_erp_table *erp_table)
0729 {
0730     return __mlxsw_sp_acl_erp_table_other_inc(erp_table,
0731                           &erp_table->num_deltas);
0732 }
0733 
0734 static void
0735 __mlxsw_sp_acl_erp_table_other_dec(struct mlxsw_sp_acl_erp_table *erp_table,
0736                    unsigned int *dec_num)
0737 {
0738     (*dec_num)--;
0739 
0740     /* If there are no C-TCAM eRP or deltas in use, the state we
0741      * transition to depends on the number of A-TCAM eRPs currently
0742      * in use.
0743      */
0744     if (erp_table->num_ctcam_erps > 0 || erp_table->num_deltas > 0)
0745         return;
0746 
0747     switch (erp_table->num_atcam_erps) {
0748     case 2:
0749         /* Keep using the eRP table, but correctly set the
0750          * operations pointer so that when an A-TCAM eRP is
0751          * deleted we will transition to use the master mask
0752          */
0753         erp_table->ops = &erp_two_masks_ops;
0754         break;
0755     case 1:
0756         /* We only kept the eRP table because we had C-TCAM
0757          * eRPs in use. Now that the last C-TCAM eRP is gone we
0758          * can stop using the table and transition to use the
0759          * master mask
0760          */
0761         mlxsw_sp_acl_erp_region_master_mask_trans(erp_table);
0762         erp_table->ops = &erp_single_mask_ops;
0763         break;
0764     case 0:
0765         /* There are no more eRPs of any kind used by the region
0766          * so free its eRP table and transition to initial state
0767          */
0768         mlxsw_sp_acl_erp_table_disable(erp_table);
0769         mlxsw_sp_acl_erp_table_free(erp_table->erp_core,
0770                         erp_table->num_max_atcam_erps,
0771                         erp_table->aregion->type,
0772                         erp_table->base_index);
0773         erp_table->ops = &erp_no_mask_ops;
0774         break;
0775     default:
0776         break;
0777     }
0778 }
0779 
0780 static void mlxsw_sp_acl_erp_ctcam_dec(struct mlxsw_sp_acl_erp_table *erp_table)
0781 {
0782     __mlxsw_sp_acl_erp_table_other_dec(erp_table,
0783                        &erp_table->num_ctcam_erps);
0784 }
0785 
0786 static void mlxsw_sp_acl_erp_delta_dec(struct mlxsw_sp_acl_erp_table *erp_table)
0787 {
0788     __mlxsw_sp_acl_erp_table_other_dec(erp_table,
0789                        &erp_table->num_deltas);
0790 }
0791 
0792 static struct mlxsw_sp_acl_erp *
0793 mlxsw_sp_acl_erp_ctcam_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
0794                    struct mlxsw_sp_acl_erp_key *key)
0795 {
0796     struct mlxsw_sp_acl_erp *erp;
0797     int err;
0798 
0799     erp = kzalloc(sizeof(*erp), GFP_KERNEL);
0800     if (!erp)
0801         return ERR_PTR(-ENOMEM);
0802 
0803     memcpy(&erp->key, key, sizeof(*key));
0804     bitmap_from_arr32(erp->mask_bitmap, (u32 *) key->mask,
0805               MLXSW_SP_ACL_TCAM_MASK_LEN);
0806 
0807     err = mlxsw_sp_acl_erp_ctcam_inc(erp_table);
0808     if (err)
0809         goto err_erp_ctcam_inc;
0810 
0811     erp->erp_table = erp_table;
0812 
0813     err = mlxsw_sp_acl_erp_master_mask_set(erp_table, &erp->key);
0814     if (err)
0815         goto err_master_mask_set;
0816 
0817     err = mlxsw_sp_acl_erp_region_ctcam_enable(erp_table);
0818     if (err)
0819         goto err_erp_region_ctcam_enable;
0820 
0821     return erp;
0822 
0823 err_erp_region_ctcam_enable:
0824     mlxsw_sp_acl_erp_master_mask_clear(erp_table, &erp->key);
0825 err_master_mask_set:
0826     mlxsw_sp_acl_erp_ctcam_dec(erp_table);
0827 err_erp_ctcam_inc:
0828     kfree(erp);
0829     return ERR_PTR(err);
0830 }
0831 
0832 static void
0833 mlxsw_sp_acl_erp_ctcam_mask_destroy(struct mlxsw_sp_acl_erp *erp)
0834 {
0835     struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table;
0836 
0837     mlxsw_sp_acl_erp_region_ctcam_disable(erp_table);
0838     mlxsw_sp_acl_erp_master_mask_clear(erp_table, &erp->key);
0839     mlxsw_sp_acl_erp_ctcam_dec(erp_table);
0840     kfree(erp);
0841 }
0842 
0843 static struct mlxsw_sp_acl_erp *
0844 mlxsw_sp_acl_erp_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
0845                  struct mlxsw_sp_acl_erp_key *key)
0846 {
0847     struct mlxsw_sp_acl_erp *erp;
0848     int err;
0849 
0850     if (key->ctcam)
0851         return mlxsw_sp_acl_erp_ctcam_mask_create(erp_table, key);
0852 
0853     /* Expand the eRP table for the new eRP, if needed */
0854     err = mlxsw_sp_acl_erp_table_expand(erp_table);
0855     if (err)
0856         return ERR_PTR(err);
0857 
0858     erp = mlxsw_sp_acl_erp_generic_create(erp_table, key);
0859     if (IS_ERR(erp))
0860         return erp;
0861 
0862     err = mlxsw_sp_acl_erp_index_get(erp_table, &erp->index);
0863     if (err)
0864         goto err_erp_index_get;
0865 
0866     err = mlxsw_sp_acl_erp_table_erp_add(erp_table, erp);
0867     if (err)
0868         goto err_table_erp_add;
0869 
0870     err = mlxsw_sp_acl_erp_region_erp_add(erp_table, erp);
0871     if (err)
0872         goto err_region_erp_add;
0873 
0874     erp_table->ops = &erp_multiple_masks_ops;
0875 
0876     return erp;
0877 
0878 err_region_erp_add:
0879     mlxsw_sp_acl_erp_table_erp_del(erp);
0880 err_table_erp_add:
0881     mlxsw_sp_acl_erp_index_put(erp_table, erp->index);
0882 err_erp_index_get:
0883     mlxsw_sp_acl_erp_generic_destroy(erp);
0884     return ERR_PTR(err);
0885 }
0886 
0887 static void
0888 mlxsw_sp_acl_erp_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
0889                   struct mlxsw_sp_acl_erp *erp)
0890 {
0891     if (erp->key.ctcam)
0892         return mlxsw_sp_acl_erp_ctcam_mask_destroy(erp);
0893 
0894     mlxsw_sp_acl_erp_region_erp_del(erp);
0895     mlxsw_sp_acl_erp_table_erp_del(erp);
0896     mlxsw_sp_acl_erp_index_put(erp_table, erp->index);
0897     mlxsw_sp_acl_erp_generic_destroy(erp);
0898 
0899     if (erp_table->num_atcam_erps == 2 && erp_table->num_ctcam_erps == 0 &&
0900         erp_table->num_deltas == 0)
0901         erp_table->ops = &erp_two_masks_ops;
0902 }
0903 
0904 static struct mlxsw_sp_acl_erp *
0905 mlxsw_sp_acl_erp_second_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
0906                     struct mlxsw_sp_acl_erp_key *key)
0907 {
0908     struct mlxsw_sp_acl_erp *erp;
0909     int err;
0910 
0911     if (key->ctcam)
0912         return mlxsw_sp_acl_erp_ctcam_mask_create(erp_table, key);
0913 
0914     /* Transition to use eRP table instead of master mask */
0915     err = mlxsw_sp_acl_erp_region_table_trans(erp_table);
0916     if (err)
0917         return ERR_PTR(err);
0918 
0919     erp = mlxsw_sp_acl_erp_generic_create(erp_table, key);
0920     if (IS_ERR(erp)) {
0921         err = PTR_ERR(erp);
0922         goto err_erp_create;
0923     }
0924 
0925     err = mlxsw_sp_acl_erp_index_get(erp_table, &erp->index);
0926     if (err)
0927         goto err_erp_index_get;
0928 
0929     err = mlxsw_sp_acl_erp_table_erp_add(erp_table, erp);
0930     if (err)
0931         goto err_table_erp_add;
0932 
0933     err = mlxsw_sp_acl_erp_region_erp_add(erp_table, erp);
0934     if (err)
0935         goto err_region_erp_add;
0936 
0937     erp_table->ops = &erp_two_masks_ops;
0938 
0939     return erp;
0940 
0941 err_region_erp_add:
0942     mlxsw_sp_acl_erp_table_erp_del(erp);
0943 err_table_erp_add:
0944     mlxsw_sp_acl_erp_index_put(erp_table, erp->index);
0945 err_erp_index_get:
0946     mlxsw_sp_acl_erp_generic_destroy(erp);
0947 err_erp_create:
0948     mlxsw_sp_acl_erp_region_master_mask_trans(erp_table);
0949     return ERR_PTR(err);
0950 }
0951 
0952 static void
0953 mlxsw_sp_acl_erp_second_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
0954                      struct mlxsw_sp_acl_erp *erp)
0955 {
0956     if (erp->key.ctcam)
0957         return mlxsw_sp_acl_erp_ctcam_mask_destroy(erp);
0958 
0959     mlxsw_sp_acl_erp_region_erp_del(erp);
0960     mlxsw_sp_acl_erp_table_erp_del(erp);
0961     mlxsw_sp_acl_erp_index_put(erp_table, erp->index);
0962     mlxsw_sp_acl_erp_generic_destroy(erp);
0963     /* Transition to use master mask instead of eRP table */
0964     mlxsw_sp_acl_erp_region_master_mask_trans(erp_table);
0965 
0966     erp_table->ops = &erp_single_mask_ops;
0967 }
0968 
0969 static struct mlxsw_sp_acl_erp *
0970 mlxsw_sp_acl_erp_first_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
0971                    struct mlxsw_sp_acl_erp_key *key)
0972 {
0973     struct mlxsw_sp_acl_erp *erp;
0974 
0975     if (key->ctcam)
0976         return ERR_PTR(-EINVAL);
0977 
0978     erp = mlxsw_sp_acl_erp_generic_create(erp_table, key);
0979     if (IS_ERR(erp))
0980         return erp;
0981 
0982     erp_table->ops = &erp_single_mask_ops;
0983 
0984     return erp;
0985 }
0986 
0987 static void
0988 mlxsw_sp_acl_erp_first_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
0989                     struct mlxsw_sp_acl_erp *erp)
0990 {
0991     mlxsw_sp_acl_erp_generic_destroy(erp);
0992     erp_table->ops = &erp_no_mask_ops;
0993 }
0994 
0995 static void
0996 mlxsw_sp_acl_erp_no_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
0997                  struct mlxsw_sp_acl_erp *erp)
0998 {
0999     WARN_ON(1);
1000 }
1001 
1002 struct mlxsw_sp_acl_erp_mask *
1003 mlxsw_sp_acl_erp_mask_get(struct mlxsw_sp_acl_atcam_region *aregion,
1004               const char *mask, bool ctcam)
1005 {
1006     struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1007     struct mlxsw_sp_acl_erp_key key;
1008     struct objagg_obj *objagg_obj;
1009 
1010     memcpy(key.mask, mask, MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN);
1011     key.ctcam = ctcam;
1012     mutex_lock(&erp_table->objagg_lock);
1013     objagg_obj = objagg_obj_get(erp_table->objagg, &key);
1014     mutex_unlock(&erp_table->objagg_lock);
1015     if (IS_ERR(objagg_obj))
1016         return ERR_CAST(objagg_obj);
1017     return (struct mlxsw_sp_acl_erp_mask *) objagg_obj;
1018 }
1019 
1020 void mlxsw_sp_acl_erp_mask_put(struct mlxsw_sp_acl_atcam_region *aregion,
1021                    struct mlxsw_sp_acl_erp_mask *erp_mask)
1022 {
1023     struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask;
1024     struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1025 
1026     mutex_lock(&erp_table->objagg_lock);
1027     objagg_obj_put(erp_table->objagg, objagg_obj);
1028     mutex_unlock(&erp_table->objagg_lock);
1029 }
1030 
1031 int mlxsw_sp_acl_erp_bf_insert(struct mlxsw_sp *mlxsw_sp,
1032                    struct mlxsw_sp_acl_atcam_region *aregion,
1033                    struct mlxsw_sp_acl_erp_mask *erp_mask,
1034                    struct mlxsw_sp_acl_atcam_entry *aentry)
1035 {
1036     struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask;
1037     const struct mlxsw_sp_acl_erp *erp = objagg_obj_root_priv(objagg_obj);
1038     unsigned int erp_bank;
1039 
1040     if (!mlxsw_sp_acl_erp_table_is_used(erp->erp_table))
1041         return 0;
1042 
1043     erp_bank = mlxsw_sp_acl_erp_bank_get(erp);
1044     return mlxsw_sp_acl_bf_entry_add(mlxsw_sp,
1045                     erp->erp_table->erp_core->bf,
1046                     aregion, erp_bank, aentry);
1047 }
1048 
1049 void mlxsw_sp_acl_erp_bf_remove(struct mlxsw_sp *mlxsw_sp,
1050                 struct mlxsw_sp_acl_atcam_region *aregion,
1051                 struct mlxsw_sp_acl_erp_mask *erp_mask,
1052                 struct mlxsw_sp_acl_atcam_entry *aentry)
1053 {
1054     struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask;
1055     const struct mlxsw_sp_acl_erp *erp = objagg_obj_root_priv(objagg_obj);
1056     unsigned int erp_bank;
1057 
1058     if (!mlxsw_sp_acl_erp_table_is_used(erp->erp_table))
1059         return;
1060 
1061     erp_bank = mlxsw_sp_acl_erp_bank_get(erp);
1062     mlxsw_sp_acl_bf_entry_del(mlxsw_sp,
1063                   erp->erp_table->erp_core->bf,
1064                   aregion, erp_bank, aentry);
1065 }
1066 
1067 bool
1068 mlxsw_sp_acl_erp_mask_is_ctcam(const struct mlxsw_sp_acl_erp_mask *erp_mask)
1069 {
1070     struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask;
1071     const struct mlxsw_sp_acl_erp_key *key = objagg_obj_raw(objagg_obj);
1072 
1073     return key->ctcam;
1074 }
1075 
1076 u8 mlxsw_sp_acl_erp_mask_erp_id(const struct mlxsw_sp_acl_erp_mask *erp_mask)
1077 {
1078     struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask;
1079     const struct mlxsw_sp_acl_erp *erp = objagg_obj_root_priv(objagg_obj);
1080 
1081     return erp->id;
1082 }
1083 
1084 struct mlxsw_sp_acl_erp_delta {
1085     struct mlxsw_sp_acl_erp_key key;
1086     u16 start;
1087     u8 mask;
1088 };
1089 
1090 u16 mlxsw_sp_acl_erp_delta_start(const struct mlxsw_sp_acl_erp_delta *delta)
1091 {
1092     return delta->start;
1093 }
1094 
1095 u8 mlxsw_sp_acl_erp_delta_mask(const struct mlxsw_sp_acl_erp_delta *delta)
1096 {
1097     return delta->mask;
1098 }
1099 
1100 u8 mlxsw_sp_acl_erp_delta_value(const struct mlxsw_sp_acl_erp_delta *delta,
1101                 const char *enc_key)
1102 {
1103     u16 start = delta->start;
1104     u8 mask = delta->mask;
1105     u16 tmp;
1106 
1107     if (!mask)
1108         return 0;
1109 
1110     tmp = (unsigned char) enc_key[__MASK_IDX(start / 8)];
1111     if (start / 8 + 1 < __MASK_LEN)
1112         tmp |= (unsigned char) enc_key[__MASK_IDX(start / 8 + 1)] << 8;
1113     tmp >>= start % 8;
1114     tmp &= mask;
1115     return tmp;
1116 }
1117 
1118 void mlxsw_sp_acl_erp_delta_clear(const struct mlxsw_sp_acl_erp_delta *delta,
1119                   const char *enc_key)
1120 {
1121     u16 start = delta->start;
1122     u8 mask = delta->mask;
1123     unsigned char *byte;
1124     u16 tmp;
1125 
1126     tmp = mask;
1127     tmp <<= start % 8;
1128     tmp = ~tmp;
1129 
1130     byte = (unsigned char *) &enc_key[__MASK_IDX(start / 8)];
1131     *byte &= tmp & 0xff;
1132     if (start / 8 + 1 < __MASK_LEN) {
1133         byte = (unsigned char *) &enc_key[__MASK_IDX(start / 8 + 1)];
1134         *byte &= (tmp >> 8) & 0xff;
1135     }
1136 }
1137 
1138 static const struct mlxsw_sp_acl_erp_delta
1139 mlxsw_sp_acl_erp_delta_default = {};
1140 
1141 const struct mlxsw_sp_acl_erp_delta *
1142 mlxsw_sp_acl_erp_delta(const struct mlxsw_sp_acl_erp_mask *erp_mask)
1143 {
1144     struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask;
1145     const struct mlxsw_sp_acl_erp_delta *delta;
1146 
1147     delta = objagg_obj_delta_priv(objagg_obj);
1148     if (!delta)
1149         delta = &mlxsw_sp_acl_erp_delta_default;
1150     return delta;
1151 }
1152 
1153 static int
1154 mlxsw_sp_acl_erp_delta_fill(const struct mlxsw_sp_acl_erp_key *parent_key,
1155                 const struct mlxsw_sp_acl_erp_key *key,
1156                 u16 *delta_start, u8 *delta_mask)
1157 {
1158     int offset = 0;
1159     int si = -1;
1160     u16 pmask;
1161     u16 mask;
1162     int i;
1163 
1164     /* The difference between 2 masks can be up to 8 consecutive bits. */
1165     for (i = 0; i < __MASK_LEN; i++) {
1166         if (parent_key->mask[__MASK_IDX(i)] == key->mask[__MASK_IDX(i)])
1167             continue;
1168         if (si == -1)
1169             si = i;
1170         else if (si != i - 1)
1171             return -EINVAL;
1172     }
1173     if (si == -1) {
1174         /* The masks are the same, this can happen in case eRPs with
1175          * the same mask were created in both A-TCAM and C-TCAM.
1176          * The only possible condition under which this can happen
1177          * is identical rule insertion. Delta is not possible here.
1178          */
1179         return -EINVAL;
1180     }
1181     pmask = (unsigned char) parent_key->mask[__MASK_IDX(si)];
1182     mask = (unsigned char) key->mask[__MASK_IDX(si)];
1183     if (si + 1 < __MASK_LEN) {
1184         pmask |= (unsigned char) parent_key->mask[__MASK_IDX(si + 1)] << 8;
1185         mask |= (unsigned char) key->mask[__MASK_IDX(si + 1)] << 8;
1186     }
1187 
1188     if ((pmask ^ mask) & pmask)
1189         return -EINVAL;
1190     mask &= ~pmask;
1191     while (!(mask & (1 << offset)))
1192         offset++;
1193     while (!(mask & 1))
1194         mask >>= 1;
1195     if (mask & 0xff00)
1196         return -EINVAL;
1197 
1198     *delta_start = si * 8 + offset;
1199     *delta_mask = mask;
1200 
1201     return 0;
1202 }
1203 
1204 static bool mlxsw_sp_acl_erp_delta_check(void *priv, const void *parent_obj,
1205                      const void *obj)
1206 {
1207     const struct mlxsw_sp_acl_erp_key *parent_key = parent_obj;
1208     const struct mlxsw_sp_acl_erp_key *key = obj;
1209     u16 delta_start;
1210     u8 delta_mask;
1211     int err;
1212 
1213     err = mlxsw_sp_acl_erp_delta_fill(parent_key, key,
1214                       &delta_start, &delta_mask);
1215     return err ? false : true;
1216 }
1217 
1218 static int mlxsw_sp_acl_erp_hints_obj_cmp(const void *obj1, const void *obj2)
1219 {
1220     const struct mlxsw_sp_acl_erp_key *key1 = obj1;
1221     const struct mlxsw_sp_acl_erp_key *key2 = obj2;
1222 
1223     /* For hints purposes, two objects are considered equal
1224      * in case the masks are the same. Does not matter what
1225      * the "ctcam" value is.
1226      */
1227     return memcmp(key1->mask, key2->mask, sizeof(key1->mask));
1228 }
1229 
1230 static void *mlxsw_sp_acl_erp_delta_create(void *priv, void *parent_obj,
1231                        void *obj)
1232 {
1233     struct mlxsw_sp_acl_erp_key *parent_key = parent_obj;
1234     struct mlxsw_sp_acl_atcam_region *aregion = priv;
1235     struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1236     struct mlxsw_sp_acl_erp_key *key = obj;
1237     struct mlxsw_sp_acl_erp_delta *delta;
1238     u16 delta_start;
1239     u8 delta_mask;
1240     int err;
1241 
1242     if (parent_key->ctcam || key->ctcam)
1243         return ERR_PTR(-EINVAL);
1244     err = mlxsw_sp_acl_erp_delta_fill(parent_key, key,
1245                       &delta_start, &delta_mask);
1246     if (err)
1247         return ERR_PTR(-EINVAL);
1248 
1249     delta = kzalloc(sizeof(*delta), GFP_KERNEL);
1250     if (!delta)
1251         return ERR_PTR(-ENOMEM);
1252     delta->start = delta_start;
1253     delta->mask = delta_mask;
1254 
1255     err = mlxsw_sp_acl_erp_delta_inc(erp_table);
1256     if (err)
1257         goto err_erp_delta_inc;
1258 
1259     memcpy(&delta->key, key, sizeof(*key));
1260     err = mlxsw_sp_acl_erp_master_mask_set(erp_table, &delta->key);
1261     if (err)
1262         goto err_master_mask_set;
1263 
1264     return delta;
1265 
1266 err_master_mask_set:
1267     mlxsw_sp_acl_erp_delta_dec(erp_table);
1268 err_erp_delta_inc:
1269     kfree(delta);
1270     return ERR_PTR(err);
1271 }
1272 
1273 static void mlxsw_sp_acl_erp_delta_destroy(void *priv, void *delta_priv)
1274 {
1275     struct mlxsw_sp_acl_erp_delta *delta = delta_priv;
1276     struct mlxsw_sp_acl_atcam_region *aregion = priv;
1277     struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1278 
1279     mlxsw_sp_acl_erp_master_mask_clear(erp_table, &delta->key);
1280     mlxsw_sp_acl_erp_delta_dec(erp_table);
1281     kfree(delta);
1282 }
1283 
1284 static void *mlxsw_sp_acl_erp_root_create(void *priv, void *obj,
1285                       unsigned int root_id)
1286 {
1287     struct mlxsw_sp_acl_atcam_region *aregion = priv;
1288     struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1289     struct mlxsw_sp_acl_erp_key *key = obj;
1290 
1291     if (!key->ctcam &&
1292         root_id != OBJAGG_OBJ_ROOT_ID_INVALID &&
1293         root_id >= MLXSW_SP_ACL_ERP_MAX_PER_REGION)
1294         return ERR_PTR(-ENOBUFS);
1295     return erp_table->ops->erp_create(erp_table, key);
1296 }
1297 
1298 static void mlxsw_sp_acl_erp_root_destroy(void *priv, void *root_priv)
1299 {
1300     struct mlxsw_sp_acl_atcam_region *aregion = priv;
1301     struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1302 
1303     erp_table->ops->erp_destroy(erp_table, root_priv);
1304 }
1305 
1306 static const struct objagg_ops mlxsw_sp_acl_erp_objagg_ops = {
1307     .obj_size = sizeof(struct mlxsw_sp_acl_erp_key),
1308     .delta_check = mlxsw_sp_acl_erp_delta_check,
1309     .hints_obj_cmp = mlxsw_sp_acl_erp_hints_obj_cmp,
1310     .delta_create = mlxsw_sp_acl_erp_delta_create,
1311     .delta_destroy = mlxsw_sp_acl_erp_delta_destroy,
1312     .root_create = mlxsw_sp_acl_erp_root_create,
1313     .root_destroy = mlxsw_sp_acl_erp_root_destroy,
1314 };
1315 
1316 static struct mlxsw_sp_acl_erp_table *
1317 mlxsw_sp_acl_erp_table_create(struct mlxsw_sp_acl_atcam_region *aregion,
1318                   struct objagg_hints *hints)
1319 {
1320     struct mlxsw_sp_acl_erp_table *erp_table;
1321     int err;
1322 
1323     erp_table = kzalloc(sizeof(*erp_table), GFP_KERNEL);
1324     if (!erp_table)
1325         return ERR_PTR(-ENOMEM);
1326 
1327     erp_table->objagg = objagg_create(&mlxsw_sp_acl_erp_objagg_ops,
1328                       hints, aregion);
1329     if (IS_ERR(erp_table->objagg)) {
1330         err = PTR_ERR(erp_table->objagg);
1331         goto err_objagg_create;
1332     }
1333 
1334     erp_table->erp_core = aregion->atcam->erp_core;
1335     erp_table->ops = &erp_no_mask_ops;
1336     INIT_LIST_HEAD(&erp_table->atcam_erps_list);
1337     erp_table->aregion = aregion;
1338     mutex_init(&erp_table->objagg_lock);
1339 
1340     return erp_table;
1341 
1342 err_objagg_create:
1343     kfree(erp_table);
1344     return ERR_PTR(err);
1345 }
1346 
1347 static void
1348 mlxsw_sp_acl_erp_table_destroy(struct mlxsw_sp_acl_erp_table *erp_table)
1349 {
1350     WARN_ON(!list_empty(&erp_table->atcam_erps_list));
1351     mutex_destroy(&erp_table->objagg_lock);
1352     objagg_destroy(erp_table->objagg);
1353     kfree(erp_table);
1354 }
1355 
1356 static int
1357 mlxsw_sp_acl_erp_master_mask_init(struct mlxsw_sp_acl_atcam_region *aregion)
1358 {
1359     struct mlxsw_sp *mlxsw_sp = aregion->region->mlxsw_sp;
1360     char percr_pl[MLXSW_REG_PERCR_LEN];
1361 
1362     mlxsw_reg_percr_pack(percr_pl, aregion->region->id);
1363     return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(percr), percr_pl);
1364 }
1365 
1366 static int
1367 mlxsw_sp_acl_erp_region_param_init(struct mlxsw_sp_acl_atcam_region *aregion)
1368 {
1369     struct mlxsw_sp *mlxsw_sp = aregion->region->mlxsw_sp;
1370     char pererp_pl[MLXSW_REG_PERERP_LEN];
1371 
1372     mlxsw_reg_pererp_pack(pererp_pl, aregion->region->id, false, false, 0,
1373                   0, 0);
1374     return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl);
1375 }
1376 
1377 static int
1378 mlxsw_sp_acl_erp_hints_check(struct mlxsw_sp *mlxsw_sp,
1379                  struct mlxsw_sp_acl_atcam_region *aregion,
1380                  struct objagg_hints *hints, bool *p_rehash_needed)
1381 {
1382     struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1383     const struct objagg_stats *ostats;
1384     const struct objagg_stats *hstats;
1385     int err;
1386 
1387     *p_rehash_needed = false;
1388 
1389     mutex_lock(&erp_table->objagg_lock);
1390     ostats = objagg_stats_get(erp_table->objagg);
1391     mutex_unlock(&erp_table->objagg_lock);
1392     if (IS_ERR(ostats)) {
1393         dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to get ERP stats\n");
1394         return PTR_ERR(ostats);
1395     }
1396 
1397     hstats = objagg_hints_stats_get(hints);
1398     if (IS_ERR(hstats)) {
1399         dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to get ERP hints stats\n");
1400         err = PTR_ERR(hstats);
1401         goto err_hints_stats_get;
1402     }
1403 
1404     /* Very basic criterion for now. */
1405     if (hstats->root_count < ostats->root_count)
1406         *p_rehash_needed = true;
1407 
1408     err = 0;
1409 
1410     objagg_stats_put(hstats);
1411 err_hints_stats_get:
1412     objagg_stats_put(ostats);
1413     return err;
1414 }
1415 
1416 void *
1417 mlxsw_sp_acl_erp_rehash_hints_get(struct mlxsw_sp_acl_atcam_region *aregion)
1418 {
1419     struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1420     struct mlxsw_sp *mlxsw_sp = aregion->region->mlxsw_sp;
1421     struct objagg_hints *hints;
1422     bool rehash_needed;
1423     int err;
1424 
1425     mutex_lock(&erp_table->objagg_lock);
1426     hints = objagg_hints_get(erp_table->objagg,
1427                  OBJAGG_OPT_ALGO_SIMPLE_GREEDY);
1428     mutex_unlock(&erp_table->objagg_lock);
1429     if (IS_ERR(hints)) {
1430         dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to create ERP hints\n");
1431         return ERR_CAST(hints);
1432     }
1433     err = mlxsw_sp_acl_erp_hints_check(mlxsw_sp, aregion, hints,
1434                        &rehash_needed);
1435     if (err)
1436         goto errout;
1437 
1438     if (!rehash_needed) {
1439         err = -EAGAIN;
1440         goto errout;
1441     }
1442     return hints;
1443 
1444 errout:
1445     objagg_hints_put(hints);
1446     return ERR_PTR(err);
1447 }
1448 
1449 void mlxsw_sp_acl_erp_rehash_hints_put(void *hints_priv)
1450 {
1451     struct objagg_hints *hints = hints_priv;
1452 
1453     objagg_hints_put(hints);
1454 }
1455 
1456 int mlxsw_sp_acl_erp_region_init(struct mlxsw_sp_acl_atcam_region *aregion,
1457                  void *hints_priv)
1458 {
1459     struct mlxsw_sp_acl_erp_table *erp_table;
1460     struct objagg_hints *hints = hints_priv;
1461     int err;
1462 
1463     erp_table = mlxsw_sp_acl_erp_table_create(aregion, hints);
1464     if (IS_ERR(erp_table))
1465         return PTR_ERR(erp_table);
1466     aregion->erp_table = erp_table;
1467 
1468     /* Initialize the region's master mask to all zeroes */
1469     err = mlxsw_sp_acl_erp_master_mask_init(aregion);
1470     if (err)
1471         goto err_erp_master_mask_init;
1472 
1473     /* Initialize the region to not use the eRP table */
1474     err = mlxsw_sp_acl_erp_region_param_init(aregion);
1475     if (err)
1476         goto err_erp_region_param_init;
1477 
1478     return 0;
1479 
1480 err_erp_region_param_init:
1481 err_erp_master_mask_init:
1482     mlxsw_sp_acl_erp_table_destroy(erp_table);
1483     return err;
1484 }
1485 
1486 void mlxsw_sp_acl_erp_region_fini(struct mlxsw_sp_acl_atcam_region *aregion)
1487 {
1488     mlxsw_sp_acl_erp_table_destroy(aregion->erp_table);
1489 }
1490 
1491 static int
1492 mlxsw_sp_acl_erp_tables_sizes_query(struct mlxsw_sp *mlxsw_sp,
1493                     struct mlxsw_sp_acl_erp_core *erp_core)
1494 {
1495     unsigned int size;
1496 
1497     if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_2KB) ||
1498         !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_4KB) ||
1499         !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_8KB) ||
1500         !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_12KB))
1501         return -EIO;
1502 
1503     size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_2KB);
1504     erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_2KB] = size;
1505 
1506     size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_4KB);
1507     erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_4KB] = size;
1508 
1509     size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_8KB);
1510     erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_8KB] = size;
1511 
1512     size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_12KB);
1513     erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_12KB] = size;
1514 
1515     return 0;
1516 }
1517 
1518 static int mlxsw_sp_acl_erp_tables_init(struct mlxsw_sp *mlxsw_sp,
1519                     struct mlxsw_sp_acl_erp_core *erp_core)
1520 {
1521     unsigned int erpt_bank_size;
1522     int err;
1523 
1524     if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_MAX_ERPT_BANK_SIZE) ||
1525         !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_MAX_ERPT_BANKS))
1526         return -EIO;
1527     erpt_bank_size = MLXSW_CORE_RES_GET(mlxsw_sp->core,
1528                         ACL_MAX_ERPT_BANK_SIZE);
1529     erp_core->num_erp_banks = MLXSW_CORE_RES_GET(mlxsw_sp->core,
1530                              ACL_MAX_ERPT_BANKS);
1531 
1532     erp_core->erp_tables = gen_pool_create(0, -1);
1533     if (!erp_core->erp_tables)
1534         return -ENOMEM;
1535     gen_pool_set_algo(erp_core->erp_tables, gen_pool_best_fit, NULL);
1536 
1537     err = gen_pool_add(erp_core->erp_tables,
1538                MLXSW_SP_ACL_ERP_GENALLOC_OFFSET, erpt_bank_size,
1539                -1);
1540     if (err)
1541         goto err_gen_pool_add;
1542 
1543     erp_core->bf = mlxsw_sp_acl_bf_init(mlxsw_sp, erp_core->num_erp_banks);
1544     if (IS_ERR(erp_core->bf)) {
1545         err = PTR_ERR(erp_core->bf);
1546         goto err_bf_init;
1547     }
1548 
1549     /* Different regions require masks of different sizes */
1550     err = mlxsw_sp_acl_erp_tables_sizes_query(mlxsw_sp, erp_core);
1551     if (err)
1552         goto err_erp_tables_sizes_query;
1553 
1554     return 0;
1555 
1556 err_erp_tables_sizes_query:
1557     mlxsw_sp_acl_bf_fini(erp_core->bf);
1558 err_bf_init:
1559 err_gen_pool_add:
1560     gen_pool_destroy(erp_core->erp_tables);
1561     return err;
1562 }
1563 
1564 static void mlxsw_sp_acl_erp_tables_fini(struct mlxsw_sp *mlxsw_sp,
1565                      struct mlxsw_sp_acl_erp_core *erp_core)
1566 {
1567     mlxsw_sp_acl_bf_fini(erp_core->bf);
1568     gen_pool_destroy(erp_core->erp_tables);
1569 }
1570 
1571 int mlxsw_sp_acl_erps_init(struct mlxsw_sp *mlxsw_sp,
1572                struct mlxsw_sp_acl_atcam *atcam)
1573 {
1574     struct mlxsw_sp_acl_erp_core *erp_core;
1575     int err;
1576 
1577     erp_core = kzalloc(sizeof(*erp_core), GFP_KERNEL);
1578     if (!erp_core)
1579         return -ENOMEM;
1580     erp_core->mlxsw_sp = mlxsw_sp;
1581     atcam->erp_core = erp_core;
1582 
1583     err = mlxsw_sp_acl_erp_tables_init(mlxsw_sp, erp_core);
1584     if (err)
1585         goto err_erp_tables_init;
1586 
1587     return 0;
1588 
1589 err_erp_tables_init:
1590     kfree(erp_core);
1591     return err;
1592 }
1593 
1594 void mlxsw_sp_acl_erps_fini(struct mlxsw_sp *mlxsw_sp,
1595                 struct mlxsw_sp_acl_atcam *atcam)
1596 {
1597     mlxsw_sp_acl_erp_tables_fini(mlxsw_sp, atcam->erp_core);
1598     kfree(atcam->erp_core);
1599 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!