sw/rxe/rxe_pool.c

0001 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
0002 /*
0003  * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
0004  * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
0005  */
0006
0007 #include "rxe.h"
0008
0009 #define RXE_POOL_TIMEOUT    (200)
0010 #define RXE_POOL_ALIGN      (16)
0011
0012 static const struct rxe_type_info {
0013     const char *name;
0014     size_t size;
0015     size_t elem_offset;
0016     void (*cleanup)(struct rxe_pool_elem *elem);
0017     u32 min_index;
0018     u32 max_index;
0019     u32 max_elem;
0020 } rxe_type_info[RXE_NUM_TYPES] = {
0021     [RXE_TYPE_UC] = {
0022         .name       = "uc",
0023         .size       = sizeof(struct rxe_ucontext),
0024         .elem_offset    = offsetof(struct rxe_ucontext, elem),
0025         .min_index  = 1,
0026         .max_index  = UINT_MAX,
0027         .max_elem   = UINT_MAX,
0028     },
0029     [RXE_TYPE_PD] = {
0030         .name       = "pd",
0031         .size       = sizeof(struct rxe_pd),
0032         .elem_offset    = offsetof(struct rxe_pd, elem),
0033         .min_index  = 1,
0034         .max_index  = UINT_MAX,
0035         .max_elem   = UINT_MAX,
0036     },
0037     [RXE_TYPE_AH] = {
0038         .name       = "ah",
0039         .size       = sizeof(struct rxe_ah),
0040         .elem_offset    = offsetof(struct rxe_ah, elem),
0041         .min_index  = RXE_MIN_AH_INDEX,
0042         .max_index  = RXE_MAX_AH_INDEX,
0043         .max_elem   = RXE_MAX_AH_INDEX - RXE_MIN_AH_INDEX + 1,
0044     },
0045     [RXE_TYPE_SRQ] = {
0046         .name       = "srq",
0047         .size       = sizeof(struct rxe_srq),
0048         .elem_offset    = offsetof(struct rxe_srq, elem),
0049         .cleanup    = rxe_srq_cleanup,
0050         .min_index  = RXE_MIN_SRQ_INDEX,
0051         .max_index  = RXE_MAX_SRQ_INDEX,
0052         .max_elem   = RXE_MAX_SRQ_INDEX - RXE_MIN_SRQ_INDEX + 1,
0053     },
0054     [RXE_TYPE_QP] = {
0055         .name       = "qp",
0056         .size       = sizeof(struct rxe_qp),
0057         .elem_offset    = offsetof(struct rxe_qp, elem),
0058         .cleanup    = rxe_qp_cleanup,
0059         .min_index  = RXE_MIN_QP_INDEX,
0060         .max_index  = RXE_MAX_QP_INDEX,
0061         .max_elem   = RXE_MAX_QP_INDEX - RXE_MIN_QP_INDEX + 1,
0062     },
0063     [RXE_TYPE_CQ] = {
0064         .name       = "cq",
0065         .size       = sizeof(struct rxe_cq),
0066         .elem_offset    = offsetof(struct rxe_cq, elem),
0067         .cleanup    = rxe_cq_cleanup,
0068         .min_index  = 1,
0069         .max_index  = UINT_MAX,
0070         .max_elem   = UINT_MAX,
0071     },
0072     [RXE_TYPE_MR] = {
0073         .name       = "mr",
0074         .size       = sizeof(struct rxe_mr),
0075         .elem_offset    = offsetof(struct rxe_mr, elem),
0076         .cleanup    = rxe_mr_cleanup,
0077         .min_index  = RXE_MIN_MR_INDEX,
0078         .max_index  = RXE_MAX_MR_INDEX,
0079         .max_elem   = RXE_MAX_MR_INDEX - RXE_MIN_MR_INDEX + 1,
0080     },
0081     [RXE_TYPE_MW] = {
0082         .name       = "mw",
0083         .size       = sizeof(struct rxe_mw),
0084         .elem_offset    = offsetof(struct rxe_mw, elem),
0085         .cleanup    = rxe_mw_cleanup,
0086         .min_index  = RXE_MIN_MW_INDEX,
0087         .max_index  = RXE_MAX_MW_INDEX,
0088         .max_elem   = RXE_MAX_MW_INDEX - RXE_MIN_MW_INDEX + 1,
0089     },
0090 };
0091
0092 void rxe_pool_init(struct rxe_dev *rxe, struct rxe_pool *pool,
0093            enum rxe_elem_type type)
0094 {
0095     const struct rxe_type_info *info = &rxe_type_info[type];
0096
0097     memset(pool, 0, sizeof(*pool));
0098
0099     pool->rxe       = rxe;
0100     pool->name      = info->name;
0101     pool->type      = type;
0102     pool->max_elem      = info->max_elem;
0103     pool->elem_size     = ALIGN(info->size, RXE_POOL_ALIGN);
0104     pool->elem_offset   = info->elem_offset;
0105     pool->cleanup       = info->cleanup;
0106
0107     atomic_set(&pool->num_elem, 0);
0108
0109     xa_init_flags(&pool->xa, XA_FLAGS_ALLOC);
0110     pool->limit.min = info->min_index;
0111     pool->limit.max = info->max_index;
0112 }
0113
0114 void rxe_pool_cleanup(struct rxe_pool *pool)
0115 {
0116     WARN_ON(!xa_empty(&pool->xa));
0117 }
0118
0119 void *rxe_alloc(struct rxe_pool *pool)
0120 {
0121     struct rxe_pool_elem *elem;
0122     void *obj;
0123     int err;
0124
0125     if (WARN_ON(!(pool->type == RXE_TYPE_MR)))
0126         return NULL;
0127
0128     if (atomic_inc_return(&pool->num_elem) > pool->max_elem)
0129         goto err_cnt;
0130
0131     obj = kzalloc(pool->elem_size, GFP_KERNEL);
0132     if (!obj)
0133         goto err_cnt;
0134
0135     elem = (struct rxe_pool_elem *)((u8 *)obj + pool->elem_offset);
0136
0137     elem->pool = pool;
0138     elem->obj = obj;
0139     kref_init(&elem->ref_cnt);
0140     init_completion(&elem->complete);
0141
0142     /* allocate index in array but leave pointer as NULL so it
0143      * can't be looked up until rxe_finalize() is called
0144      */
0145     err = xa_alloc_cyclic(&pool->xa, &elem->index, NULL, pool->limit,
0146                   &pool->next, GFP_KERNEL);
0147     if (err < 0)
0148         goto err_free;
0149
0150     return obj;
0151
0152 err_free:
0153     kfree(obj);
0154 err_cnt:
0155     atomic_dec(&pool->num_elem);
0156     return NULL;
0157 }
0158
0159 int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_elem *elem,
0160                 bool sleepable)
0161 {
0162     int err;
0163     gfp_t gfp_flags;
0164
0165     if (WARN_ON(pool->type == RXE_TYPE_MR))
0166         return -EINVAL;
0167
0168     if (atomic_inc_return(&pool->num_elem) > pool->max_elem)
0169         goto err_cnt;
0170
0171     elem->pool = pool;
0172     elem->obj = (u8 *)elem - pool->elem_offset;
0173     kref_init(&elem->ref_cnt);
0174     init_completion(&elem->complete);
0175
0176     /* AH objects are unique in that the create_ah verb
0177      * can be called in atomic context. If the create_ah
0178      * call is not sleepable use GFP_ATOMIC.
0179      */
0180     gfp_flags = sleepable ? GFP_KERNEL : GFP_ATOMIC;
0181
0182     if (sleepable)
0183         might_sleep();
0184     err = xa_alloc_cyclic(&pool->xa, &elem->index, NULL, pool->limit,
0185                   &pool->next, gfp_flags);
0186     if (err < 0)
0187         goto err_cnt;
0188
0189     return 0;
0190
0191 err_cnt:
0192     atomic_dec(&pool->num_elem);
0193     return -EINVAL;
0194 }
0195
0196 void *rxe_pool_get_index(struct rxe_pool *pool, u32 index)
0197 {
0198     struct rxe_pool_elem *elem;
0199     struct xarray *xa = &pool->xa;
0200     void *obj;
0201
0202     rcu_read_lock();
0203     elem = xa_load(xa, index);
0204     if (elem && kref_get_unless_zero(&elem->ref_cnt))
0205         obj = elem->obj;
0206     else
0207         obj = NULL;
0208     rcu_read_unlock();
0209
0210     return obj;
0211 }
0212
0213 static void rxe_elem_release(struct kref *kref)
0214 {
0215     struct rxe_pool_elem *elem = container_of(kref, typeof(*elem), ref_cnt);
0216
0217     complete(&elem->complete);
0218 }
0219
0220 int __rxe_cleanup(struct rxe_pool_elem *elem, bool sleepable)
0221 {
0222     struct rxe_pool *pool = elem->pool;
0223     struct xarray *xa = &pool->xa;
0224     static int timeout = RXE_POOL_TIMEOUT;
0225     int ret, err = 0;
0226     void *xa_ret;
0227
0228     if (sleepable)
0229         might_sleep();
0230
0231     /* erase xarray entry to prevent looking up
0232      * the pool elem from its index
0233      */
0234     xa_ret = xa_erase(xa, elem->index);
0235     WARN_ON(xa_err(xa_ret));
0236
0237     /* if this is the last call to rxe_put complete the
0238      * object. It is safe to touch obj->elem after this since
0239      * it is freed below
0240      */
0241     __rxe_put(elem);
0242
0243     /* wait until all references to the object have been
0244      * dropped before final object specific cleanup and
0245      * return to rdma-core
0246      */
0247     if (sleepable) {
0248         if (!completion_done(&elem->complete) && timeout) {
0249             ret = wait_for_completion_timeout(&elem->complete,
0250                     timeout);
0251
0252             /* Shouldn't happen. There are still references to
0253              * the object but, rather than deadlock, free the
0254              * object or pass back to rdma-core.
0255              */
0256             if (WARN_ON(!ret))
0257                 err = -EINVAL;
0258         }
0259     } else {
0260         unsigned long until = jiffies + timeout;
0261
0262         /* AH objects are unique in that the destroy_ah verb
0263          * can be called in atomic context. This delay
0264          * replaces the wait_for_completion call above
0265          * when the destroy_ah call is not sleepable
0266          */
0267         while (!completion_done(&elem->complete) &&
0268                 time_before(jiffies, until))
0269             mdelay(1);
0270
0271         if (WARN_ON(!completion_done(&elem->complete)))
0272             err = -EINVAL;
0273     }
0274
0275     if (pool->cleanup)
0276         pool->cleanup(elem);
0277
0278     if (pool->type == RXE_TYPE_MR)
0279         kfree_rcu(elem->obj);
0280
0281     atomic_dec(&pool->num_elem);
0282
0283     return err;
0284 }
0285
0286 int __rxe_get(struct rxe_pool_elem *elem)
0287 {
0288     return kref_get_unless_zero(&elem->ref_cnt);
0289 }
0290
0291 int __rxe_put(struct rxe_pool_elem *elem)
0292 {
0293     return kref_put(&elem->ref_cnt, rxe_elem_release);
0294 }
0295
0296 void __rxe_finalize(struct rxe_pool_elem *elem)
0297 {
0298     void *xa_ret;
0299
0300     xa_ret = xa_store(&elem->pool->xa, elem->index, elem, GFP_KERNEL);
0301     WARN_ON(xa_err(xa_ret));
0302 }