the-tree/lib/sg_pool.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 #include <linux/module.h>
0003 #include <linux/scatterlist.h>
0004 #include <linux/mempool.h>
0005 #include <linux/slab.h>
0006
0007 #define SG_MEMPOOL_NR       ARRAY_SIZE(sg_pools)
0008 #define SG_MEMPOOL_SIZE     2
0009
0010 struct sg_pool {
0011     size_t      size;
0012     char        *name;
0013     struct kmem_cache   *slab;
0014     mempool_t   *pool;
0015 };
0016
0017 #define SP(x) { .size = x, "sgpool-" __stringify(x) }
0018 #if (SG_CHUNK_SIZE < 32)
0019 #error SG_CHUNK_SIZE is too small (must be 32 or greater)
0020 #endif
0021 static struct sg_pool sg_pools[] = {
0022     SP(8),
0023     SP(16),
0024 #if (SG_CHUNK_SIZE > 32)
0025     SP(32),
0026 #if (SG_CHUNK_SIZE > 64)
0027     SP(64),
0028 #if (SG_CHUNK_SIZE > 128)
0029     SP(128),
0030 #if (SG_CHUNK_SIZE > 256)
0031 #error SG_CHUNK_SIZE is too large (256 MAX)
0032 #endif
0033 #endif
0034 #endif
0035 #endif
0036     SP(SG_CHUNK_SIZE)
0037 };
0038 #undef SP
0039
0040 static inline unsigned int sg_pool_index(unsigned short nents)
0041 {
0042     unsigned int index;
0043
0044     BUG_ON(nents > SG_CHUNK_SIZE);
0045
0046     if (nents <= 8)
0047         index = 0;
0048     else
0049         index = get_count_order(nents) - 3;
0050
0051     return index;
0052 }
0053
0054 static void sg_pool_free(struct scatterlist *sgl, unsigned int nents)
0055 {
0056     struct sg_pool *sgp;
0057
0058     sgp = sg_pools + sg_pool_index(nents);
0059     mempool_free(sgl, sgp->pool);
0060 }
0061
0062 static struct scatterlist *sg_pool_alloc(unsigned int nents, gfp_t gfp_mask)
0063 {
0064     struct sg_pool *sgp;
0065
0066     sgp = sg_pools + sg_pool_index(nents);
0067     return mempool_alloc(sgp->pool, gfp_mask);
0068 }
0069
0070 /**
0071  * sg_free_table_chained - Free a previously mapped sg table
0072  * @table:  The sg table header to use
0073  * @nents_first_chunk: size of the first_chunk SGL passed to
0074  *      sg_alloc_table_chained
0075  *
0076  *  Description:
0077  *    Free an sg table previously allocated and setup with
0078  *    sg_alloc_table_chained().
0079  *
0080  *    @nents_first_chunk has to be same with that same parameter passed
0081  *    to sg_alloc_table_chained().
0082  *
0083  **/
0084 void sg_free_table_chained(struct sg_table *table,
0085         unsigned nents_first_chunk)
0086 {
0087     if (table->orig_nents <= nents_first_chunk)
0088         return;
0089
0090     if (nents_first_chunk == 1)
0091         nents_first_chunk = 0;
0092
0093     __sg_free_table(table, SG_CHUNK_SIZE, nents_first_chunk, sg_pool_free,
0094             table->orig_nents);
0095 }
0096 EXPORT_SYMBOL_GPL(sg_free_table_chained);
0097
0098 /**
0099  * sg_alloc_table_chained - Allocate and chain SGLs in an sg table
0100  * @table:  The sg table header to use
0101  * @nents:  Number of entries in sg list
0102  * @first_chunk: first SGL
0103  * @nents_first_chunk: number of the SGL of @first_chunk
0104  *
0105  *  Description:
0106  *    Allocate and chain SGLs in an sg table. If @nents@ is larger than
0107  *    @nents_first_chunk a chained sg table will be setup. @first_chunk is
0108  *    ignored if nents_first_chunk <= 1 because user expects the SGL points
0109  *    non-chain SGL.
0110  *
0111  **/
0112 int sg_alloc_table_chained(struct sg_table *table, int nents,
0113         struct scatterlist *first_chunk, unsigned nents_first_chunk)
0114 {
0115     int ret;
0116
0117     BUG_ON(!nents);
0118
0119     if (first_chunk && nents_first_chunk) {
0120         if (nents <= nents_first_chunk) {
0121             table->nents = table->orig_nents = nents;
0122             sg_init_table(table->sgl, nents);
0123             return 0;
0124         }
0125     }
0126
0127     /* User supposes that the 1st SGL includes real entry */
0128     if (nents_first_chunk <= 1) {
0129         first_chunk = NULL;
0130         nents_first_chunk = 0;
0131     }
0132
0133     ret = __sg_alloc_table(table, nents, SG_CHUNK_SIZE,
0134                    first_chunk, nents_first_chunk,
0135                    GFP_ATOMIC, sg_pool_alloc);
0136     if (unlikely(ret))
0137         sg_free_table_chained(table, nents_first_chunk);
0138     return ret;
0139 }
0140 EXPORT_SYMBOL_GPL(sg_alloc_table_chained);
0141
0142 static __init int sg_pool_init(void)
0143 {
0144     int i;
0145
0146     for (i = 0; i < SG_MEMPOOL_NR; i++) {
0147         struct sg_pool *sgp = sg_pools + i;
0148         int size = sgp->size * sizeof(struct scatterlist);
0149
0150         sgp->slab = kmem_cache_create(sgp->name, size, 0,
0151                 SLAB_HWCACHE_ALIGN, NULL);
0152         if (!sgp->slab) {
0153             printk(KERN_ERR "SG_POOL: can't init sg slab %s\n",
0154                     sgp->name);
0155             goto cleanup_sdb;
0156         }
0157
0158         sgp->pool = mempool_create_slab_pool(SG_MEMPOOL_SIZE,
0159                              sgp->slab);
0160         if (!sgp->pool) {
0161             printk(KERN_ERR "SG_POOL: can't init sg mempool %s\n",
0162                     sgp->name);
0163             goto cleanup_sdb;
0164         }
0165     }
0166
0167     return 0;
0168
0169 cleanup_sdb:
0170     for (i = 0; i < SG_MEMPOOL_NR; i++) {
0171         struct sg_pool *sgp = sg_pools + i;
0172
0173         mempool_destroy(sgp->pool);
0174         kmem_cache_destroy(sgp->slab);
0175     }
0176
0177     return -ENOMEM;
0178 }
0179
0180 static __exit void sg_pool_exit(void)
0181 {
0182     int i;
0183
0184     for (i = 0; i < SG_MEMPOOL_NR; i++) {
0185         struct sg_pool *sgp = sg_pools + i;
0186         mempool_destroy(sgp->pool);
0187         kmem_cache_destroy(sgp->slab);
0188     }
0189 }
0190
0191 module_init(sg_pool_init);
0192 module_exit(sg_pool_exit);