crypto/hisilicon/sgl.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /* Copyright (c) 2019 HiSilicon Limited. */
0003 #include <linux/align.h>
0004 #include <linux/dma-mapping.h>
0005 #include <linux/hisi_acc_qm.h>
0006 #include <linux/module.h>
0007 #include <linux/slab.h>
0008
0009 #define HISI_ACC_SGL_SGE_NR_MIN     1
0010 #define HISI_ACC_SGL_NR_MAX     256
0011 #define HISI_ACC_SGL_ALIGN_SIZE     64
0012 #define HISI_ACC_MEM_BLOCK_NR       5
0013
0014 struct acc_hw_sge {
0015     dma_addr_t buf;
0016     void *page_ctrl;
0017     __le32 len;
0018     __le32 pad;
0019     __le32 pad0;
0020     __le32 pad1;
0021 };
0022
0023 /* use default sgl head size 64B */
0024 struct hisi_acc_hw_sgl {
0025     dma_addr_t next_dma;
0026     __le16 entry_sum_in_chain;
0027     __le16 entry_sum_in_sgl;
0028     __le16 entry_length_in_sgl;
0029     __le16 pad0;
0030     __le64 pad1[5];
0031     struct hisi_acc_hw_sgl *next;
0032     struct acc_hw_sge sge_entries[];
0033 } __aligned(1);
0034
0035 struct hisi_acc_sgl_pool {
0036     struct mem_block {
0037         struct hisi_acc_hw_sgl *sgl;
0038         dma_addr_t sgl_dma;
0039         size_t size;
0040     } mem_block[HISI_ACC_MEM_BLOCK_NR];
0041     u32 sgl_num_per_block;
0042     u32 block_num;
0043     u32 count;
0044     u32 sge_nr;
0045     size_t sgl_size;
0046 };
0047
0048 /**
0049  * hisi_acc_create_sgl_pool() - Create a hw sgl pool.
0050  * @dev: The device which hw sgl pool belongs to.
0051  * @count: Count of hisi_acc_hw_sgl in pool.
0052  * @sge_nr: The count of sge in hw_sgl
0053  *
0054  * This function creates a hw sgl pool, after this user can get hw sgl memory
0055  * from it.
0056  */
0057 struct hisi_acc_sgl_pool *hisi_acc_create_sgl_pool(struct device *dev,
0058                            u32 count, u32 sge_nr)
0059 {
0060     u32 sgl_size, block_size, sgl_num_per_block, block_num, remain_sgl;
0061     struct hisi_acc_sgl_pool *pool;
0062     struct mem_block *block;
0063     u32 i, j;
0064
0065     if (!dev || !count || !sge_nr || sge_nr > HISI_ACC_SGL_SGE_NR_MAX)
0066         return ERR_PTR(-EINVAL);
0067
0068     sgl_size = ALIGN(sizeof(struct acc_hw_sge) * sge_nr +
0069              sizeof(struct hisi_acc_hw_sgl),
0070              HISI_ACC_SGL_ALIGN_SIZE);
0071
0072     /*
0073      * the pool may allocate a block of memory of size PAGE_SIZE * 2^(MAX_ORDER - 1),
0074      * block size may exceed 2^31 on ia64, so the max of block size is 2^31
0075      */
0076     block_size = 1 << (PAGE_SHIFT + MAX_ORDER <= 32 ?
0077                PAGE_SHIFT + MAX_ORDER - 1 : 31);
0078     sgl_num_per_block = block_size / sgl_size;
0079     block_num = count / sgl_num_per_block;
0080     remain_sgl = count % sgl_num_per_block;
0081
0082     if ((!remain_sgl && block_num > HISI_ACC_MEM_BLOCK_NR) ||
0083         (remain_sgl > 0 && block_num > HISI_ACC_MEM_BLOCK_NR - 1))
0084         return ERR_PTR(-EINVAL);
0085
0086     pool = kzalloc(sizeof(*pool), GFP_KERNEL);
0087     if (!pool)
0088         return ERR_PTR(-ENOMEM);
0089     block = pool->mem_block;
0090
0091     for (i = 0; i < block_num; i++) {
0092         block[i].sgl = dma_alloc_coherent(dev, block_size,
0093                           &block[i].sgl_dma,
0094                           GFP_KERNEL);
0095         if (!block[i].sgl) {
0096             dev_err(dev, "Fail to allocate hw SG buffer!\n");
0097             goto err_free_mem;
0098         }
0099
0100         block[i].size = block_size;
0101     }
0102
0103     if (remain_sgl > 0) {
0104         block[i].sgl = dma_alloc_coherent(dev, remain_sgl * sgl_size,
0105                           &block[i].sgl_dma,
0106                           GFP_KERNEL);
0107         if (!block[i].sgl) {
0108             dev_err(dev, "Fail to allocate remained hw SG buffer!\n");
0109             goto err_free_mem;
0110         }
0111
0112         block[i].size = remain_sgl * sgl_size;
0113     }
0114
0115     pool->sgl_num_per_block = sgl_num_per_block;
0116     pool->block_num = remain_sgl ? block_num + 1 : block_num;
0117     pool->count = count;
0118     pool->sgl_size = sgl_size;
0119     pool->sge_nr = sge_nr;
0120
0121     return pool;
0122
0123 err_free_mem:
0124     for (j = 0; j < i; j++) {
0125         dma_free_coherent(dev, block_size, block[j].sgl,
0126                   block[j].sgl_dma);
0127         memset(block + j, 0, sizeof(*block));
0128     }
0129     kfree(pool);
0130     return ERR_PTR(-ENOMEM);
0131 }
0132 EXPORT_SYMBOL_GPL(hisi_acc_create_sgl_pool);
0133
0134 /**
0135  * hisi_acc_free_sgl_pool() - Free a hw sgl pool.
0136  * @dev: The device which hw sgl pool belongs to.
0137  * @pool: Pointer of pool.
0138  *
0139  * This function frees memory of a hw sgl pool.
0140  */
0141 void hisi_acc_free_sgl_pool(struct device *dev, struct hisi_acc_sgl_pool *pool)
0142 {
0143     struct mem_block *block;
0144     int i;
0145
0146     if (!dev || !pool)
0147         return;
0148
0149     block = pool->mem_block;
0150
0151     for (i = 0; i < pool->block_num; i++)
0152         dma_free_coherent(dev, block[i].size, block[i].sgl,
0153                   block[i].sgl_dma);
0154
0155     kfree(pool);
0156 }
0157 EXPORT_SYMBOL_GPL(hisi_acc_free_sgl_pool);
0158
0159 static struct hisi_acc_hw_sgl *acc_get_sgl(struct hisi_acc_sgl_pool *pool,
0160                        u32 index, dma_addr_t *hw_sgl_dma)
0161 {
0162     struct mem_block *block;
0163     u32 block_index, offset;
0164
0165     if (!pool || !hw_sgl_dma || index >= pool->count)
0166         return ERR_PTR(-EINVAL);
0167
0168     block = pool->mem_block;
0169     block_index = index / pool->sgl_num_per_block;
0170     offset = index % pool->sgl_num_per_block;
0171
0172     *hw_sgl_dma = block[block_index].sgl_dma + pool->sgl_size * offset;
0173     return (void *)block[block_index].sgl + pool->sgl_size * offset;
0174 }
0175
0176 static void sg_map_to_hw_sg(struct scatterlist *sgl,
0177                 struct acc_hw_sge *hw_sge)
0178 {
0179     hw_sge->buf = sg_dma_address(sgl);
0180     hw_sge->len = cpu_to_le32(sg_dma_len(sgl));
0181     hw_sge->page_ctrl = sg_virt(sgl);
0182 }
0183
0184 static void inc_hw_sgl_sge(struct hisi_acc_hw_sgl *hw_sgl)
0185 {
0186     u16 var = le16_to_cpu(hw_sgl->entry_sum_in_sgl);
0187
0188     var++;
0189     hw_sgl->entry_sum_in_sgl = cpu_to_le16(var);
0190 }
0191
0192 static void update_hw_sgl_sum_sge(struct hisi_acc_hw_sgl *hw_sgl, u16 sum)
0193 {
0194     hw_sgl->entry_sum_in_chain = cpu_to_le16(sum);
0195 }
0196
0197 static void clear_hw_sgl_sge(struct hisi_acc_hw_sgl *hw_sgl)
0198 {
0199     struct acc_hw_sge *hw_sge = hw_sgl->sge_entries;
0200     int i;
0201
0202     for (i = 0; i < le16_to_cpu(hw_sgl->entry_sum_in_sgl); i++) {
0203         hw_sge[i].page_ctrl = NULL;
0204         hw_sge[i].buf = 0;
0205         hw_sge[i].len = 0;
0206     }
0207 }
0208
0209 /**
0210  * hisi_acc_sg_buf_map_to_hw_sgl - Map a scatterlist to a hw sgl.
0211  * @dev: The device which hw sgl belongs to.
0212  * @sgl: Scatterlist which will be mapped to hw sgl.
0213  * @pool: Pool which hw sgl memory will be allocated in.
0214  * @index: Index of hisi_acc_hw_sgl in pool.
0215  * @hw_sgl_dma: The dma address of allocated hw sgl.
0216  *
0217  * This function builds hw sgl according input sgl, user can use hw_sgl_dma
0218  * as src/dst in its BD. Only support single hw sgl currently.
0219  */
0220 struct hisi_acc_hw_sgl *
0221 hisi_acc_sg_buf_map_to_hw_sgl(struct device *dev,
0222                   struct scatterlist *sgl,
0223                   struct hisi_acc_sgl_pool *pool,
0224                   u32 index, dma_addr_t *hw_sgl_dma)
0225 {
0226     struct hisi_acc_hw_sgl *curr_hw_sgl;
0227     dma_addr_t curr_sgl_dma = 0;
0228     struct acc_hw_sge *curr_hw_sge;
0229     struct scatterlist *sg;
0230     int i, sg_n, sg_n_mapped;
0231
0232     if (!dev || !sgl || !pool || !hw_sgl_dma)
0233         return ERR_PTR(-EINVAL);
0234
0235     sg_n = sg_nents(sgl);
0236
0237     sg_n_mapped = dma_map_sg(dev, sgl, sg_n, DMA_BIDIRECTIONAL);
0238     if (!sg_n_mapped) {
0239         dev_err(dev, "DMA mapping for SG error!\n");
0240         return ERR_PTR(-EINVAL);
0241     }
0242
0243     if (sg_n_mapped > pool->sge_nr) {
0244         dev_err(dev, "the number of entries in input scatterlist is bigger than SGL pool setting.\n");
0245         return ERR_PTR(-EINVAL);
0246     }
0247
0248     curr_hw_sgl = acc_get_sgl(pool, index, &curr_sgl_dma);
0249     if (IS_ERR(curr_hw_sgl)) {
0250         dev_err(dev, "Get SGL error!\n");
0251         dma_unmap_sg(dev, sgl, sg_n, DMA_BIDIRECTIONAL);
0252         return ERR_PTR(-ENOMEM);
0253
0254     }
0255     curr_hw_sgl->entry_length_in_sgl = cpu_to_le16(pool->sge_nr);
0256     curr_hw_sge = curr_hw_sgl->sge_entries;
0257
0258     for_each_sg(sgl, sg, sg_n_mapped, i) {
0259         sg_map_to_hw_sg(sg, curr_hw_sge);
0260         inc_hw_sgl_sge(curr_hw_sgl);
0261         curr_hw_sge++;
0262     }
0263
0264     update_hw_sgl_sum_sge(curr_hw_sgl, pool->sge_nr);
0265     *hw_sgl_dma = curr_sgl_dma;
0266
0267     return curr_hw_sgl;
0268 }
0269 EXPORT_SYMBOL_GPL(hisi_acc_sg_buf_map_to_hw_sgl);
0270
0271 /**
0272  * hisi_acc_sg_buf_unmap() - Unmap allocated hw sgl.
0273  * @dev: The device which hw sgl belongs to.
0274  * @sgl: Related scatterlist.
0275  * @hw_sgl: Virtual address of hw sgl.
0276  *
0277  * This function unmaps allocated hw sgl.
0278  */
0279 void hisi_acc_sg_buf_unmap(struct device *dev, struct scatterlist *sgl,
0280                struct hisi_acc_hw_sgl *hw_sgl)
0281 {
0282     if (!dev || !sgl || !hw_sgl)
0283         return;
0284
0285     dma_unmap_sg(dev, sgl, sg_nents(sgl), DMA_BIDIRECTIONAL);
0286     clear_hw_sgl_sge(hw_sgl);
0287     hw_sgl->entry_sum_in_chain = 0;
0288     hw_sgl->entry_sum_in_sgl = 0;
0289     hw_sgl->entry_length_in_sgl = 0;
0290 }
0291 EXPORT_SYMBOL_GPL(hisi_acc_sg_buf_unmap);