Back to home page

OSCL-LXR
 
 

     

0001 /*
0002  * Copyright (c) 2016 Hisilicon Limited.
0003  * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
0004  *
0005  * This software is available to you under a choice of one of two
0006  * licenses.  You may choose to be licensed under the terms of the GNU
0007  * General Public License (GPL) Version 2, available from the file
0008  * COPYING in the main directory of this source tree, or the
0009  * OpenIB.org BSD license below:
0010  *
0011  *     Redistribution and use in source and binary forms, with or
0012  *     without modification, are permitted provided that the following
0013  *     conditions are met:
0014  *
0015  *      - Redistributions of source code must retain the above
0016  *        copyright notice, this list of conditions and the following
0017  *        disclaimer.
0018  *
0019  *      - Redistributions in binary form must reproduce the above
0020  *        copyright notice, this list of conditions and the following
0021  *        disclaimer in the documentation and/or other materials
0022  *        provided with the distribution.
0023  *
0024  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
0025  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
0026  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
0027  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
0028  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
0029  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
0030  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
0031  * SOFTWARE.
0032  */
0033 
0034 #include "hns_roce_device.h"
0035 #include "hns_roce_hem.h"
0036 #include "hns_roce_common.h"
0037 
0038 #define HEM_INDEX_BUF           BIT(0)
0039 #define HEM_INDEX_L0            BIT(1)
0040 #define HEM_INDEX_L1            BIT(2)
0041 struct hns_roce_hem_index {
0042     u64 buf;
0043     u64 l0;
0044     u64 l1;
0045     u32 inited; /* indicate which index is available */
0046 };
0047 
0048 bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type)
0049 {
0050     int hop_num = 0;
0051 
0052     switch (type) {
0053     case HEM_TYPE_QPC:
0054         hop_num = hr_dev->caps.qpc_hop_num;
0055         break;
0056     case HEM_TYPE_MTPT:
0057         hop_num = hr_dev->caps.mpt_hop_num;
0058         break;
0059     case HEM_TYPE_CQC:
0060         hop_num = hr_dev->caps.cqc_hop_num;
0061         break;
0062     case HEM_TYPE_SRQC:
0063         hop_num = hr_dev->caps.srqc_hop_num;
0064         break;
0065     case HEM_TYPE_SCCC:
0066         hop_num = hr_dev->caps.sccc_hop_num;
0067         break;
0068     case HEM_TYPE_QPC_TIMER:
0069         hop_num = hr_dev->caps.qpc_timer_hop_num;
0070         break;
0071     case HEM_TYPE_CQC_TIMER:
0072         hop_num = hr_dev->caps.cqc_timer_hop_num;
0073         break;
0074     case HEM_TYPE_GMV:
0075         hop_num = hr_dev->caps.gmv_hop_num;
0076         break;
0077     default:
0078         return false;
0079     }
0080 
0081     return hop_num ? true : false;
0082 }
0083 
0084 static bool hns_roce_check_hem_null(struct hns_roce_hem **hem, u64 hem_idx,
0085                     u32 bt_chunk_num, u64 hem_max_num)
0086 {
0087     u64 start_idx = round_down(hem_idx, bt_chunk_num);
0088     u64 check_max_num = start_idx + bt_chunk_num;
0089     u64 i;
0090 
0091     for (i = start_idx; (i < check_max_num) && (i < hem_max_num); i++)
0092         if (i != hem_idx && hem[i])
0093             return false;
0094 
0095     return true;
0096 }
0097 
0098 static bool hns_roce_check_bt_null(u64 **bt, u64 ba_idx, u32 bt_chunk_num)
0099 {
0100     u64 start_idx = round_down(ba_idx, bt_chunk_num);
0101     int i;
0102 
0103     for (i = 0; i < bt_chunk_num; i++)
0104         if (i != ba_idx && bt[start_idx + i])
0105             return false;
0106 
0107     return true;
0108 }
0109 
0110 static int hns_roce_get_bt_num(u32 table_type, u32 hop_num)
0111 {
0112     if (check_whether_bt_num_3(table_type, hop_num))
0113         return 3;
0114     else if (check_whether_bt_num_2(table_type, hop_num))
0115         return 2;
0116     else if (check_whether_bt_num_1(table_type, hop_num))
0117         return 1;
0118     else
0119         return 0;
0120 }
0121 
0122 static int get_hem_table_config(struct hns_roce_dev *hr_dev,
0123                 struct hns_roce_hem_mhop *mhop,
0124                 u32 type)
0125 {
0126     struct device *dev = hr_dev->dev;
0127 
0128     switch (type) {
0129     case HEM_TYPE_QPC:
0130         mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_buf_pg_sz
0131                          + PAGE_SHIFT);
0132         mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_ba_pg_sz
0133                          + PAGE_SHIFT);
0134         mhop->ba_l0_num = hr_dev->caps.qpc_bt_num;
0135         mhop->hop_num = hr_dev->caps.qpc_hop_num;
0136         break;
0137     case HEM_TYPE_MTPT:
0138         mhop->buf_chunk_size = 1 << (hr_dev->caps.mpt_buf_pg_sz
0139                          + PAGE_SHIFT);
0140         mhop->bt_chunk_size = 1 << (hr_dev->caps.mpt_ba_pg_sz
0141                          + PAGE_SHIFT);
0142         mhop->ba_l0_num = hr_dev->caps.mpt_bt_num;
0143         mhop->hop_num = hr_dev->caps.mpt_hop_num;
0144         break;
0145     case HEM_TYPE_CQC:
0146         mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_buf_pg_sz
0147                          + PAGE_SHIFT);
0148         mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_ba_pg_sz
0149                         + PAGE_SHIFT);
0150         mhop->ba_l0_num = hr_dev->caps.cqc_bt_num;
0151         mhop->hop_num = hr_dev->caps.cqc_hop_num;
0152         break;
0153     case HEM_TYPE_SCCC:
0154         mhop->buf_chunk_size = 1 << (hr_dev->caps.sccc_buf_pg_sz
0155                          + PAGE_SHIFT);
0156         mhop->bt_chunk_size = 1 << (hr_dev->caps.sccc_ba_pg_sz
0157                         + PAGE_SHIFT);
0158         mhop->ba_l0_num = hr_dev->caps.sccc_bt_num;
0159         mhop->hop_num = hr_dev->caps.sccc_hop_num;
0160         break;
0161     case HEM_TYPE_QPC_TIMER:
0162         mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_timer_buf_pg_sz
0163                          + PAGE_SHIFT);
0164         mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_timer_ba_pg_sz
0165                         + PAGE_SHIFT);
0166         mhop->ba_l0_num = hr_dev->caps.qpc_timer_bt_num;
0167         mhop->hop_num = hr_dev->caps.qpc_timer_hop_num;
0168         break;
0169     case HEM_TYPE_CQC_TIMER:
0170         mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_timer_buf_pg_sz
0171                          + PAGE_SHIFT);
0172         mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_timer_ba_pg_sz
0173                         + PAGE_SHIFT);
0174         mhop->ba_l0_num = hr_dev->caps.cqc_timer_bt_num;
0175         mhop->hop_num = hr_dev->caps.cqc_timer_hop_num;
0176         break;
0177     case HEM_TYPE_SRQC:
0178         mhop->buf_chunk_size = 1 << (hr_dev->caps.srqc_buf_pg_sz
0179                          + PAGE_SHIFT);
0180         mhop->bt_chunk_size = 1 << (hr_dev->caps.srqc_ba_pg_sz
0181                          + PAGE_SHIFT);
0182         mhop->ba_l0_num = hr_dev->caps.srqc_bt_num;
0183         mhop->hop_num = hr_dev->caps.srqc_hop_num;
0184         break;
0185     case HEM_TYPE_GMV:
0186         mhop->buf_chunk_size = 1 << (hr_dev->caps.gmv_buf_pg_sz +
0187                          PAGE_SHIFT);
0188         mhop->bt_chunk_size = 1 << (hr_dev->caps.gmv_ba_pg_sz +
0189                         PAGE_SHIFT);
0190         mhop->ba_l0_num = hr_dev->caps.gmv_bt_num;
0191         mhop->hop_num = hr_dev->caps.gmv_hop_num;
0192         break;
0193     default:
0194         dev_err(dev, "table %u not support multi-hop addressing!\n",
0195             type);
0196         return -EINVAL;
0197     }
0198 
0199     return 0;
0200 }
0201 
0202 int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev,
0203                struct hns_roce_hem_table *table, unsigned long *obj,
0204                struct hns_roce_hem_mhop *mhop)
0205 {
0206     struct device *dev = hr_dev->dev;
0207     u32 chunk_ba_num;
0208     u32 chunk_size;
0209     u32 table_idx;
0210     u32 bt_num;
0211 
0212     if (get_hem_table_config(hr_dev, mhop, table->type))
0213         return -EINVAL;
0214 
0215     if (!obj)
0216         return 0;
0217 
0218     /*
0219      * QPC/MTPT/CQC/SRQC/SCCC alloc hem for buffer pages.
0220      * MTT/CQE alloc hem for bt pages.
0221      */
0222     bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num);
0223     chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN;
0224     chunk_size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size :
0225                   mhop->bt_chunk_size;
0226     table_idx = *obj / (chunk_size / table->obj_size);
0227     switch (bt_num) {
0228     case 3:
0229         mhop->l2_idx = table_idx & (chunk_ba_num - 1);
0230         mhop->l1_idx = table_idx / chunk_ba_num & (chunk_ba_num - 1);
0231         mhop->l0_idx = (table_idx / chunk_ba_num) / chunk_ba_num;
0232         break;
0233     case 2:
0234         mhop->l1_idx = table_idx & (chunk_ba_num - 1);
0235         mhop->l0_idx = table_idx / chunk_ba_num;
0236         break;
0237     case 1:
0238         mhop->l0_idx = table_idx;
0239         break;
0240     default:
0241         dev_err(dev, "table %u not support hop_num = %u!\n",
0242             table->type, mhop->hop_num);
0243         return -EINVAL;
0244     }
0245     if (mhop->l0_idx >= mhop->ba_l0_num)
0246         mhop->l0_idx %= mhop->ba_l0_num;
0247 
0248     return 0;
0249 }
0250 
0251 static struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev,
0252                            int npages,
0253                            unsigned long hem_alloc_size,
0254                            gfp_t gfp_mask)
0255 {
0256     struct hns_roce_hem_chunk *chunk = NULL;
0257     struct hns_roce_hem *hem;
0258     struct scatterlist *mem;
0259     int order;
0260     void *buf;
0261 
0262     WARN_ON(gfp_mask & __GFP_HIGHMEM);
0263 
0264     hem = kmalloc(sizeof(*hem),
0265               gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));
0266     if (!hem)
0267         return NULL;
0268 
0269     INIT_LIST_HEAD(&hem->chunk_list);
0270 
0271     order = get_order(hem_alloc_size);
0272 
0273     while (npages > 0) {
0274         if (!chunk) {
0275             chunk = kmalloc(sizeof(*chunk),
0276                 gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));
0277             if (!chunk)
0278                 goto fail;
0279 
0280             sg_init_table(chunk->mem, HNS_ROCE_HEM_CHUNK_LEN);
0281             chunk->npages = 0;
0282             chunk->nsg = 0;
0283             memset(chunk->buf, 0, sizeof(chunk->buf));
0284             list_add_tail(&chunk->list, &hem->chunk_list);
0285         }
0286 
0287         while (1 << order > npages)
0288             --order;
0289 
0290         /*
0291          * Alloc memory one time. If failed, don't alloc small block
0292          * memory, directly return fail.
0293          */
0294         mem = &chunk->mem[chunk->npages];
0295         buf = dma_alloc_coherent(hr_dev->dev, PAGE_SIZE << order,
0296                 &sg_dma_address(mem), gfp_mask);
0297         if (!buf)
0298             goto fail;
0299 
0300         chunk->buf[chunk->npages] = buf;
0301         sg_dma_len(mem) = PAGE_SIZE << order;
0302 
0303         ++chunk->npages;
0304         ++chunk->nsg;
0305         npages -= 1 << order;
0306     }
0307 
0308     return hem;
0309 
0310 fail:
0311     hns_roce_free_hem(hr_dev, hem);
0312     return NULL;
0313 }
0314 
0315 void hns_roce_free_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem *hem)
0316 {
0317     struct hns_roce_hem_chunk *chunk, *tmp;
0318     int i;
0319 
0320     if (!hem)
0321         return;
0322 
0323     list_for_each_entry_safe(chunk, tmp, &hem->chunk_list, list) {
0324         for (i = 0; i < chunk->npages; ++i)
0325             dma_free_coherent(hr_dev->dev,
0326                    sg_dma_len(&chunk->mem[i]),
0327                    chunk->buf[i],
0328                    sg_dma_address(&chunk->mem[i]));
0329         kfree(chunk);
0330     }
0331 
0332     kfree(hem);
0333 }
0334 
0335 static int calc_hem_config(struct hns_roce_dev *hr_dev,
0336                struct hns_roce_hem_table *table, unsigned long obj,
0337                struct hns_roce_hem_mhop *mhop,
0338                struct hns_roce_hem_index *index)
0339 {
0340     struct ib_device *ibdev = &hr_dev->ib_dev;
0341     unsigned long mhop_obj = obj;
0342     u32 l0_idx, l1_idx, l2_idx;
0343     u32 chunk_ba_num;
0344     u32 bt_num;
0345     int ret;
0346 
0347     ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, mhop);
0348     if (ret)
0349         return ret;
0350 
0351     l0_idx = mhop->l0_idx;
0352     l1_idx = mhop->l1_idx;
0353     l2_idx = mhop->l2_idx;
0354     chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN;
0355     bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num);
0356     switch (bt_num) {
0357     case 3:
0358         index->l1 = l0_idx * chunk_ba_num + l1_idx;
0359         index->l0 = l0_idx;
0360         index->buf = l0_idx * chunk_ba_num * chunk_ba_num +
0361                  l1_idx * chunk_ba_num + l2_idx;
0362         break;
0363     case 2:
0364         index->l0 = l0_idx;
0365         index->buf = l0_idx * chunk_ba_num + l1_idx;
0366         break;
0367     case 1:
0368         index->buf = l0_idx;
0369         break;
0370     default:
0371         ibdev_err(ibdev, "table %u not support mhop.hop_num = %u!\n",
0372               table->type, mhop->hop_num);
0373         return -EINVAL;
0374     }
0375 
0376     if (unlikely(index->buf >= table->num_hem)) {
0377         ibdev_err(ibdev, "table %u exceed hem limt idx %llu, max %lu!\n",
0378               table->type, index->buf, table->num_hem);
0379         return -EINVAL;
0380     }
0381 
0382     return 0;
0383 }
0384 
0385 static void free_mhop_hem(struct hns_roce_dev *hr_dev,
0386               struct hns_roce_hem_table *table,
0387               struct hns_roce_hem_mhop *mhop,
0388               struct hns_roce_hem_index *index)
0389 {
0390     u32 bt_size = mhop->bt_chunk_size;
0391     struct device *dev = hr_dev->dev;
0392 
0393     if (index->inited & HEM_INDEX_BUF) {
0394         hns_roce_free_hem(hr_dev, table->hem[index->buf]);
0395         table->hem[index->buf] = NULL;
0396     }
0397 
0398     if (index->inited & HEM_INDEX_L1) {
0399         dma_free_coherent(dev, bt_size, table->bt_l1[index->l1],
0400                   table->bt_l1_dma_addr[index->l1]);
0401         table->bt_l1[index->l1] = NULL;
0402     }
0403 
0404     if (index->inited & HEM_INDEX_L0) {
0405         dma_free_coherent(dev, bt_size, table->bt_l0[index->l0],
0406                   table->bt_l0_dma_addr[index->l0]);
0407         table->bt_l0[index->l0] = NULL;
0408     }
0409 }
0410 
0411 static int alloc_mhop_hem(struct hns_roce_dev *hr_dev,
0412               struct hns_roce_hem_table *table,
0413               struct hns_roce_hem_mhop *mhop,
0414               struct hns_roce_hem_index *index)
0415 {
0416     u32 bt_size = mhop->bt_chunk_size;
0417     struct device *dev = hr_dev->dev;
0418     struct hns_roce_hem_iter iter;
0419     gfp_t flag;
0420     u64 bt_ba;
0421     u32 size;
0422     int ret;
0423 
0424     /* alloc L1 BA's chunk */
0425     if ((check_whether_bt_num_3(table->type, mhop->hop_num) ||
0426          check_whether_bt_num_2(table->type, mhop->hop_num)) &&
0427          !table->bt_l0[index->l0]) {
0428         table->bt_l0[index->l0] = dma_alloc_coherent(dev, bt_size,
0429                         &table->bt_l0_dma_addr[index->l0],
0430                         GFP_KERNEL);
0431         if (!table->bt_l0[index->l0]) {
0432             ret = -ENOMEM;
0433             goto out;
0434         }
0435         index->inited |= HEM_INDEX_L0;
0436     }
0437 
0438     /* alloc L2 BA's chunk */
0439     if (check_whether_bt_num_3(table->type, mhop->hop_num) &&
0440         !table->bt_l1[index->l1])  {
0441         table->bt_l1[index->l1] = dma_alloc_coherent(dev, bt_size,
0442                         &table->bt_l1_dma_addr[index->l1],
0443                         GFP_KERNEL);
0444         if (!table->bt_l1[index->l1]) {
0445             ret = -ENOMEM;
0446             goto err_alloc_hem;
0447         }
0448         index->inited |= HEM_INDEX_L1;
0449         *(table->bt_l0[index->l0] + mhop->l1_idx) =
0450                            table->bt_l1_dma_addr[index->l1];
0451     }
0452 
0453     /*
0454      * alloc buffer space chunk for QPC/MTPT/CQC/SRQC/SCCC.
0455      * alloc bt space chunk for MTT/CQE.
0456      */
0457     size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : bt_size;
0458     flag = (table->lowmem ? GFP_KERNEL : GFP_HIGHUSER) | __GFP_NOWARN;
0459     table->hem[index->buf] = hns_roce_alloc_hem(hr_dev, size >> PAGE_SHIFT,
0460                             size, flag);
0461     if (!table->hem[index->buf]) {
0462         ret = -ENOMEM;
0463         goto err_alloc_hem;
0464     }
0465 
0466     index->inited |= HEM_INDEX_BUF;
0467     hns_roce_hem_first(table->hem[index->buf], &iter);
0468     bt_ba = hns_roce_hem_addr(&iter);
0469     if (table->type < HEM_TYPE_MTT) {
0470         if (mhop->hop_num == 2)
0471             *(table->bt_l1[index->l1] + mhop->l2_idx) = bt_ba;
0472         else if (mhop->hop_num == 1)
0473             *(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba;
0474     } else if (mhop->hop_num == 2) {
0475         *(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba;
0476     }
0477 
0478     return 0;
0479 err_alloc_hem:
0480     free_mhop_hem(hr_dev, table, mhop, index);
0481 out:
0482     return ret;
0483 }
0484 
0485 static int set_mhop_hem(struct hns_roce_dev *hr_dev,
0486             struct hns_roce_hem_table *table, unsigned long obj,
0487             struct hns_roce_hem_mhop *mhop,
0488             struct hns_roce_hem_index *index)
0489 {
0490     struct ib_device *ibdev = &hr_dev->ib_dev;
0491     u32 step_idx;
0492     int ret = 0;
0493 
0494     if (index->inited & HEM_INDEX_L0) {
0495         ret = hr_dev->hw->set_hem(hr_dev, table, obj, 0);
0496         if (ret) {
0497             ibdev_err(ibdev, "set HEM step 0 failed!\n");
0498             goto out;
0499         }
0500     }
0501 
0502     if (index->inited & HEM_INDEX_L1) {
0503         ret = hr_dev->hw->set_hem(hr_dev, table, obj, 1);
0504         if (ret) {
0505             ibdev_err(ibdev, "set HEM step 1 failed!\n");
0506             goto out;
0507         }
0508     }
0509 
0510     if (index->inited & HEM_INDEX_BUF) {
0511         if (mhop->hop_num == HNS_ROCE_HOP_NUM_0)
0512             step_idx = 0;
0513         else
0514             step_idx = mhop->hop_num;
0515         ret = hr_dev->hw->set_hem(hr_dev, table, obj, step_idx);
0516         if (ret)
0517             ibdev_err(ibdev, "set HEM step last failed!\n");
0518     }
0519 out:
0520     return ret;
0521 }
0522 
0523 static int hns_roce_table_mhop_get(struct hns_roce_dev *hr_dev,
0524                    struct hns_roce_hem_table *table,
0525                    unsigned long obj)
0526 {
0527     struct ib_device *ibdev = &hr_dev->ib_dev;
0528     struct hns_roce_hem_index index = {};
0529     struct hns_roce_hem_mhop mhop = {};
0530     int ret;
0531 
0532     ret = calc_hem_config(hr_dev, table, obj, &mhop, &index);
0533     if (ret) {
0534         ibdev_err(ibdev, "calc hem config failed!\n");
0535         return ret;
0536     }
0537 
0538     mutex_lock(&table->mutex);
0539     if (table->hem[index.buf]) {
0540         refcount_inc(&table->hem[index.buf]->refcount);
0541         goto out;
0542     }
0543 
0544     ret = alloc_mhop_hem(hr_dev, table, &mhop, &index);
0545     if (ret) {
0546         ibdev_err(ibdev, "alloc mhop hem failed!\n");
0547         goto out;
0548     }
0549 
0550     /* set HEM base address to hardware */
0551     if (table->type < HEM_TYPE_MTT) {
0552         ret = set_mhop_hem(hr_dev, table, obj, &mhop, &index);
0553         if (ret) {
0554             ibdev_err(ibdev, "set HEM address to HW failed!\n");
0555             goto err_alloc;
0556         }
0557     }
0558 
0559     refcount_set(&table->hem[index.buf]->refcount, 1);
0560     goto out;
0561 
0562 err_alloc:
0563     free_mhop_hem(hr_dev, table, &mhop, &index);
0564 out:
0565     mutex_unlock(&table->mutex);
0566     return ret;
0567 }
0568 
0569 int hns_roce_table_get(struct hns_roce_dev *hr_dev,
0570                struct hns_roce_hem_table *table, unsigned long obj)
0571 {
0572     struct device *dev = hr_dev->dev;
0573     unsigned long i;
0574     int ret = 0;
0575 
0576     if (hns_roce_check_whether_mhop(hr_dev, table->type))
0577         return hns_roce_table_mhop_get(hr_dev, table, obj);
0578 
0579     i = obj / (table->table_chunk_size / table->obj_size);
0580 
0581     mutex_lock(&table->mutex);
0582 
0583     if (table->hem[i]) {
0584         refcount_inc(&table->hem[i]->refcount);
0585         goto out;
0586     }
0587 
0588     table->hem[i] = hns_roce_alloc_hem(hr_dev,
0589                        table->table_chunk_size >> PAGE_SHIFT,
0590                        table->table_chunk_size,
0591                        (table->lowmem ? GFP_KERNEL :
0592                     GFP_HIGHUSER) | __GFP_NOWARN);
0593     if (!table->hem[i]) {
0594         ret = -ENOMEM;
0595         goto out;
0596     }
0597 
0598     /* Set HEM base address(128K/page, pa) to Hardware */
0599     if (hr_dev->hw->set_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT)) {
0600         hns_roce_free_hem(hr_dev, table->hem[i]);
0601         table->hem[i] = NULL;
0602         ret = -ENODEV;
0603         dev_err(dev, "set HEM base address to HW failed.\n");
0604         goto out;
0605     }
0606 
0607     refcount_set(&table->hem[i]->refcount, 1);
0608 out:
0609     mutex_unlock(&table->mutex);
0610     return ret;
0611 }
0612 
0613 static void clear_mhop_hem(struct hns_roce_dev *hr_dev,
0614                struct hns_roce_hem_table *table, unsigned long obj,
0615                struct hns_roce_hem_mhop *mhop,
0616                struct hns_roce_hem_index *index)
0617 {
0618     struct ib_device *ibdev = &hr_dev->ib_dev;
0619     u32 hop_num = mhop->hop_num;
0620     u32 chunk_ba_num;
0621     u32 step_idx;
0622 
0623     index->inited = HEM_INDEX_BUF;
0624     chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN;
0625     if (check_whether_bt_num_2(table->type, hop_num)) {
0626         if (hns_roce_check_hem_null(table->hem, index->buf,
0627                         chunk_ba_num, table->num_hem))
0628             index->inited |= HEM_INDEX_L0;
0629     } else if (check_whether_bt_num_3(table->type, hop_num)) {
0630         if (hns_roce_check_hem_null(table->hem, index->buf,
0631                         chunk_ba_num, table->num_hem)) {
0632             index->inited |= HEM_INDEX_L1;
0633             if (hns_roce_check_bt_null(table->bt_l1, index->l1,
0634                            chunk_ba_num))
0635                 index->inited |= HEM_INDEX_L0;
0636         }
0637     }
0638 
0639     if (table->type < HEM_TYPE_MTT) {
0640         if (hop_num == HNS_ROCE_HOP_NUM_0)
0641             step_idx = 0;
0642         else
0643             step_idx = hop_num;
0644 
0645         if (hr_dev->hw->clear_hem(hr_dev, table, obj, step_idx))
0646             ibdev_warn(ibdev, "failed to clear hop%u HEM.\n", hop_num);
0647 
0648         if (index->inited & HEM_INDEX_L1)
0649             if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1))
0650                 ibdev_warn(ibdev, "failed to clear HEM step 1.\n");
0651 
0652         if (index->inited & HEM_INDEX_L0)
0653             if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0))
0654                 ibdev_warn(ibdev, "failed to clear HEM step 0.\n");
0655     }
0656 }
0657 
0658 static void hns_roce_table_mhop_put(struct hns_roce_dev *hr_dev,
0659                     struct hns_roce_hem_table *table,
0660                     unsigned long obj,
0661                     int check_refcount)
0662 {
0663     struct ib_device *ibdev = &hr_dev->ib_dev;
0664     struct hns_roce_hem_index index = {};
0665     struct hns_roce_hem_mhop mhop = {};
0666     int ret;
0667 
0668     ret = calc_hem_config(hr_dev, table, obj, &mhop, &index);
0669     if (ret) {
0670         ibdev_err(ibdev, "calc hem config failed!\n");
0671         return;
0672     }
0673 
0674     if (!check_refcount)
0675         mutex_lock(&table->mutex);
0676     else if (!refcount_dec_and_mutex_lock(&table->hem[index.buf]->refcount,
0677                           &table->mutex))
0678         return;
0679 
0680     clear_mhop_hem(hr_dev, table, obj, &mhop, &index);
0681     free_mhop_hem(hr_dev, table, &mhop, &index);
0682 
0683     mutex_unlock(&table->mutex);
0684 }
0685 
0686 void hns_roce_table_put(struct hns_roce_dev *hr_dev,
0687             struct hns_roce_hem_table *table, unsigned long obj)
0688 {
0689     struct device *dev = hr_dev->dev;
0690     unsigned long i;
0691 
0692     if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
0693         hns_roce_table_mhop_put(hr_dev, table, obj, 1);
0694         return;
0695     }
0696 
0697     i = obj / (table->table_chunk_size / table->obj_size);
0698 
0699     if (!refcount_dec_and_mutex_lock(&table->hem[i]->refcount,
0700                      &table->mutex))
0701         return;
0702 
0703     if (hr_dev->hw->clear_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT))
0704         dev_warn(dev, "failed to clear HEM base address.\n");
0705 
0706     hns_roce_free_hem(hr_dev, table->hem[i]);
0707     table->hem[i] = NULL;
0708 
0709     mutex_unlock(&table->mutex);
0710 }
0711 
0712 void *hns_roce_table_find(struct hns_roce_dev *hr_dev,
0713               struct hns_roce_hem_table *table,
0714               unsigned long obj, dma_addr_t *dma_handle)
0715 {
0716     struct hns_roce_hem_chunk *chunk;
0717     struct hns_roce_hem_mhop mhop;
0718     struct hns_roce_hem *hem;
0719     unsigned long mhop_obj = obj;
0720     unsigned long obj_per_chunk;
0721     unsigned long idx_offset;
0722     int offset, dma_offset;
0723     void *addr = NULL;
0724     u32 hem_idx = 0;
0725     int length;
0726     int i, j;
0727 
0728     if (!table->lowmem)
0729         return NULL;
0730 
0731     mutex_lock(&table->mutex);
0732 
0733     if (!hns_roce_check_whether_mhop(hr_dev, table->type)) {
0734         obj_per_chunk = table->table_chunk_size / table->obj_size;
0735         hem = table->hem[obj / obj_per_chunk];
0736         idx_offset = obj % obj_per_chunk;
0737         dma_offset = offset = idx_offset * table->obj_size;
0738     } else {
0739         u32 seg_size = 64; /* 8 bytes per BA and 8 BA per segment */
0740 
0741         if (hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop))
0742             goto out;
0743         /* mtt mhop */
0744         i = mhop.l0_idx;
0745         j = mhop.l1_idx;
0746         if (mhop.hop_num == 2)
0747             hem_idx = i * (mhop.bt_chunk_size / BA_BYTE_LEN) + j;
0748         else if (mhop.hop_num == 1 ||
0749              mhop.hop_num == HNS_ROCE_HOP_NUM_0)
0750             hem_idx = i;
0751 
0752         hem = table->hem[hem_idx];
0753         dma_offset = offset = obj * seg_size % mhop.bt_chunk_size;
0754         if (mhop.hop_num == 2)
0755             dma_offset = offset = 0;
0756     }
0757 
0758     if (!hem)
0759         goto out;
0760 
0761     list_for_each_entry(chunk, &hem->chunk_list, list) {
0762         for (i = 0; i < chunk->npages; ++i) {
0763             length = sg_dma_len(&chunk->mem[i]);
0764             if (dma_handle && dma_offset >= 0) {
0765                 if (length > (u32)dma_offset)
0766                     *dma_handle = sg_dma_address(
0767                         &chunk->mem[i]) + dma_offset;
0768                 dma_offset -= length;
0769             }
0770 
0771             if (length > (u32)offset) {
0772                 addr = chunk->buf[i] + offset;
0773                 goto out;
0774             }
0775             offset -= length;
0776         }
0777     }
0778 
0779 out:
0780     mutex_unlock(&table->mutex);
0781     return addr;
0782 }
0783 
0784 int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev,
0785                 struct hns_roce_hem_table *table, u32 type,
0786                 unsigned long obj_size, unsigned long nobj,
0787                 int use_lowmem)
0788 {
0789     unsigned long obj_per_chunk;
0790     unsigned long num_hem;
0791 
0792     if (!hns_roce_check_whether_mhop(hr_dev, type)) {
0793         table->table_chunk_size = hr_dev->caps.chunk_sz;
0794         obj_per_chunk = table->table_chunk_size / obj_size;
0795         num_hem = DIV_ROUND_UP(nobj, obj_per_chunk);
0796 
0797         table->hem = kcalloc(num_hem, sizeof(*table->hem), GFP_KERNEL);
0798         if (!table->hem)
0799             return -ENOMEM;
0800     } else {
0801         struct hns_roce_hem_mhop mhop = {};
0802         unsigned long buf_chunk_size;
0803         unsigned long bt_chunk_size;
0804         unsigned long bt_chunk_num;
0805         unsigned long num_bt_l0;
0806         u32 hop_num;
0807 
0808         if (get_hem_table_config(hr_dev, &mhop, type))
0809             return -EINVAL;
0810 
0811         buf_chunk_size = mhop.buf_chunk_size;
0812         bt_chunk_size = mhop.bt_chunk_size;
0813         num_bt_l0 = mhop.ba_l0_num;
0814         hop_num = mhop.hop_num;
0815 
0816         obj_per_chunk = buf_chunk_size / obj_size;
0817         num_hem = DIV_ROUND_UP(nobj, obj_per_chunk);
0818         bt_chunk_num = bt_chunk_size / BA_BYTE_LEN;
0819 
0820         if (type >= HEM_TYPE_MTT)
0821             num_bt_l0 = bt_chunk_num;
0822 
0823         table->hem = kcalloc(num_hem, sizeof(*table->hem),
0824                      GFP_KERNEL);
0825         if (!table->hem)
0826             goto err_kcalloc_hem_buf;
0827 
0828         if (check_whether_bt_num_3(type, hop_num)) {
0829             unsigned long num_bt_l1;
0830 
0831             num_bt_l1 = DIV_ROUND_UP(num_hem, bt_chunk_num);
0832             table->bt_l1 = kcalloc(num_bt_l1,
0833                            sizeof(*table->bt_l1),
0834                            GFP_KERNEL);
0835             if (!table->bt_l1)
0836                 goto err_kcalloc_bt_l1;
0837 
0838             table->bt_l1_dma_addr = kcalloc(num_bt_l1,
0839                          sizeof(*table->bt_l1_dma_addr),
0840                          GFP_KERNEL);
0841 
0842             if (!table->bt_l1_dma_addr)
0843                 goto err_kcalloc_l1_dma;
0844         }
0845 
0846         if (check_whether_bt_num_2(type, hop_num) ||
0847             check_whether_bt_num_3(type, hop_num)) {
0848             table->bt_l0 = kcalloc(num_bt_l0, sizeof(*table->bt_l0),
0849                            GFP_KERNEL);
0850             if (!table->bt_l0)
0851                 goto err_kcalloc_bt_l0;
0852 
0853             table->bt_l0_dma_addr = kcalloc(num_bt_l0,
0854                          sizeof(*table->bt_l0_dma_addr),
0855                          GFP_KERNEL);
0856             if (!table->bt_l0_dma_addr)
0857                 goto err_kcalloc_l0_dma;
0858         }
0859     }
0860 
0861     table->type = type;
0862     table->num_hem = num_hem;
0863     table->obj_size = obj_size;
0864     table->lowmem = use_lowmem;
0865     mutex_init(&table->mutex);
0866 
0867     return 0;
0868 
0869 err_kcalloc_l0_dma:
0870     kfree(table->bt_l0);
0871     table->bt_l0 = NULL;
0872 
0873 err_kcalloc_bt_l0:
0874     kfree(table->bt_l1_dma_addr);
0875     table->bt_l1_dma_addr = NULL;
0876 
0877 err_kcalloc_l1_dma:
0878     kfree(table->bt_l1);
0879     table->bt_l1 = NULL;
0880 
0881 err_kcalloc_bt_l1:
0882     kfree(table->hem);
0883     table->hem = NULL;
0884 
0885 err_kcalloc_hem_buf:
0886     return -ENOMEM;
0887 }
0888 
0889 static void hns_roce_cleanup_mhop_hem_table(struct hns_roce_dev *hr_dev,
0890                         struct hns_roce_hem_table *table)
0891 {
0892     struct hns_roce_hem_mhop mhop;
0893     u32 buf_chunk_size;
0894     u64 obj;
0895     int i;
0896 
0897     if (hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop))
0898         return;
0899     buf_chunk_size = table->type < HEM_TYPE_MTT ? mhop.buf_chunk_size :
0900                     mhop.bt_chunk_size;
0901 
0902     for (i = 0; i < table->num_hem; ++i) {
0903         obj = i * buf_chunk_size / table->obj_size;
0904         if (table->hem[i])
0905             hns_roce_table_mhop_put(hr_dev, table, obj, 0);
0906     }
0907 
0908     kfree(table->hem);
0909     table->hem = NULL;
0910     kfree(table->bt_l1);
0911     table->bt_l1 = NULL;
0912     kfree(table->bt_l1_dma_addr);
0913     table->bt_l1_dma_addr = NULL;
0914     kfree(table->bt_l0);
0915     table->bt_l0 = NULL;
0916     kfree(table->bt_l0_dma_addr);
0917     table->bt_l0_dma_addr = NULL;
0918 }
0919 
0920 void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev,
0921                 struct hns_roce_hem_table *table)
0922 {
0923     struct device *dev = hr_dev->dev;
0924     unsigned long i;
0925 
0926     if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
0927         hns_roce_cleanup_mhop_hem_table(hr_dev, table);
0928         return;
0929     }
0930 
0931     for (i = 0; i < table->num_hem; ++i)
0932         if (table->hem[i]) {
0933             if (hr_dev->hw->clear_hem(hr_dev, table,
0934                 i * table->table_chunk_size / table->obj_size, 0))
0935                 dev_err(dev, "Clear HEM base address failed.\n");
0936 
0937             hns_roce_free_hem(hr_dev, table->hem[i]);
0938         }
0939 
0940     kfree(table->hem);
0941 }
0942 
0943 void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev)
0944 {
0945     if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ)
0946         hns_roce_cleanup_hem_table(hr_dev,
0947                        &hr_dev->srq_table.table);
0948     hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table);
0949     if (hr_dev->caps.qpc_timer_entry_sz)
0950         hns_roce_cleanup_hem_table(hr_dev,
0951                        &hr_dev->qpc_timer_table);
0952     if (hr_dev->caps.cqc_timer_entry_sz)
0953         hns_roce_cleanup_hem_table(hr_dev,
0954                        &hr_dev->cqc_timer_table);
0955     if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL)
0956         hns_roce_cleanup_hem_table(hr_dev,
0957                        &hr_dev->qp_table.sccc_table);
0958     if (hr_dev->caps.trrl_entry_sz)
0959         hns_roce_cleanup_hem_table(hr_dev,
0960                        &hr_dev->qp_table.trrl_table);
0961 
0962     if (hr_dev->caps.gmv_entry_sz)
0963         hns_roce_cleanup_hem_table(hr_dev, &hr_dev->gmv_table);
0964 
0965     hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table);
0966     hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.qp_table);
0967     hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table);
0968 }
0969 
0970 struct hns_roce_hem_item {
0971     struct list_head list; /* link all hems in the same bt level */
0972     struct list_head sibling; /* link all hems in last hop for mtt */
0973     void *addr;
0974     dma_addr_t dma_addr;
0975     size_t count; /* max ba numbers */
0976     int start; /* start buf offset in this hem */
0977     int end; /* end buf offset in this hem */
0978 };
0979 
0980 /* All HEM items are linked in a tree structure */
0981 struct hns_roce_hem_head {
0982     struct list_head branch[HNS_ROCE_MAX_BT_REGION];
0983     struct list_head root;
0984     struct list_head leaf;
0985 };
0986 
0987 static struct hns_roce_hem_item *
0988 hem_list_alloc_item(struct hns_roce_dev *hr_dev, int start, int end, int count,
0989             bool exist_bt, int bt_level)
0990 {
0991     struct hns_roce_hem_item *hem;
0992 
0993     hem = kzalloc(sizeof(*hem), GFP_KERNEL);
0994     if (!hem)
0995         return NULL;
0996 
0997     if (exist_bt) {
0998         hem->addr = dma_alloc_coherent(hr_dev->dev, count * BA_BYTE_LEN,
0999                            &hem->dma_addr, GFP_KERNEL);
1000         if (!hem->addr) {
1001             kfree(hem);
1002             return NULL;
1003         }
1004     }
1005 
1006     hem->count = count;
1007     hem->start = start;
1008     hem->end = end;
1009     INIT_LIST_HEAD(&hem->list);
1010     INIT_LIST_HEAD(&hem->sibling);
1011 
1012     return hem;
1013 }
1014 
1015 static void hem_list_free_item(struct hns_roce_dev *hr_dev,
1016                    struct hns_roce_hem_item *hem, bool exist_bt)
1017 {
1018     if (exist_bt)
1019         dma_free_coherent(hr_dev->dev, hem->count * BA_BYTE_LEN,
1020                   hem->addr, hem->dma_addr);
1021     kfree(hem);
1022 }
1023 
1024 static void hem_list_free_all(struct hns_roce_dev *hr_dev,
1025                   struct list_head *head, bool exist_bt)
1026 {
1027     struct hns_roce_hem_item *hem, *temp_hem;
1028 
1029     list_for_each_entry_safe(hem, temp_hem, head, list) {
1030         list_del(&hem->list);
1031         hem_list_free_item(hr_dev, hem, exist_bt);
1032     }
1033 }
1034 
1035 static void hem_list_link_bt(struct hns_roce_dev *hr_dev, void *base_addr,
1036                  u64 table_addr)
1037 {
1038     *(u64 *)(base_addr) = table_addr;
1039 }
1040 
1041 /* assign L0 table address to hem from root bt */
1042 static void hem_list_assign_bt(struct hns_roce_dev *hr_dev,
1043                    struct hns_roce_hem_item *hem, void *cpu_addr,
1044                    u64 phy_addr)
1045 {
1046     hem->addr = cpu_addr;
1047     hem->dma_addr = (dma_addr_t)phy_addr;
1048 }
1049 
1050 static inline bool hem_list_page_is_in_range(struct hns_roce_hem_item *hem,
1051                          int offset)
1052 {
1053     return (hem->start <= offset && offset <= hem->end);
1054 }
1055 
1056 static struct hns_roce_hem_item *hem_list_search_item(struct list_head *ba_list,
1057                               int page_offset)
1058 {
1059     struct hns_roce_hem_item *hem, *temp_hem;
1060     struct hns_roce_hem_item *found = NULL;
1061 
1062     list_for_each_entry_safe(hem, temp_hem, ba_list, list) {
1063         if (hem_list_page_is_in_range(hem, page_offset)) {
1064             found = hem;
1065             break;
1066         }
1067     }
1068 
1069     return found;
1070 }
1071 
1072 static bool hem_list_is_bottom_bt(int hopnum, int bt_level)
1073 {
1074     /*
1075      * hopnum    base address table levels
1076      * 0        L0(buf)
1077      * 1        L0 -> buf
1078      * 2        L0 -> L1 -> buf
1079      * 3        L0 -> L1 -> L2 -> buf
1080      */
1081     return bt_level >= (hopnum ? hopnum - 1 : hopnum);
1082 }
1083 
1084 /*
1085  * calc base address entries num
1086  * @hopnum: num of mutihop addressing
1087  * @bt_level: base address table level
1088  * @unit: ba entries per bt page
1089  */
1090 static u32 hem_list_calc_ba_range(int hopnum, int bt_level, int unit)
1091 {
1092     u32 step;
1093     int max;
1094     int i;
1095 
1096     if (hopnum <= bt_level)
1097         return 0;
1098     /*
1099      * hopnum  bt_level   range
1100      * 1          0       unit
1101      * ------------
1102      * 2          0       unit * unit
1103      * 2          1       unit
1104      * ------------
1105      * 3          0       unit * unit * unit
1106      * 3          1       unit * unit
1107      * 3          2       unit
1108      */
1109     step = 1;
1110     max = hopnum - bt_level;
1111     for (i = 0; i < max; i++)
1112         step = step * unit;
1113 
1114     return step;
1115 }
1116 
1117 /*
1118  * calc the root ba entries which could cover all regions
1119  * @regions: buf region array
1120  * @region_cnt: array size of @regions
1121  * @unit: ba entries per bt page
1122  */
1123 int hns_roce_hem_list_calc_root_ba(const struct hns_roce_buf_region *regions,
1124                    int region_cnt, int unit)
1125 {
1126     struct hns_roce_buf_region *r;
1127     int total = 0;
1128     int step;
1129     int i;
1130 
1131     for (i = 0; i < region_cnt; i++) {
1132         r = (struct hns_roce_buf_region *)&regions[i];
1133         if (r->hopnum > 1) {
1134             step = hem_list_calc_ba_range(r->hopnum, 1, unit);
1135             if (step > 0)
1136                 total += (r->count + step - 1) / step;
1137         } else {
1138             total += r->count;
1139         }
1140     }
1141 
1142     return total;
1143 }
1144 
1145 static int hem_list_alloc_mid_bt(struct hns_roce_dev *hr_dev,
1146                  const struct hns_roce_buf_region *r, int unit,
1147                  int offset, struct list_head *mid_bt,
1148                  struct list_head *btm_bt)
1149 {
1150     struct hns_roce_hem_item *hem_ptrs[HNS_ROCE_MAX_BT_LEVEL] = { NULL };
1151     struct list_head temp_list[HNS_ROCE_MAX_BT_LEVEL];
1152     struct hns_roce_hem_item *cur, *pre;
1153     const int hopnum = r->hopnum;
1154     int start_aligned;
1155     int distance;
1156     int ret = 0;
1157     int max_ofs;
1158     int level;
1159     u32 step;
1160     int end;
1161 
1162     if (hopnum <= 1)
1163         return 0;
1164 
1165     if (hopnum > HNS_ROCE_MAX_BT_LEVEL) {
1166         dev_err(hr_dev->dev, "invalid hopnum %d!\n", hopnum);
1167         return -EINVAL;
1168     }
1169 
1170     if (offset < r->offset) {
1171         dev_err(hr_dev->dev, "invalid offset %d, min %u!\n",
1172             offset, r->offset);
1173         return -EINVAL;
1174     }
1175 
1176     distance = offset - r->offset;
1177     max_ofs = r->offset + r->count - 1;
1178     for (level = 0; level < hopnum; level++)
1179         INIT_LIST_HEAD(&temp_list[level]);
1180 
1181     /* config L1 bt to last bt and link them to corresponding parent */
1182     for (level = 1; level < hopnum; level++) {
1183         cur = hem_list_search_item(&mid_bt[level], offset);
1184         if (cur) {
1185             hem_ptrs[level] = cur;
1186             continue;
1187         }
1188 
1189         step = hem_list_calc_ba_range(hopnum, level, unit);
1190         if (step < 1) {
1191             ret = -EINVAL;
1192             goto err_exit;
1193         }
1194 
1195         start_aligned = (distance / step) * step + r->offset;
1196         end = min_t(int, start_aligned + step - 1, max_ofs);
1197         cur = hem_list_alloc_item(hr_dev, start_aligned, end, unit,
1198                       true, level);
1199         if (!cur) {
1200             ret = -ENOMEM;
1201             goto err_exit;
1202         }
1203         hem_ptrs[level] = cur;
1204         list_add(&cur->list, &temp_list[level]);
1205         if (hem_list_is_bottom_bt(hopnum, level))
1206             list_add(&cur->sibling, &temp_list[0]);
1207 
1208         /* link bt to parent bt */
1209         if (level > 1) {
1210             pre = hem_ptrs[level - 1];
1211             step = (cur->start - pre->start) / step * BA_BYTE_LEN;
1212             hem_list_link_bt(hr_dev, pre->addr + step,
1213                      cur->dma_addr);
1214         }
1215     }
1216 
1217     list_splice(&temp_list[0], btm_bt);
1218     for (level = 1; level < hopnum; level++)
1219         list_splice(&temp_list[level], &mid_bt[level]);
1220 
1221     return 0;
1222 
1223 err_exit:
1224     for (level = 1; level < hopnum; level++)
1225         hem_list_free_all(hr_dev, &temp_list[level], true);
1226 
1227     return ret;
1228 }
1229 
1230 static struct hns_roce_hem_item *
1231 alloc_root_hem(struct hns_roce_dev *hr_dev, int unit, int *max_ba_num,
1232            const struct hns_roce_buf_region *regions, int region_cnt)
1233 {
1234     const struct hns_roce_buf_region *r;
1235     struct hns_roce_hem_item *hem;
1236     int ba_num;
1237     int offset;
1238 
1239     ba_num = hns_roce_hem_list_calc_root_ba(regions, region_cnt, unit);
1240     if (ba_num < 1)
1241         return ERR_PTR(-ENOMEM);
1242 
1243     if (ba_num > unit)
1244         return ERR_PTR(-ENOBUFS);
1245 
1246     offset = regions[0].offset;
1247     /* indicate to last region */
1248     r = &regions[region_cnt - 1];
1249     hem = hem_list_alloc_item(hr_dev, offset, r->offset + r->count - 1,
1250                   ba_num, true, 0);
1251     if (!hem)
1252         return ERR_PTR(-ENOMEM);
1253 
1254     *max_ba_num = ba_num;
1255 
1256     return hem;
1257 }
1258 
1259 static int alloc_fake_root_bt(struct hns_roce_dev *hr_dev, void *cpu_base,
1260                   u64 phy_base, const struct hns_roce_buf_region *r,
1261                   struct list_head *branch_head,
1262                   struct list_head *leaf_head)
1263 {
1264     struct hns_roce_hem_item *hem;
1265 
1266     hem = hem_list_alloc_item(hr_dev, r->offset, r->offset + r->count - 1,
1267                   r->count, false, 0);
1268     if (!hem)
1269         return -ENOMEM;
1270 
1271     hem_list_assign_bt(hr_dev, hem, cpu_base, phy_base);
1272     list_add(&hem->list, branch_head);
1273     list_add(&hem->sibling, leaf_head);
1274 
1275     return r->count;
1276 }
1277 
1278 static int setup_middle_bt(struct hns_roce_dev *hr_dev, void *cpu_base,
1279                int unit, const struct hns_roce_buf_region *r,
1280                const struct list_head *branch_head)
1281 {
1282     struct hns_roce_hem_item *hem, *temp_hem;
1283     int total = 0;
1284     int offset;
1285     int step;
1286 
1287     step = hem_list_calc_ba_range(r->hopnum, 1, unit);
1288     if (step < 1)
1289         return -EINVAL;
1290 
1291     /* if exist mid bt, link L1 to L0 */
1292     list_for_each_entry_safe(hem, temp_hem, branch_head, list) {
1293         offset = (hem->start - r->offset) / step * BA_BYTE_LEN;
1294         hem_list_link_bt(hr_dev, cpu_base + offset, hem->dma_addr);
1295         total++;
1296     }
1297 
1298     return total;
1299 }
1300 
1301 static int
1302 setup_root_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem_list *hem_list,
1303            int unit, int max_ba_num, struct hns_roce_hem_head *head,
1304            const struct hns_roce_buf_region *regions, int region_cnt)
1305 {
1306     const struct hns_roce_buf_region *r;
1307     struct hns_roce_hem_item *root_hem;
1308     void *cpu_base;
1309     u64 phy_base;
1310     int i, total;
1311     int ret;
1312 
1313     root_hem = list_first_entry(&head->root,
1314                     struct hns_roce_hem_item, list);
1315     if (!root_hem)
1316         return -ENOMEM;
1317 
1318     total = 0;
1319     for (i = 0; i < region_cnt && total < max_ba_num; i++) {
1320         r = &regions[i];
1321         if (!r->count)
1322             continue;
1323 
1324         /* all regions's mid[x][0] shared the root_bt's trunk */
1325         cpu_base = root_hem->addr + total * BA_BYTE_LEN;
1326         phy_base = root_hem->dma_addr + total * BA_BYTE_LEN;
1327 
1328         /* if hopnum is 0 or 1, cut a new fake hem from the root bt
1329          * which's address share to all regions.
1330          */
1331         if (hem_list_is_bottom_bt(r->hopnum, 0))
1332             ret = alloc_fake_root_bt(hr_dev, cpu_base, phy_base, r,
1333                          &head->branch[i], &head->leaf);
1334         else
1335             ret = setup_middle_bt(hr_dev, cpu_base, unit, r,
1336                           &hem_list->mid_bt[i][1]);
1337 
1338         if (ret < 0)
1339             return ret;
1340 
1341         total += ret;
1342     }
1343 
1344     list_splice(&head->leaf, &hem_list->btm_bt);
1345     list_splice(&head->root, &hem_list->root_bt);
1346     for (i = 0; i < region_cnt; i++)
1347         list_splice(&head->branch[i], &hem_list->mid_bt[i][0]);
1348 
1349     return 0;
1350 }
1351 
1352 static int hem_list_alloc_root_bt(struct hns_roce_dev *hr_dev,
1353                   struct hns_roce_hem_list *hem_list, int unit,
1354                   const struct hns_roce_buf_region *regions,
1355                   int region_cnt)
1356 {
1357     struct hns_roce_hem_item *root_hem;
1358     struct hns_roce_hem_head head;
1359     int max_ba_num;
1360     int ret;
1361     int i;
1362 
1363     root_hem = hem_list_search_item(&hem_list->root_bt, regions[0].offset);
1364     if (root_hem)
1365         return 0;
1366 
1367     max_ba_num = 0;
1368     root_hem = alloc_root_hem(hr_dev, unit, &max_ba_num, regions,
1369                   region_cnt);
1370     if (IS_ERR(root_hem))
1371         return PTR_ERR(root_hem);
1372 
1373     /* List head for storing all allocated HEM items */
1374     INIT_LIST_HEAD(&head.root);
1375     INIT_LIST_HEAD(&head.leaf);
1376     for (i = 0; i < region_cnt; i++)
1377         INIT_LIST_HEAD(&head.branch[i]);
1378 
1379     hem_list->root_ba = root_hem->dma_addr;
1380     list_add(&root_hem->list, &head.root);
1381     ret = setup_root_hem(hr_dev, hem_list, unit, max_ba_num, &head, regions,
1382                  region_cnt);
1383     if (ret) {
1384         for (i = 0; i < region_cnt; i++)
1385             hem_list_free_all(hr_dev, &head.branch[i], false);
1386 
1387         hem_list_free_all(hr_dev, &head.root, true);
1388     }
1389 
1390     return ret;
1391 }
1392 
1393 /* construct the base address table and link them by address hop config */
1394 int hns_roce_hem_list_request(struct hns_roce_dev *hr_dev,
1395                   struct hns_roce_hem_list *hem_list,
1396                   const struct hns_roce_buf_region *regions,
1397                   int region_cnt, unsigned int bt_pg_shift)
1398 {
1399     const struct hns_roce_buf_region *r;
1400     int ofs, end;
1401     int unit;
1402     int ret;
1403     int i;
1404 
1405     if (region_cnt > HNS_ROCE_MAX_BT_REGION) {
1406         dev_err(hr_dev->dev, "invalid region region_cnt %d!\n",
1407             region_cnt);
1408         return -EINVAL;
1409     }
1410 
1411     unit = (1 << bt_pg_shift) / BA_BYTE_LEN;
1412     for (i = 0; i < region_cnt; i++) {
1413         r = &regions[i];
1414         if (!r->count)
1415             continue;
1416 
1417         end = r->offset + r->count;
1418         for (ofs = r->offset; ofs < end; ofs += unit) {
1419             ret = hem_list_alloc_mid_bt(hr_dev, r, unit, ofs,
1420                             hem_list->mid_bt[i],
1421                             &hem_list->btm_bt);
1422             if (ret) {
1423                 dev_err(hr_dev->dev,
1424                     "alloc hem trunk fail ret=%d!\n", ret);
1425                 goto err_alloc;
1426             }
1427         }
1428     }
1429 
1430     ret = hem_list_alloc_root_bt(hr_dev, hem_list, unit, regions,
1431                      region_cnt);
1432     if (ret)
1433         dev_err(hr_dev->dev, "alloc hem root fail ret=%d!\n", ret);
1434     else
1435         return 0;
1436 
1437 err_alloc:
1438     hns_roce_hem_list_release(hr_dev, hem_list);
1439 
1440     return ret;
1441 }
1442 
1443 void hns_roce_hem_list_release(struct hns_roce_dev *hr_dev,
1444                    struct hns_roce_hem_list *hem_list)
1445 {
1446     int i, j;
1447 
1448     for (i = 0; i < HNS_ROCE_MAX_BT_REGION; i++)
1449         for (j = 0; j < HNS_ROCE_MAX_BT_LEVEL; j++)
1450             hem_list_free_all(hr_dev, &hem_list->mid_bt[i][j],
1451                       j != 0);
1452 
1453     hem_list_free_all(hr_dev, &hem_list->root_bt, true);
1454     INIT_LIST_HEAD(&hem_list->btm_bt);
1455     hem_list->root_ba = 0;
1456 }
1457 
1458 void hns_roce_hem_list_init(struct hns_roce_hem_list *hem_list)
1459 {
1460     int i, j;
1461 
1462     INIT_LIST_HEAD(&hem_list->root_bt);
1463     INIT_LIST_HEAD(&hem_list->btm_bt);
1464     for (i = 0; i < HNS_ROCE_MAX_BT_REGION; i++)
1465         for (j = 0; j < HNS_ROCE_MAX_BT_LEVEL; j++)
1466             INIT_LIST_HEAD(&hem_list->mid_bt[i][j]);
1467 }
1468 
1469 void *hns_roce_hem_list_find_mtt(struct hns_roce_dev *hr_dev,
1470                  struct hns_roce_hem_list *hem_list,
1471                  int offset, int *mtt_cnt, u64 *phy_addr)
1472 {
1473     struct list_head *head = &hem_list->btm_bt;
1474     struct hns_roce_hem_item *hem, *temp_hem;
1475     void *cpu_base = NULL;
1476     u64 phy_base = 0;
1477     int nr = 0;
1478 
1479     list_for_each_entry_safe(hem, temp_hem, head, sibling) {
1480         if (hem_list_page_is_in_range(hem, offset)) {
1481             nr = offset - hem->start;
1482             cpu_base = hem->addr + nr * BA_BYTE_LEN;
1483             phy_base = hem->dma_addr + nr * BA_BYTE_LEN;
1484             nr = hem->end + 1 - offset;
1485             break;
1486         }
1487     }
1488 
1489     if (mtt_cnt)
1490         *mtt_cnt = nr;
1491 
1492     if (phy_addr)
1493         *phy_addr = phy_base;
1494 
1495     return cpu_base;
1496 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!