Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Copyright (c) 2021, HiSilicon Ltd.
0004  */
0005 
0006 #include <linux/device.h>
0007 #include <linux/eventfd.h>
0008 #include <linux/file.h>
0009 #include <linux/hisi_acc_qm.h>
0010 #include <linux/interrupt.h>
0011 #include <linux/module.h>
0012 #include <linux/pci.h>
0013 #include <linux/vfio.h>
0014 #include <linux/vfio_pci_core.h>
0015 #include <linux/anon_inodes.h>
0016 
0017 #include "hisi_acc_vfio_pci.h"
0018 
0019 /* return 0 on VM acc device ready, -ETIMEDOUT hardware timeout */
0020 static int qm_wait_dev_not_ready(struct hisi_qm *qm)
0021 {
0022     u32 val;
0023 
0024     return readl_relaxed_poll_timeout(qm->io_base + QM_VF_STATE,
0025                 val, !(val & 0x1), MB_POLL_PERIOD_US,
0026                 MB_POLL_TIMEOUT_US);
0027 }
0028 
0029 /*
0030  * Each state Reg is checked 100 times,
0031  * with a delay of 100 microseconds after each check
0032  */
0033 static u32 qm_check_reg_state(struct hisi_qm *qm, u32 regs)
0034 {
0035     int check_times = 0;
0036     u32 state;
0037 
0038     state = readl(qm->io_base + regs);
0039     while (state && check_times < ERROR_CHECK_TIMEOUT) {
0040         udelay(CHECK_DELAY_TIME);
0041         state = readl(qm->io_base + regs);
0042         check_times++;
0043     }
0044 
0045     return state;
0046 }
0047 
0048 static int qm_read_regs(struct hisi_qm *qm, u32 reg_addr,
0049             u32 *data, u8 nums)
0050 {
0051     int i;
0052 
0053     if (nums < 1 || nums > QM_REGS_MAX_LEN)
0054         return -EINVAL;
0055 
0056     for (i = 0; i < nums; i++) {
0057         data[i] = readl(qm->io_base + reg_addr);
0058         reg_addr += QM_REG_ADDR_OFFSET;
0059     }
0060 
0061     return 0;
0062 }
0063 
0064 static int qm_write_regs(struct hisi_qm *qm, u32 reg,
0065              u32 *data, u8 nums)
0066 {
0067     int i;
0068 
0069     if (nums < 1 || nums > QM_REGS_MAX_LEN)
0070         return -EINVAL;
0071 
0072     for (i = 0; i < nums; i++)
0073         writel(data[i], qm->io_base + reg + i * QM_REG_ADDR_OFFSET);
0074 
0075     return 0;
0076 }
0077 
0078 static int qm_get_vft(struct hisi_qm *qm, u32 *base)
0079 {
0080     u64 sqc_vft;
0081     u32 qp_num;
0082     int ret;
0083 
0084     ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_VFT_V2, 0, 0, 1);
0085     if (ret)
0086         return ret;
0087 
0088     sqc_vft = readl(qm->io_base + QM_MB_CMD_DATA_ADDR_L) |
0089           ((u64)readl(qm->io_base + QM_MB_CMD_DATA_ADDR_H) <<
0090           QM_XQC_ADDR_OFFSET);
0091     *base = QM_SQC_VFT_BASE_MASK_V2 & (sqc_vft >> QM_SQC_VFT_BASE_SHIFT_V2);
0092     qp_num = (QM_SQC_VFT_NUM_MASK_V2 &
0093           (sqc_vft >> QM_SQC_VFT_NUM_SHIFT_V2)) + 1;
0094 
0095     return qp_num;
0096 }
0097 
0098 static int qm_get_sqc(struct hisi_qm *qm, u64 *addr)
0099 {
0100     int ret;
0101 
0102     ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_BT, 0, 0, 1);
0103     if (ret)
0104         return ret;
0105 
0106     *addr = readl(qm->io_base + QM_MB_CMD_DATA_ADDR_L) |
0107           ((u64)readl(qm->io_base + QM_MB_CMD_DATA_ADDR_H) <<
0108           QM_XQC_ADDR_OFFSET);
0109 
0110     return 0;
0111 }
0112 
0113 static int qm_get_cqc(struct hisi_qm *qm, u64 *addr)
0114 {
0115     int ret;
0116 
0117     ret = hisi_qm_mb(qm, QM_MB_CMD_CQC_BT, 0, 0, 1);
0118     if (ret)
0119         return ret;
0120 
0121     *addr = readl(qm->io_base + QM_MB_CMD_DATA_ADDR_L) |
0122           ((u64)readl(qm->io_base + QM_MB_CMD_DATA_ADDR_H) <<
0123           QM_XQC_ADDR_OFFSET);
0124 
0125     return 0;
0126 }
0127 
0128 static int qm_get_regs(struct hisi_qm *qm, struct acc_vf_data *vf_data)
0129 {
0130     struct device *dev = &qm->pdev->dev;
0131     int ret;
0132 
0133     ret = qm_read_regs(qm, QM_VF_AEQ_INT_MASK, &vf_data->aeq_int_mask, 1);
0134     if (ret) {
0135         dev_err(dev, "failed to read QM_VF_AEQ_INT_MASK\n");
0136         return ret;
0137     }
0138 
0139     ret = qm_read_regs(qm, QM_VF_EQ_INT_MASK, &vf_data->eq_int_mask, 1);
0140     if (ret) {
0141         dev_err(dev, "failed to read QM_VF_EQ_INT_MASK\n");
0142         return ret;
0143     }
0144 
0145     ret = qm_read_regs(qm, QM_IFC_INT_SOURCE_V,
0146                &vf_data->ifc_int_source, 1);
0147     if (ret) {
0148         dev_err(dev, "failed to read QM_IFC_INT_SOURCE_V\n");
0149         return ret;
0150     }
0151 
0152     ret = qm_read_regs(qm, QM_IFC_INT_MASK, &vf_data->ifc_int_mask, 1);
0153     if (ret) {
0154         dev_err(dev, "failed to read QM_IFC_INT_MASK\n");
0155         return ret;
0156     }
0157 
0158     ret = qm_read_regs(qm, QM_IFC_INT_SET_V, &vf_data->ifc_int_set, 1);
0159     if (ret) {
0160         dev_err(dev, "failed to read QM_IFC_INT_SET_V\n");
0161         return ret;
0162     }
0163 
0164     ret = qm_read_regs(qm, QM_PAGE_SIZE, &vf_data->page_size, 1);
0165     if (ret) {
0166         dev_err(dev, "failed to read QM_PAGE_SIZE\n");
0167         return ret;
0168     }
0169 
0170     /* QM_EQC_DW has 7 regs */
0171     ret = qm_read_regs(qm, QM_EQC_DW0, vf_data->qm_eqc_dw, 7);
0172     if (ret) {
0173         dev_err(dev, "failed to read QM_EQC_DW\n");
0174         return ret;
0175     }
0176 
0177     /* QM_AEQC_DW has 7 regs */
0178     ret = qm_read_regs(qm, QM_AEQC_DW0, vf_data->qm_aeqc_dw, 7);
0179     if (ret) {
0180         dev_err(dev, "failed to read QM_AEQC_DW\n");
0181         return ret;
0182     }
0183 
0184     return 0;
0185 }
0186 
0187 static int qm_set_regs(struct hisi_qm *qm, struct acc_vf_data *vf_data)
0188 {
0189     struct device *dev = &qm->pdev->dev;
0190     int ret;
0191 
0192     /* check VF state */
0193     if (unlikely(hisi_qm_wait_mb_ready(qm))) {
0194         dev_err(&qm->pdev->dev, "QM device is not ready to write\n");
0195         return -EBUSY;
0196     }
0197 
0198     ret = qm_write_regs(qm, QM_VF_AEQ_INT_MASK, &vf_data->aeq_int_mask, 1);
0199     if (ret) {
0200         dev_err(dev, "failed to write QM_VF_AEQ_INT_MASK\n");
0201         return ret;
0202     }
0203 
0204     ret = qm_write_regs(qm, QM_VF_EQ_INT_MASK, &vf_data->eq_int_mask, 1);
0205     if (ret) {
0206         dev_err(dev, "failed to write QM_VF_EQ_INT_MASK\n");
0207         return ret;
0208     }
0209 
0210     ret = qm_write_regs(qm, QM_IFC_INT_SOURCE_V,
0211                 &vf_data->ifc_int_source, 1);
0212     if (ret) {
0213         dev_err(dev, "failed to write QM_IFC_INT_SOURCE_V\n");
0214         return ret;
0215     }
0216 
0217     ret = qm_write_regs(qm, QM_IFC_INT_MASK, &vf_data->ifc_int_mask, 1);
0218     if (ret) {
0219         dev_err(dev, "failed to write QM_IFC_INT_MASK\n");
0220         return ret;
0221     }
0222 
0223     ret = qm_write_regs(qm, QM_IFC_INT_SET_V, &vf_data->ifc_int_set, 1);
0224     if (ret) {
0225         dev_err(dev, "failed to write QM_IFC_INT_SET_V\n");
0226         return ret;
0227     }
0228 
0229     ret = qm_write_regs(qm, QM_QUE_ISO_CFG_V, &vf_data->que_iso_cfg, 1);
0230     if (ret) {
0231         dev_err(dev, "failed to write QM_QUE_ISO_CFG_V\n");
0232         return ret;
0233     }
0234 
0235     ret = qm_write_regs(qm, QM_PAGE_SIZE, &vf_data->page_size, 1);
0236     if (ret) {
0237         dev_err(dev, "failed to write QM_PAGE_SIZE\n");
0238         return ret;
0239     }
0240 
0241     /* QM_EQC_DW has 7 regs */
0242     ret = qm_write_regs(qm, QM_EQC_DW0, vf_data->qm_eqc_dw, 7);
0243     if (ret) {
0244         dev_err(dev, "failed to write QM_EQC_DW\n");
0245         return ret;
0246     }
0247 
0248     /* QM_AEQC_DW has 7 regs */
0249     ret = qm_write_regs(qm, QM_AEQC_DW0, vf_data->qm_aeqc_dw, 7);
0250     if (ret) {
0251         dev_err(dev, "failed to write QM_AEQC_DW\n");
0252         return ret;
0253     }
0254 
0255     return 0;
0256 }
0257 
0258 static void qm_db(struct hisi_qm *qm, u16 qn, u8 cmd,
0259           u16 index, u8 priority)
0260 {
0261     u64 doorbell;
0262     u64 dbase;
0263     u16 randata = 0;
0264 
0265     if (cmd == QM_DOORBELL_CMD_SQ || cmd == QM_DOORBELL_CMD_CQ)
0266         dbase = QM_DOORBELL_SQ_CQ_BASE_V2;
0267     else
0268         dbase = QM_DOORBELL_EQ_AEQ_BASE_V2;
0269 
0270     doorbell = qn | ((u64)cmd << QM_DB_CMD_SHIFT_V2) |
0271            ((u64)randata << QM_DB_RAND_SHIFT_V2) |
0272            ((u64)index << QM_DB_INDEX_SHIFT_V2)  |
0273            ((u64)priority << QM_DB_PRIORITY_SHIFT_V2);
0274 
0275     writeq(doorbell, qm->io_base + dbase);
0276 }
0277 
0278 static int pf_qm_get_qp_num(struct hisi_qm *qm, int vf_id, u32 *rbase)
0279 {
0280     unsigned int val;
0281     u64 sqc_vft;
0282     u32 qp_num;
0283     int ret;
0284 
0285     ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val,
0286                      val & BIT(0), MB_POLL_PERIOD_US,
0287                      MB_POLL_TIMEOUT_US);
0288     if (ret)
0289         return ret;
0290 
0291     writel(0x1, qm->io_base + QM_VFT_CFG_OP_WR);
0292     /* 0 mean SQC VFT */
0293     writel(0x0, qm->io_base + QM_VFT_CFG_TYPE);
0294     writel(vf_id, qm->io_base + QM_VFT_CFG);
0295 
0296     writel(0x0, qm->io_base + QM_VFT_CFG_RDY);
0297     writel(0x1, qm->io_base + QM_VFT_CFG_OP_ENABLE);
0298 
0299     ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val,
0300                      val & BIT(0), MB_POLL_PERIOD_US,
0301                      MB_POLL_TIMEOUT_US);
0302     if (ret)
0303         return ret;
0304 
0305     sqc_vft = readl(qm->io_base + QM_VFT_CFG_DATA_L) |
0306           ((u64)readl(qm->io_base + QM_VFT_CFG_DATA_H) <<
0307           QM_XQC_ADDR_OFFSET);
0308     *rbase = QM_SQC_VFT_BASE_MASK_V2 &
0309           (sqc_vft >> QM_SQC_VFT_BASE_SHIFT_V2);
0310     qp_num = (QM_SQC_VFT_NUM_MASK_V2 &
0311           (sqc_vft >> QM_SQC_VFT_NUM_SHIFT_V2)) + 1;
0312 
0313     return qp_num;
0314 }
0315 
0316 static void qm_dev_cmd_init(struct hisi_qm *qm)
0317 {
0318     /* Clear VF communication status registers. */
0319     writel(0x1, qm->io_base + QM_IFC_INT_SOURCE_V);
0320 
0321     /* Enable pf and vf communication. */
0322     writel(0x0, qm->io_base + QM_IFC_INT_MASK);
0323 }
0324 
0325 static int vf_qm_cache_wb(struct hisi_qm *qm)
0326 {
0327     unsigned int val;
0328 
0329     writel(0x1, qm->io_base + QM_CACHE_WB_START);
0330     if (readl_relaxed_poll_timeout(qm->io_base + QM_CACHE_WB_DONE,
0331                        val, val & BIT(0), MB_POLL_PERIOD_US,
0332                        MB_POLL_TIMEOUT_US)) {
0333         dev_err(&qm->pdev->dev, "vf QM writeback sqc cache fail\n");
0334         return -EINVAL;
0335     }
0336 
0337     return 0;
0338 }
0339 
0340 static struct hisi_acc_vf_core_device *hssi_acc_drvdata(struct pci_dev *pdev)
0341 {
0342     struct vfio_pci_core_device *core_device = dev_get_drvdata(&pdev->dev);
0343 
0344     return container_of(core_device, struct hisi_acc_vf_core_device,
0345                 core_device);
0346 }
0347 
0348 static void vf_qm_fun_reset(struct hisi_acc_vf_core_device *hisi_acc_vdev,
0349                 struct hisi_qm *qm)
0350 {
0351     int i;
0352 
0353     for (i = 0; i < qm->qp_num; i++)
0354         qm_db(qm, i, QM_DOORBELL_CMD_SQ, 0, 1);
0355 }
0356 
0357 static int vf_qm_func_stop(struct hisi_qm *qm)
0358 {
0359     return hisi_qm_mb(qm, QM_MB_CMD_PAUSE_QM, 0, 0, 0);
0360 }
0361 
0362 static int vf_qm_check_match(struct hisi_acc_vf_core_device *hisi_acc_vdev,
0363                  struct hisi_acc_vf_migration_file *migf)
0364 {
0365     struct acc_vf_data *vf_data = &migf->vf_data;
0366     struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm;
0367     struct hisi_qm *pf_qm = hisi_acc_vdev->pf_qm;
0368     struct device *dev = &vf_qm->pdev->dev;
0369     u32 que_iso_state;
0370     int ret;
0371 
0372     if (migf->total_length < QM_MATCH_SIZE)
0373         return -EINVAL;
0374 
0375     if (vf_data->acc_magic != ACC_DEV_MAGIC) {
0376         dev_err(dev, "failed to match ACC_DEV_MAGIC\n");
0377         return -EINVAL;
0378     }
0379 
0380     if (vf_data->dev_id != hisi_acc_vdev->vf_dev->device) {
0381         dev_err(dev, "failed to match VF devices\n");
0382         return -EINVAL;
0383     }
0384 
0385     /* vf qp num check */
0386     ret = qm_get_vft(vf_qm, &vf_qm->qp_base);
0387     if (ret <= 0) {
0388         dev_err(dev, "failed to get vft qp nums\n");
0389         return -EINVAL;
0390     }
0391 
0392     if (ret != vf_data->qp_num) {
0393         dev_err(dev, "failed to match VF qp num\n");
0394         return -EINVAL;
0395     }
0396 
0397     vf_qm->qp_num = ret;
0398 
0399     /* vf isolation state check */
0400     ret = qm_read_regs(pf_qm, QM_QUE_ISO_CFG_V, &que_iso_state, 1);
0401     if (ret) {
0402         dev_err(dev, "failed to read QM_QUE_ISO_CFG_V\n");
0403         return ret;
0404     }
0405 
0406     if (vf_data->que_iso_cfg != que_iso_state) {
0407         dev_err(dev, "failed to match isolation state\n");
0408         return ret;
0409     }
0410 
0411     ret = qm_write_regs(vf_qm, QM_VF_STATE, &vf_data->vf_qm_state, 1);
0412     if (ret) {
0413         dev_err(dev, "failed to write QM_VF_STATE\n");
0414         return ret;
0415     }
0416 
0417     hisi_acc_vdev->vf_qm_state = vf_data->vf_qm_state;
0418     return 0;
0419 }
0420 
0421 static int vf_qm_get_match_data(struct hisi_acc_vf_core_device *hisi_acc_vdev,
0422                 struct acc_vf_data *vf_data)
0423 {
0424     struct hisi_qm *pf_qm = hisi_acc_vdev->pf_qm;
0425     struct device *dev = &pf_qm->pdev->dev;
0426     int vf_id = hisi_acc_vdev->vf_id;
0427     int ret;
0428 
0429     vf_data->acc_magic = ACC_DEV_MAGIC;
0430     /* save device id */
0431     vf_data->dev_id = hisi_acc_vdev->vf_dev->device;
0432 
0433     /* vf qp num save from PF */
0434     ret = pf_qm_get_qp_num(pf_qm, vf_id, &vf_data->qp_base);
0435     if (ret <= 0) {
0436         dev_err(dev, "failed to get vft qp nums!\n");
0437         return -EINVAL;
0438     }
0439 
0440     vf_data->qp_num = ret;
0441 
0442     /* VF isolation state save from PF */
0443     ret = qm_read_regs(pf_qm, QM_QUE_ISO_CFG_V, &vf_data->que_iso_cfg, 1);
0444     if (ret) {
0445         dev_err(dev, "failed to read QM_QUE_ISO_CFG_V!\n");
0446         return ret;
0447     }
0448 
0449     return 0;
0450 }
0451 
0452 static int vf_qm_load_data(struct hisi_acc_vf_core_device *hisi_acc_vdev,
0453                struct hisi_acc_vf_migration_file *migf)
0454 {
0455     struct hisi_qm *qm = &hisi_acc_vdev->vf_qm;
0456     struct device *dev = &qm->pdev->dev;
0457     struct acc_vf_data *vf_data = &migf->vf_data;
0458     int ret;
0459 
0460     /* Return if only match data was transferred */
0461     if (migf->total_length == QM_MATCH_SIZE)
0462         return 0;
0463 
0464     if (migf->total_length < sizeof(struct acc_vf_data))
0465         return -EINVAL;
0466 
0467     qm->eqe_dma = vf_data->eqe_dma;
0468     qm->aeqe_dma = vf_data->aeqe_dma;
0469     qm->sqc_dma = vf_data->sqc_dma;
0470     qm->cqc_dma = vf_data->cqc_dma;
0471 
0472     qm->qp_base = vf_data->qp_base;
0473     qm->qp_num = vf_data->qp_num;
0474 
0475     ret = qm_set_regs(qm, vf_data);
0476     if (ret) {
0477         dev_err(dev, "Set VF regs failed\n");
0478         return ret;
0479     }
0480 
0481     ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_BT, qm->sqc_dma, 0, 0);
0482     if (ret) {
0483         dev_err(dev, "Set sqc failed\n");
0484         return ret;
0485     }
0486 
0487     ret = hisi_qm_mb(qm, QM_MB_CMD_CQC_BT, qm->cqc_dma, 0, 0);
0488     if (ret) {
0489         dev_err(dev, "Set cqc failed\n");
0490         return ret;
0491     }
0492 
0493     qm_dev_cmd_init(qm);
0494     return 0;
0495 }
0496 
0497 static int vf_qm_state_save(struct hisi_acc_vf_core_device *hisi_acc_vdev,
0498                 struct hisi_acc_vf_migration_file *migf)
0499 {
0500     struct acc_vf_data *vf_data = &migf->vf_data;
0501     struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm;
0502     struct device *dev = &vf_qm->pdev->dev;
0503     int ret;
0504 
0505     ret = vf_qm_get_match_data(hisi_acc_vdev, vf_data);
0506     if (ret)
0507         return ret;
0508 
0509     if (unlikely(qm_wait_dev_not_ready(vf_qm))) {
0510         /* Update state and return with match data */
0511         vf_data->vf_qm_state = QM_NOT_READY;
0512         hisi_acc_vdev->vf_qm_state = vf_data->vf_qm_state;
0513         migf->total_length = QM_MATCH_SIZE;
0514         return 0;
0515     }
0516 
0517     vf_data->vf_qm_state = QM_READY;
0518     hisi_acc_vdev->vf_qm_state = vf_data->vf_qm_state;
0519 
0520     ret = vf_qm_cache_wb(vf_qm);
0521     if (ret) {
0522         dev_err(dev, "failed to writeback QM Cache!\n");
0523         return ret;
0524     }
0525 
0526     ret = qm_get_regs(vf_qm, vf_data);
0527     if (ret)
0528         return -EINVAL;
0529 
0530     /* Every reg is 32 bit, the dma address is 64 bit. */
0531     vf_data->eqe_dma = vf_data->qm_eqc_dw[2];
0532     vf_data->eqe_dma <<= QM_XQC_ADDR_OFFSET;
0533     vf_data->eqe_dma |= vf_data->qm_eqc_dw[1];
0534     vf_data->aeqe_dma = vf_data->qm_aeqc_dw[2];
0535     vf_data->aeqe_dma <<= QM_XQC_ADDR_OFFSET;
0536     vf_data->aeqe_dma |= vf_data->qm_aeqc_dw[1];
0537 
0538     /* Through SQC_BT/CQC_BT to get sqc and cqc address */
0539     ret = qm_get_sqc(vf_qm, &vf_data->sqc_dma);
0540     if (ret) {
0541         dev_err(dev, "failed to read SQC addr!\n");
0542         return -EINVAL;
0543     }
0544 
0545     ret = qm_get_cqc(vf_qm, &vf_data->cqc_dma);
0546     if (ret) {
0547         dev_err(dev, "failed to read CQC addr!\n");
0548         return -EINVAL;
0549     }
0550 
0551     migf->total_length = sizeof(struct acc_vf_data);
0552     return 0;
0553 }
0554 
0555 /* Check the PF's RAS state and Function INT state */
0556 static int
0557 hisi_acc_check_int_state(struct hisi_acc_vf_core_device *hisi_acc_vdev)
0558 {
0559     struct hisi_qm *vfqm = &hisi_acc_vdev->vf_qm;
0560     struct hisi_qm *qm = hisi_acc_vdev->pf_qm;
0561     struct pci_dev *vf_pdev = hisi_acc_vdev->vf_dev;
0562     struct device *dev = &qm->pdev->dev;
0563     u32 state;
0564 
0565     /* Check RAS state */
0566     state = qm_check_reg_state(qm, QM_ABNORMAL_INT_STATUS);
0567     if (state) {
0568         dev_err(dev, "failed to check QM RAS state!\n");
0569         return -EBUSY;
0570     }
0571 
0572     /* Check Function Communication state between PF and VF */
0573     state = qm_check_reg_state(vfqm, QM_IFC_INT_STATUS);
0574     if (state) {
0575         dev_err(dev, "failed to check QM IFC INT state!\n");
0576         return -EBUSY;
0577     }
0578     state = qm_check_reg_state(vfqm, QM_IFC_INT_SET_V);
0579     if (state) {
0580         dev_err(dev, "failed to check QM IFC INT SET state!\n");
0581         return -EBUSY;
0582     }
0583 
0584     /* Check submodule task state */
0585     switch (vf_pdev->device) {
0586     case PCI_DEVICE_ID_HUAWEI_SEC_VF:
0587         state = qm_check_reg_state(qm, SEC_CORE_INT_STATUS);
0588         if (state) {
0589             dev_err(dev, "failed to check QM SEC Core INT state!\n");
0590             return -EBUSY;
0591         }
0592         return 0;
0593     case PCI_DEVICE_ID_HUAWEI_HPRE_VF:
0594         state = qm_check_reg_state(qm, HPRE_HAC_INT_STATUS);
0595         if (state) {
0596             dev_err(dev, "failed to check QM HPRE HAC INT state!\n");
0597             return -EBUSY;
0598         }
0599         return 0;
0600     case PCI_DEVICE_ID_HUAWEI_ZIP_VF:
0601         state = qm_check_reg_state(qm, HZIP_CORE_INT_STATUS);
0602         if (state) {
0603             dev_err(dev, "failed to check QM ZIP Core INT state!\n");
0604             return -EBUSY;
0605         }
0606         return 0;
0607     default:
0608         dev_err(dev, "failed to detect acc module type!\n");
0609         return -EINVAL;
0610     }
0611 }
0612 
0613 static void hisi_acc_vf_disable_fd(struct hisi_acc_vf_migration_file *migf)
0614 {
0615     mutex_lock(&migf->lock);
0616     migf->disabled = true;
0617     migf->total_length = 0;
0618     migf->filp->f_pos = 0;
0619     mutex_unlock(&migf->lock);
0620 }
0621 
0622 static void hisi_acc_vf_disable_fds(struct hisi_acc_vf_core_device *hisi_acc_vdev)
0623 {
0624     if (hisi_acc_vdev->resuming_migf) {
0625         hisi_acc_vf_disable_fd(hisi_acc_vdev->resuming_migf);
0626         fput(hisi_acc_vdev->resuming_migf->filp);
0627         hisi_acc_vdev->resuming_migf = NULL;
0628     }
0629 
0630     if (hisi_acc_vdev->saving_migf) {
0631         hisi_acc_vf_disable_fd(hisi_acc_vdev->saving_migf);
0632         fput(hisi_acc_vdev->saving_migf->filp);
0633         hisi_acc_vdev->saving_migf = NULL;
0634     }
0635 }
0636 
0637 /*
0638  * This function is called in all state_mutex unlock cases to
0639  * handle a 'deferred_reset' if exists.
0640  */
0641 static void
0642 hisi_acc_vf_state_mutex_unlock(struct hisi_acc_vf_core_device *hisi_acc_vdev)
0643 {
0644 again:
0645     spin_lock(&hisi_acc_vdev->reset_lock);
0646     if (hisi_acc_vdev->deferred_reset) {
0647         hisi_acc_vdev->deferred_reset = false;
0648         spin_unlock(&hisi_acc_vdev->reset_lock);
0649         hisi_acc_vdev->vf_qm_state = QM_NOT_READY;
0650         hisi_acc_vdev->mig_state = VFIO_DEVICE_STATE_RUNNING;
0651         hisi_acc_vf_disable_fds(hisi_acc_vdev);
0652         goto again;
0653     }
0654     mutex_unlock(&hisi_acc_vdev->state_mutex);
0655     spin_unlock(&hisi_acc_vdev->reset_lock);
0656 }
0657 
0658 static void hisi_acc_vf_start_device(struct hisi_acc_vf_core_device *hisi_acc_vdev)
0659 {
0660     struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm;
0661 
0662     if (hisi_acc_vdev->vf_qm_state != QM_READY)
0663         return;
0664 
0665     vf_qm_fun_reset(hisi_acc_vdev, vf_qm);
0666 }
0667 
0668 static int hisi_acc_vf_load_state(struct hisi_acc_vf_core_device *hisi_acc_vdev)
0669 {
0670     struct device *dev = &hisi_acc_vdev->vf_dev->dev;
0671     struct hisi_acc_vf_migration_file *migf = hisi_acc_vdev->resuming_migf;
0672     int ret;
0673 
0674     /* Check dev compatibility */
0675     ret = vf_qm_check_match(hisi_acc_vdev, migf);
0676     if (ret) {
0677         dev_err(dev, "failed to match the VF!\n");
0678         return ret;
0679     }
0680     /* Recover data to VF */
0681     ret = vf_qm_load_data(hisi_acc_vdev, migf);
0682     if (ret) {
0683         dev_err(dev, "failed to recover the VF!\n");
0684         return ret;
0685     }
0686 
0687     return 0;
0688 }
0689 
0690 static int hisi_acc_vf_release_file(struct inode *inode, struct file *filp)
0691 {
0692     struct hisi_acc_vf_migration_file *migf = filp->private_data;
0693 
0694     hisi_acc_vf_disable_fd(migf);
0695     mutex_destroy(&migf->lock);
0696     kfree(migf);
0697     return 0;
0698 }
0699 
0700 static ssize_t hisi_acc_vf_resume_write(struct file *filp, const char __user *buf,
0701                     size_t len, loff_t *pos)
0702 {
0703     struct hisi_acc_vf_migration_file *migf = filp->private_data;
0704     loff_t requested_length;
0705     ssize_t done = 0;
0706     int ret;
0707 
0708     if (pos)
0709         return -ESPIPE;
0710     pos = &filp->f_pos;
0711 
0712     if (*pos < 0 ||
0713         check_add_overflow((loff_t)len, *pos, &requested_length))
0714         return -EINVAL;
0715 
0716     if (requested_length > sizeof(struct acc_vf_data))
0717         return -ENOMEM;
0718 
0719     mutex_lock(&migf->lock);
0720     if (migf->disabled) {
0721         done = -ENODEV;
0722         goto out_unlock;
0723     }
0724 
0725     ret = copy_from_user(&migf->vf_data, buf, len);
0726     if (ret) {
0727         done = -EFAULT;
0728         goto out_unlock;
0729     }
0730     *pos += len;
0731     done = len;
0732     migf->total_length += len;
0733 out_unlock:
0734     mutex_unlock(&migf->lock);
0735     return done;
0736 }
0737 
0738 static const struct file_operations hisi_acc_vf_resume_fops = {
0739     .owner = THIS_MODULE,
0740     .write = hisi_acc_vf_resume_write,
0741     .release = hisi_acc_vf_release_file,
0742     .llseek = no_llseek,
0743 };
0744 
0745 static struct hisi_acc_vf_migration_file *
0746 hisi_acc_vf_pci_resume(struct hisi_acc_vf_core_device *hisi_acc_vdev)
0747 {
0748     struct hisi_acc_vf_migration_file *migf;
0749 
0750     migf = kzalloc(sizeof(*migf), GFP_KERNEL);
0751     if (!migf)
0752         return ERR_PTR(-ENOMEM);
0753 
0754     migf->filp = anon_inode_getfile("hisi_acc_vf_mig", &hisi_acc_vf_resume_fops, migf,
0755                     O_WRONLY);
0756     if (IS_ERR(migf->filp)) {
0757         int err = PTR_ERR(migf->filp);
0758 
0759         kfree(migf);
0760         return ERR_PTR(err);
0761     }
0762 
0763     stream_open(migf->filp->f_inode, migf->filp);
0764     mutex_init(&migf->lock);
0765     return migf;
0766 }
0767 
0768 static ssize_t hisi_acc_vf_save_read(struct file *filp, char __user *buf, size_t len,
0769                      loff_t *pos)
0770 {
0771     struct hisi_acc_vf_migration_file *migf = filp->private_data;
0772     ssize_t done = 0;
0773     int ret;
0774 
0775     if (pos)
0776         return -ESPIPE;
0777     pos = &filp->f_pos;
0778 
0779     mutex_lock(&migf->lock);
0780     if (*pos > migf->total_length) {
0781         done = -EINVAL;
0782         goto out_unlock;
0783     }
0784 
0785     if (migf->disabled) {
0786         done = -ENODEV;
0787         goto out_unlock;
0788     }
0789 
0790     len = min_t(size_t, migf->total_length - *pos, len);
0791     if (len) {
0792         ret = copy_to_user(buf, &migf->vf_data, len);
0793         if (ret) {
0794             done = -EFAULT;
0795             goto out_unlock;
0796         }
0797         *pos += len;
0798         done = len;
0799     }
0800 out_unlock:
0801     mutex_unlock(&migf->lock);
0802     return done;
0803 }
0804 
0805 static const struct file_operations hisi_acc_vf_save_fops = {
0806     .owner = THIS_MODULE,
0807     .read = hisi_acc_vf_save_read,
0808     .release = hisi_acc_vf_release_file,
0809     .llseek = no_llseek,
0810 };
0811 
0812 static struct hisi_acc_vf_migration_file *
0813 hisi_acc_vf_stop_copy(struct hisi_acc_vf_core_device *hisi_acc_vdev)
0814 {
0815     struct hisi_acc_vf_migration_file *migf;
0816     int ret;
0817 
0818     migf = kzalloc(sizeof(*migf), GFP_KERNEL);
0819     if (!migf)
0820         return ERR_PTR(-ENOMEM);
0821 
0822     migf->filp = anon_inode_getfile("hisi_acc_vf_mig", &hisi_acc_vf_save_fops, migf,
0823                     O_RDONLY);
0824     if (IS_ERR(migf->filp)) {
0825         int err = PTR_ERR(migf->filp);
0826 
0827         kfree(migf);
0828         return ERR_PTR(err);
0829     }
0830 
0831     stream_open(migf->filp->f_inode, migf->filp);
0832     mutex_init(&migf->lock);
0833 
0834     ret = vf_qm_state_save(hisi_acc_vdev, migf);
0835     if (ret) {
0836         fput(migf->filp);
0837         return ERR_PTR(ret);
0838     }
0839 
0840     return migf;
0841 }
0842 
0843 static int hisi_acc_vf_stop_device(struct hisi_acc_vf_core_device *hisi_acc_vdev)
0844 {
0845     struct device *dev = &hisi_acc_vdev->vf_dev->dev;
0846     struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm;
0847     int ret;
0848 
0849     ret = vf_qm_func_stop(vf_qm);
0850     if (ret) {
0851         dev_err(dev, "failed to stop QM VF function!\n");
0852         return ret;
0853     }
0854 
0855     ret = hisi_acc_check_int_state(hisi_acc_vdev);
0856     if (ret) {
0857         dev_err(dev, "failed to check QM INT state!\n");
0858         return ret;
0859     }
0860     return 0;
0861 }
0862 
0863 static struct file *
0864 hisi_acc_vf_set_device_state(struct hisi_acc_vf_core_device *hisi_acc_vdev,
0865                  u32 new)
0866 {
0867     u32 cur = hisi_acc_vdev->mig_state;
0868     int ret;
0869 
0870     if (cur == VFIO_DEVICE_STATE_RUNNING && new == VFIO_DEVICE_STATE_STOP) {
0871         ret = hisi_acc_vf_stop_device(hisi_acc_vdev);
0872         if (ret)
0873             return ERR_PTR(ret);
0874         return NULL;
0875     }
0876 
0877     if (cur == VFIO_DEVICE_STATE_STOP && new == VFIO_DEVICE_STATE_STOP_COPY) {
0878         struct hisi_acc_vf_migration_file *migf;
0879 
0880         migf = hisi_acc_vf_stop_copy(hisi_acc_vdev);
0881         if (IS_ERR(migf))
0882             return ERR_CAST(migf);
0883         get_file(migf->filp);
0884         hisi_acc_vdev->saving_migf = migf;
0885         return migf->filp;
0886     }
0887 
0888     if ((cur == VFIO_DEVICE_STATE_STOP_COPY && new == VFIO_DEVICE_STATE_STOP)) {
0889         hisi_acc_vf_disable_fds(hisi_acc_vdev);
0890         return NULL;
0891     }
0892 
0893     if (cur == VFIO_DEVICE_STATE_STOP && new == VFIO_DEVICE_STATE_RESUMING) {
0894         struct hisi_acc_vf_migration_file *migf;
0895 
0896         migf = hisi_acc_vf_pci_resume(hisi_acc_vdev);
0897         if (IS_ERR(migf))
0898             return ERR_CAST(migf);
0899         get_file(migf->filp);
0900         hisi_acc_vdev->resuming_migf = migf;
0901         return migf->filp;
0902     }
0903 
0904     if (cur == VFIO_DEVICE_STATE_RESUMING && new == VFIO_DEVICE_STATE_STOP) {
0905         ret = hisi_acc_vf_load_state(hisi_acc_vdev);
0906         if (ret)
0907             return ERR_PTR(ret);
0908         hisi_acc_vf_disable_fds(hisi_acc_vdev);
0909         return NULL;
0910     }
0911 
0912     if (cur == VFIO_DEVICE_STATE_STOP && new == VFIO_DEVICE_STATE_RUNNING) {
0913         hisi_acc_vf_start_device(hisi_acc_vdev);
0914         return NULL;
0915     }
0916 
0917     /*
0918      * vfio_mig_get_next_state() does not use arcs other than the above
0919      */
0920     WARN_ON(true);
0921     return ERR_PTR(-EINVAL);
0922 }
0923 
0924 static struct file *
0925 hisi_acc_vfio_pci_set_device_state(struct vfio_device *vdev,
0926                    enum vfio_device_mig_state new_state)
0927 {
0928     struct hisi_acc_vf_core_device *hisi_acc_vdev = container_of(vdev,
0929             struct hisi_acc_vf_core_device, core_device.vdev);
0930     enum vfio_device_mig_state next_state;
0931     struct file *res = NULL;
0932     int ret;
0933 
0934     mutex_lock(&hisi_acc_vdev->state_mutex);
0935     while (new_state != hisi_acc_vdev->mig_state) {
0936         ret = vfio_mig_get_next_state(vdev,
0937                           hisi_acc_vdev->mig_state,
0938                           new_state, &next_state);
0939         if (ret) {
0940             res = ERR_PTR(-EINVAL);
0941             break;
0942         }
0943 
0944         res = hisi_acc_vf_set_device_state(hisi_acc_vdev, next_state);
0945         if (IS_ERR(res))
0946             break;
0947         hisi_acc_vdev->mig_state = next_state;
0948         if (WARN_ON(res && new_state != hisi_acc_vdev->mig_state)) {
0949             fput(res);
0950             res = ERR_PTR(-EINVAL);
0951             break;
0952         }
0953     }
0954     hisi_acc_vf_state_mutex_unlock(hisi_acc_vdev);
0955     return res;
0956 }
0957 
0958 static int
0959 hisi_acc_vfio_pci_get_device_state(struct vfio_device *vdev,
0960                    enum vfio_device_mig_state *curr_state)
0961 {
0962     struct hisi_acc_vf_core_device *hisi_acc_vdev = container_of(vdev,
0963             struct hisi_acc_vf_core_device, core_device.vdev);
0964 
0965     mutex_lock(&hisi_acc_vdev->state_mutex);
0966     *curr_state = hisi_acc_vdev->mig_state;
0967     hisi_acc_vf_state_mutex_unlock(hisi_acc_vdev);
0968     return 0;
0969 }
0970 
0971 static void hisi_acc_vf_pci_aer_reset_done(struct pci_dev *pdev)
0972 {
0973     struct hisi_acc_vf_core_device *hisi_acc_vdev = hssi_acc_drvdata(pdev);
0974 
0975     if (hisi_acc_vdev->core_device.vdev.migration_flags !=
0976                 VFIO_MIGRATION_STOP_COPY)
0977         return;
0978 
0979     /*
0980      * As the higher VFIO layers are holding locks across reset and using
0981      * those same locks with the mm_lock we need to prevent ABBA deadlock
0982      * with the state_mutex and mm_lock.
0983      * In case the state_mutex was taken already we defer the cleanup work
0984      * to the unlock flow of the other running context.
0985      */
0986     spin_lock(&hisi_acc_vdev->reset_lock);
0987     hisi_acc_vdev->deferred_reset = true;
0988     if (!mutex_trylock(&hisi_acc_vdev->state_mutex)) {
0989         spin_unlock(&hisi_acc_vdev->reset_lock);
0990         return;
0991     }
0992     spin_unlock(&hisi_acc_vdev->reset_lock);
0993     hisi_acc_vf_state_mutex_unlock(hisi_acc_vdev);
0994 }
0995 
0996 static int hisi_acc_vf_qm_init(struct hisi_acc_vf_core_device *hisi_acc_vdev)
0997 {
0998     struct vfio_pci_core_device *vdev = &hisi_acc_vdev->core_device;
0999     struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm;
1000     struct pci_dev *vf_dev = vdev->pdev;
1001 
1002     /*
1003      * ACC VF dev BAR2 region consists of both functional register space
1004      * and migration control register space. For migration to work, we
1005      * need access to both. Hence, we map the entire BAR2 region here.
1006      * But unnecessarily exposing the migration BAR region to the Guest
1007      * has the potential to prevent/corrupt the Guest migration. Hence,
1008      * we restrict access to the migration control space from
1009      * Guest(Please see mmap/ioctl/read/write override functions).
1010      *
1011      * Please note that it is OK to expose the entire VF BAR if migration
1012      * is not supported or required as this cannot affect the ACC PF
1013      * configurations.
1014      *
1015      * Also the HiSilicon ACC VF devices supported by this driver on
1016      * HiSilicon hardware platforms are integrated end point devices
1017      * and the platform lacks the capability to perform any PCIe P2P
1018      * between these devices.
1019      */
1020 
1021     vf_qm->io_base =
1022         ioremap(pci_resource_start(vf_dev, VFIO_PCI_BAR2_REGION_INDEX),
1023             pci_resource_len(vf_dev, VFIO_PCI_BAR2_REGION_INDEX));
1024     if (!vf_qm->io_base)
1025         return -EIO;
1026 
1027     vf_qm->fun_type = QM_HW_VF;
1028     vf_qm->pdev = vf_dev;
1029     mutex_init(&vf_qm->mailbox_lock);
1030 
1031     return 0;
1032 }
1033 
1034 static struct hisi_qm *hisi_acc_get_pf_qm(struct pci_dev *pdev)
1035 {
1036     struct hisi_qm  *pf_qm;
1037     struct pci_driver *pf_driver;
1038 
1039     if (!pdev->is_virtfn)
1040         return NULL;
1041 
1042     switch (pdev->device) {
1043     case PCI_DEVICE_ID_HUAWEI_SEC_VF:
1044         pf_driver = hisi_sec_get_pf_driver();
1045         break;
1046     case PCI_DEVICE_ID_HUAWEI_HPRE_VF:
1047         pf_driver = hisi_hpre_get_pf_driver();
1048         break;
1049     case PCI_DEVICE_ID_HUAWEI_ZIP_VF:
1050         pf_driver = hisi_zip_get_pf_driver();
1051         break;
1052     default:
1053         return NULL;
1054     }
1055 
1056     if (!pf_driver)
1057         return NULL;
1058 
1059     pf_qm = pci_iov_get_pf_drvdata(pdev, pf_driver);
1060 
1061     return !IS_ERR(pf_qm) ? pf_qm : NULL;
1062 }
1063 
1064 static int hisi_acc_pci_rw_access_check(struct vfio_device *core_vdev,
1065                     size_t count, loff_t *ppos,
1066                     size_t *new_count)
1067 {
1068     unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
1069     struct vfio_pci_core_device *vdev =
1070         container_of(core_vdev, struct vfio_pci_core_device, vdev);
1071 
1072     if (index == VFIO_PCI_BAR2_REGION_INDEX) {
1073         loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
1074         resource_size_t end = pci_resource_len(vdev->pdev, index) / 2;
1075 
1076         /* Check if access is for migration control region */
1077         if (pos >= end)
1078             return -EINVAL;
1079 
1080         *new_count = min(count, (size_t)(end - pos));
1081     }
1082 
1083     return 0;
1084 }
1085 
1086 static int hisi_acc_vfio_pci_mmap(struct vfio_device *core_vdev,
1087                   struct vm_area_struct *vma)
1088 {
1089     struct vfio_pci_core_device *vdev =
1090         container_of(core_vdev, struct vfio_pci_core_device, vdev);
1091     unsigned int index;
1092 
1093     index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT);
1094     if (index == VFIO_PCI_BAR2_REGION_INDEX) {
1095         u64 req_len, pgoff, req_start;
1096         resource_size_t end = pci_resource_len(vdev->pdev, index) / 2;
1097 
1098         req_len = vma->vm_end - vma->vm_start;
1099         pgoff = vma->vm_pgoff &
1100             ((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1);
1101         req_start = pgoff << PAGE_SHIFT;
1102 
1103         if (req_start + req_len > end)
1104             return -EINVAL;
1105     }
1106 
1107     return vfio_pci_core_mmap(core_vdev, vma);
1108 }
1109 
1110 static ssize_t hisi_acc_vfio_pci_write(struct vfio_device *core_vdev,
1111                        const char __user *buf, size_t count,
1112                        loff_t *ppos)
1113 {
1114     size_t new_count = count;
1115     int ret;
1116 
1117     ret = hisi_acc_pci_rw_access_check(core_vdev, count, ppos, &new_count);
1118     if (ret)
1119         return ret;
1120 
1121     return vfio_pci_core_write(core_vdev, buf, new_count, ppos);
1122 }
1123 
1124 static ssize_t hisi_acc_vfio_pci_read(struct vfio_device *core_vdev,
1125                       char __user *buf, size_t count,
1126                       loff_t *ppos)
1127 {
1128     size_t new_count = count;
1129     int ret;
1130 
1131     ret = hisi_acc_pci_rw_access_check(core_vdev, count, ppos, &new_count);
1132     if (ret)
1133         return ret;
1134 
1135     return vfio_pci_core_read(core_vdev, buf, new_count, ppos);
1136 }
1137 
1138 static long hisi_acc_vfio_pci_ioctl(struct vfio_device *core_vdev, unsigned int cmd,
1139                     unsigned long arg)
1140 {
1141     if (cmd == VFIO_DEVICE_GET_REGION_INFO) {
1142         struct vfio_pci_core_device *vdev =
1143             container_of(core_vdev, struct vfio_pci_core_device, vdev);
1144         struct pci_dev *pdev = vdev->pdev;
1145         struct vfio_region_info info;
1146         unsigned long minsz;
1147 
1148         minsz = offsetofend(struct vfio_region_info, offset);
1149 
1150         if (copy_from_user(&info, (void __user *)arg, minsz))
1151             return -EFAULT;
1152 
1153         if (info.argsz < minsz)
1154             return -EINVAL;
1155 
1156         if (info.index == VFIO_PCI_BAR2_REGION_INDEX) {
1157             info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1158 
1159             /*
1160              * ACC VF dev BAR2 region consists of both functional
1161              * register space and migration control register space.
1162              * Report only the functional region to Guest.
1163              */
1164             info.size = pci_resource_len(pdev, info.index) / 2;
1165 
1166             info.flags = VFIO_REGION_INFO_FLAG_READ |
1167                     VFIO_REGION_INFO_FLAG_WRITE |
1168                     VFIO_REGION_INFO_FLAG_MMAP;
1169 
1170             return copy_to_user((void __user *)arg, &info, minsz) ?
1171                         -EFAULT : 0;
1172         }
1173     }
1174     return vfio_pci_core_ioctl(core_vdev, cmd, arg);
1175 }
1176 
1177 static int hisi_acc_vfio_pci_open_device(struct vfio_device *core_vdev)
1178 {
1179     struct hisi_acc_vf_core_device *hisi_acc_vdev = container_of(core_vdev,
1180             struct hisi_acc_vf_core_device, core_device.vdev);
1181     struct vfio_pci_core_device *vdev = &hisi_acc_vdev->core_device;
1182     int ret;
1183 
1184     ret = vfio_pci_core_enable(vdev);
1185     if (ret)
1186         return ret;
1187 
1188     if (core_vdev->mig_ops) {
1189         ret = hisi_acc_vf_qm_init(hisi_acc_vdev);
1190         if (ret) {
1191             vfio_pci_core_disable(vdev);
1192             return ret;
1193         }
1194         hisi_acc_vdev->mig_state = VFIO_DEVICE_STATE_RUNNING;
1195     }
1196 
1197     vfio_pci_core_finish_enable(vdev);
1198     return 0;
1199 }
1200 
1201 static void hisi_acc_vfio_pci_close_device(struct vfio_device *core_vdev)
1202 {
1203     struct hisi_acc_vf_core_device *hisi_acc_vdev = container_of(core_vdev,
1204             struct hisi_acc_vf_core_device, core_device.vdev);
1205     struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm;
1206 
1207     iounmap(vf_qm->io_base);
1208     vfio_pci_core_close_device(core_vdev);
1209 }
1210 
1211 static const struct vfio_migration_ops hisi_acc_vfio_pci_migrn_state_ops = {
1212     .migration_set_state = hisi_acc_vfio_pci_set_device_state,
1213     .migration_get_state = hisi_acc_vfio_pci_get_device_state,
1214 };
1215 
1216 static const struct vfio_device_ops hisi_acc_vfio_pci_migrn_ops = {
1217     .name = "hisi-acc-vfio-pci-migration",
1218     .open_device = hisi_acc_vfio_pci_open_device,
1219     .close_device = hisi_acc_vfio_pci_close_device,
1220     .ioctl = hisi_acc_vfio_pci_ioctl,
1221     .device_feature = vfio_pci_core_ioctl_feature,
1222     .read = hisi_acc_vfio_pci_read,
1223     .write = hisi_acc_vfio_pci_write,
1224     .mmap = hisi_acc_vfio_pci_mmap,
1225     .request = vfio_pci_core_request,
1226     .match = vfio_pci_core_match,
1227 };
1228 
1229 static const struct vfio_device_ops hisi_acc_vfio_pci_ops = {
1230     .name = "hisi-acc-vfio-pci",
1231     .open_device = hisi_acc_vfio_pci_open_device,
1232     .close_device = vfio_pci_core_close_device,
1233     .ioctl = vfio_pci_core_ioctl,
1234     .device_feature = vfio_pci_core_ioctl_feature,
1235     .read = vfio_pci_core_read,
1236     .write = vfio_pci_core_write,
1237     .mmap = vfio_pci_core_mmap,
1238     .request = vfio_pci_core_request,
1239     .match = vfio_pci_core_match,
1240 };
1241 
1242 static int
1243 hisi_acc_vfio_pci_migrn_init(struct hisi_acc_vf_core_device *hisi_acc_vdev,
1244                  struct pci_dev *pdev, struct hisi_qm *pf_qm)
1245 {
1246     int vf_id;
1247 
1248     vf_id = pci_iov_vf_id(pdev);
1249     if (vf_id < 0)
1250         return vf_id;
1251 
1252     hisi_acc_vdev->vf_id = vf_id + 1;
1253     hisi_acc_vdev->core_device.vdev.migration_flags =
1254                     VFIO_MIGRATION_STOP_COPY;
1255     hisi_acc_vdev->pf_qm = pf_qm;
1256     hisi_acc_vdev->vf_dev = pdev;
1257     mutex_init(&hisi_acc_vdev->state_mutex);
1258 
1259     return 0;
1260 }
1261 
1262 static int hisi_acc_vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1263 {
1264     struct hisi_acc_vf_core_device *hisi_acc_vdev;
1265     struct hisi_qm *pf_qm;
1266     int ret;
1267 
1268     hisi_acc_vdev = kzalloc(sizeof(*hisi_acc_vdev), GFP_KERNEL);
1269     if (!hisi_acc_vdev)
1270         return -ENOMEM;
1271 
1272     pf_qm = hisi_acc_get_pf_qm(pdev);
1273     if (pf_qm && pf_qm->ver >= QM_HW_V3) {
1274         ret = hisi_acc_vfio_pci_migrn_init(hisi_acc_vdev, pdev, pf_qm);
1275         if (!ret) {
1276             vfio_pci_core_init_device(&hisi_acc_vdev->core_device, pdev,
1277                           &hisi_acc_vfio_pci_migrn_ops);
1278             hisi_acc_vdev->core_device.vdev.mig_ops =
1279                     &hisi_acc_vfio_pci_migrn_state_ops;
1280         } else {
1281             pci_warn(pdev, "migration support failed, continue with generic interface\n");
1282             vfio_pci_core_init_device(&hisi_acc_vdev->core_device, pdev,
1283                           &hisi_acc_vfio_pci_ops);
1284         }
1285     } else {
1286         vfio_pci_core_init_device(&hisi_acc_vdev->core_device, pdev,
1287                       &hisi_acc_vfio_pci_ops);
1288     }
1289 
1290     dev_set_drvdata(&pdev->dev, &hisi_acc_vdev->core_device);
1291     ret = vfio_pci_core_register_device(&hisi_acc_vdev->core_device);
1292     if (ret)
1293         goto out_free;
1294     return 0;
1295 
1296 out_free:
1297     vfio_pci_core_uninit_device(&hisi_acc_vdev->core_device);
1298     kfree(hisi_acc_vdev);
1299     return ret;
1300 }
1301 
1302 static void hisi_acc_vfio_pci_remove(struct pci_dev *pdev)
1303 {
1304     struct hisi_acc_vf_core_device *hisi_acc_vdev = hssi_acc_drvdata(pdev);
1305 
1306     vfio_pci_core_unregister_device(&hisi_acc_vdev->core_device);
1307     vfio_pci_core_uninit_device(&hisi_acc_vdev->core_device);
1308     kfree(hisi_acc_vdev);
1309 }
1310 
1311 static const struct pci_device_id hisi_acc_vfio_pci_table[] = {
1312     { PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_SEC_VF) },
1313     { PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_HPRE_VF) },
1314     { PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_ZIP_VF) },
1315     { }
1316 };
1317 
1318 MODULE_DEVICE_TABLE(pci, hisi_acc_vfio_pci_table);
1319 
1320 static const struct pci_error_handlers hisi_acc_vf_err_handlers = {
1321     .reset_done = hisi_acc_vf_pci_aer_reset_done,
1322     .error_detected = vfio_pci_core_aer_err_detected,
1323 };
1324 
1325 static struct pci_driver hisi_acc_vfio_pci_driver = {
1326     .name = KBUILD_MODNAME,
1327     .id_table = hisi_acc_vfio_pci_table,
1328     .probe = hisi_acc_vfio_pci_probe,
1329     .remove = hisi_acc_vfio_pci_remove,
1330     .err_handler = &hisi_acc_vf_err_handlers,
1331     .driver_managed_dma = true,
1332 };
1333 
1334 module_pci_driver(hisi_acc_vfio_pci_driver);
1335 
1336 MODULE_LICENSE("GPL v2");
1337 MODULE_AUTHOR("Liu Longfang <liulongfang@huawei.com>");
1338 MODULE_AUTHOR("Shameer Kolothum <shameerali.kolothum.thodi@huawei.com>");
1339 MODULE_DESCRIPTION("HiSilicon VFIO PCI - VFIO PCI driver with live migration support for HiSilicon ACC device family");
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!