cavium/nitrox/nitrox_main.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 #include <linux/aer.h>
0003 #include <linux/delay.h>
0004 #include <linux/firmware.h>
0005 #include <linux/list.h>
0006 #include <linux/module.h>
0007 #include <linux/mutex.h>
0008 #include <linux/pci.h>
0009 #include <linux/pci_ids.h>
0010
0011 #include "nitrox_dev.h"
0012 #include "nitrox_common.h"
0013 #include "nitrox_csr.h"
0014 #include "nitrox_hal.h"
0015 #include "nitrox_isr.h"
0016 #include "nitrox_debugfs.h"
0017
0018 #define CNN55XX_DEV_ID  0x12
0019 #define UCODE_HLEN 48
0020 #define DEFAULT_SE_GROUP 0
0021 #define DEFAULT_AE_GROUP 0
0022
0023 #define DRIVER_VERSION "1.2"
0024 #define CNN55XX_UCD_BLOCK_SIZE 32768
0025 #define CNN55XX_MAX_UCODE_SIZE (CNN55XX_UCD_BLOCK_SIZE * 2)
0026 #define FW_DIR "cavium/"
0027 /* SE microcode */
0028 #define SE_FW   FW_DIR "cnn55xx_se.fw"
0029 /* AE microcode */
0030 #define AE_FW   FW_DIR "cnn55xx_ae.fw"
0031
0032 static const char nitrox_driver_name[] = "CNN55XX";
0033
0034 static LIST_HEAD(ndevlist);
0035 static DEFINE_MUTEX(devlist_lock);
0036 static unsigned int num_devices;
0037
0038 /*
0039  * nitrox_pci_tbl - PCI Device ID Table
0040  */
0041 static const struct pci_device_id nitrox_pci_tbl[] = {
0042     {PCI_VDEVICE(CAVIUM, CNN55XX_DEV_ID), 0},
0043     /* required last entry */
0044     {0, }
0045 };
0046 MODULE_DEVICE_TABLE(pci, nitrox_pci_tbl);
0047
0048 static unsigned int qlen = DEFAULT_CMD_QLEN;
0049 module_param(qlen, uint, 0644);
0050 MODULE_PARM_DESC(qlen, "Command queue length - default 2048");
0051
0052 /**
0053  * struct ucode - Firmware Header
0054  * @id: microcode ID
0055  * @version: firmware version
0056  * @code_size: code section size
0057  * @raz: alignment
0058  * @code: code section
0059  */
0060 struct ucode {
0061     u8 id;
0062     char version[VERSION_LEN - 1];
0063     __be32 code_size;
0064     u8 raz[12];
0065     u64 code[];
0066 };
0067
0068 /*
0069  * write_to_ucd_unit - Write Firmware to NITROX UCD unit
0070  */
0071 static void write_to_ucd_unit(struct nitrox_device *ndev, u32 ucode_size,
0072                   u64 *ucode_data, int block_num)
0073 {
0074     u32 code_size;
0075     u64 offset, data;
0076     int i = 0;
0077
0078     /*
0079      * UCD structure
0080      *
0081      *  -------------
0082      *  |    BLK 7  |
0083      *  -------------
0084      *  |    BLK 6  |
0085      *  -------------
0086      *  |    ...    |
0087      *  -------------
0088      *  |    BLK 0  |
0089      *  -------------
0090      *  Total of 8 blocks, each size 32KB
0091      */
0092
0093     /* set the block number */
0094     offset = UCD_UCODE_LOAD_BLOCK_NUM;
0095     nitrox_write_csr(ndev, offset, block_num);
0096
0097     code_size = roundup(ucode_size, 16);
0098     while (code_size) {
0099         data = ucode_data[i];
0100         /* write 8 bytes at a time */
0101         offset = UCD_UCODE_LOAD_IDX_DATAX(i);
0102         nitrox_write_csr(ndev, offset, data);
0103         code_size -= 8;
0104         i++;
0105     }
0106
0107     usleep_range(300, 400);
0108 }
0109
0110 static int nitrox_load_fw(struct nitrox_device *ndev)
0111 {
0112     const struct firmware *fw;
0113     const char *fw_name;
0114     struct ucode *ucode;
0115     u64 *ucode_data;
0116     u64 offset;
0117     union ucd_core_eid_ucode_block_num core_2_eid_val;
0118     union aqm_grp_execmsk_lo aqm_grp_execmask_lo;
0119     union aqm_grp_execmsk_hi aqm_grp_execmask_hi;
0120     u32 ucode_size;
0121     int ret, i = 0;
0122
0123     fw_name = SE_FW;
0124     dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name);
0125
0126     ret = request_firmware(&fw, fw_name, DEV(ndev));
0127     if (ret < 0) {
0128         dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name);
0129         return ret;
0130     }
0131
0132     ucode = (struct ucode *)fw->data;
0133
0134     ucode_size = be32_to_cpu(ucode->code_size) * 2;
0135     if (!ucode_size || ucode_size > CNN55XX_MAX_UCODE_SIZE) {
0136         dev_err(DEV(ndev), "Invalid ucode size: %u for firmware %s\n",
0137             ucode_size, fw_name);
0138         release_firmware(fw);
0139         return -EINVAL;
0140     }
0141     ucode_data = ucode->code;
0142
0143     /* copy the firmware version */
0144     memcpy(&ndev->hw.fw_name[0][0], ucode->version, (VERSION_LEN - 2));
0145     ndev->hw.fw_name[0][VERSION_LEN - 1] = '\0';
0146
0147     /* Load SE Firmware on UCD Block 0 */
0148     write_to_ucd_unit(ndev, ucode_size, ucode_data, 0);
0149
0150     release_firmware(fw);
0151
0152     /* put all SE cores in DEFAULT_SE_GROUP */
0153     offset = POM_GRP_EXECMASKX(DEFAULT_SE_GROUP);
0154     nitrox_write_csr(ndev, offset, (~0ULL));
0155
0156     /* write block number and firmware length
0157      * bit:<2:0> block number
0158      * bit:3 is set SE uses 32KB microcode
0159      * bit:3 is clear SE uses 64KB microcode
0160      */
0161     core_2_eid_val.value = 0ULL;
0162     core_2_eid_val.ucode_blk = 0;
0163     if (ucode_size <= CNN55XX_UCD_BLOCK_SIZE)
0164         core_2_eid_val.ucode_len = 1;
0165     else
0166         core_2_eid_val.ucode_len = 0;
0167
0168     for (i = 0; i < ndev->hw.se_cores; i++) {
0169         offset = UCD_SE_EID_UCODE_BLOCK_NUMX(i);
0170         nitrox_write_csr(ndev, offset, core_2_eid_val.value);
0171     }
0172
0173
0174     fw_name = AE_FW;
0175     dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name);
0176
0177     ret = request_firmware(&fw, fw_name, DEV(ndev));
0178     if (ret < 0) {
0179         dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name);
0180         return ret;
0181     }
0182
0183     ucode = (struct ucode *)fw->data;
0184
0185     ucode_size = be32_to_cpu(ucode->code_size) * 2;
0186     if (!ucode_size || ucode_size > CNN55XX_MAX_UCODE_SIZE) {
0187         dev_err(DEV(ndev), "Invalid ucode size: %u for firmware %s\n",
0188             ucode_size, fw_name);
0189         release_firmware(fw);
0190         return -EINVAL;
0191     }
0192     ucode_data = ucode->code;
0193
0194     /* copy the firmware version */
0195     memcpy(&ndev->hw.fw_name[1][0], ucode->version, (VERSION_LEN - 2));
0196     ndev->hw.fw_name[1][VERSION_LEN - 1] = '\0';
0197
0198     /* Load AE Firmware on UCD Block 2 */
0199     write_to_ucd_unit(ndev, ucode_size, ucode_data, 2);
0200
0201     release_firmware(fw);
0202
0203     /* put all AE cores in DEFAULT_AE_GROUP */
0204     offset = AQM_GRP_EXECMSK_LOX(DEFAULT_AE_GROUP);
0205     aqm_grp_execmask_lo.exec_0_to_39 = 0xFFFFFFFFFFULL;
0206     nitrox_write_csr(ndev, offset, aqm_grp_execmask_lo.value);
0207     offset = AQM_GRP_EXECMSK_HIX(DEFAULT_AE_GROUP);
0208     aqm_grp_execmask_hi.exec_40_to_79 = 0xFFFFFFFFFFULL;
0209     nitrox_write_csr(ndev, offset, aqm_grp_execmask_hi.value);
0210
0211     /* write block number and firmware length
0212      * bit:<2:0> block number
0213      * bit:3 is set AE uses 32KB microcode
0214      * bit:3 is clear AE uses 64KB microcode
0215      */
0216     core_2_eid_val.value = 0ULL;
0217     core_2_eid_val.ucode_blk = 2;
0218     if (ucode_size <= CNN55XX_UCD_BLOCK_SIZE)
0219         core_2_eid_val.ucode_len = 1;
0220     else
0221         core_2_eid_val.ucode_len = 0;
0222
0223     for (i = 0; i < ndev->hw.ae_cores; i++) {
0224         offset = UCD_AE_EID_UCODE_BLOCK_NUMX(i);
0225         nitrox_write_csr(ndev, offset, core_2_eid_val.value);
0226     }
0227
0228     return 0;
0229 }
0230
0231 /**
0232  * nitrox_add_to_devlist - add NITROX device to global device list
0233  * @ndev: NITROX device
0234  */
0235 static int nitrox_add_to_devlist(struct nitrox_device *ndev)
0236 {
0237     struct nitrox_device *dev;
0238     int ret = 0;
0239
0240     INIT_LIST_HEAD(&ndev->list);
0241     refcount_set(&ndev->refcnt, 1);
0242
0243     mutex_lock(&devlist_lock);
0244     list_for_each_entry(dev, &ndevlist, list) {
0245         if (dev == ndev) {
0246             ret = -EEXIST;
0247             goto unlock;
0248         }
0249     }
0250     ndev->idx = num_devices++;
0251     list_add_tail(&ndev->list, &ndevlist);
0252 unlock:
0253     mutex_unlock(&devlist_lock);
0254     return ret;
0255 }
0256
0257 /**
0258  * nitrox_remove_from_devlist - remove NITROX device from
0259  *   global device list
0260  * @ndev: NITROX device
0261  */
0262 static void nitrox_remove_from_devlist(struct nitrox_device *ndev)
0263 {
0264     mutex_lock(&devlist_lock);
0265     list_del(&ndev->list);
0266     num_devices--;
0267     mutex_unlock(&devlist_lock);
0268 }
0269
0270 struct nitrox_device *nitrox_get_first_device(void)
0271 {
0272     struct nitrox_device *ndev = NULL, *iter;
0273
0274     mutex_lock(&devlist_lock);
0275     list_for_each_entry(iter, &ndevlist, list) {
0276         if (nitrox_ready(iter)) {
0277             ndev = iter;
0278             break;
0279         }
0280     }
0281     mutex_unlock(&devlist_lock);
0282     if (!ndev)
0283         return NULL;
0284
0285     refcount_inc(&ndev->refcnt);
0286     /* barrier to sync with other cpus */
0287     smp_mb__after_atomic();
0288     return ndev;
0289 }
0290
0291 void nitrox_put_device(struct nitrox_device *ndev)
0292 {
0293     if (!ndev)
0294         return;
0295
0296     refcount_dec(&ndev->refcnt);
0297     /* barrier to sync with other cpus */
0298     smp_mb__after_atomic();
0299 }
0300
0301 static int nitrox_device_flr(struct pci_dev *pdev)
0302 {
0303     int pos = 0;
0304
0305     pos = pci_save_state(pdev);
0306     if (pos) {
0307         dev_err(&pdev->dev, "Failed to save pci state\n");
0308         return -ENOMEM;
0309     }
0310
0311     pcie_reset_flr(pdev, PCI_RESET_DO_RESET);
0312
0313     pci_restore_state(pdev);
0314
0315     return 0;
0316 }
0317
0318 static int nitrox_pf_sw_init(struct nitrox_device *ndev)
0319 {
0320     int err;
0321
0322     err = nitrox_common_sw_init(ndev);
0323     if (err)
0324         return err;
0325
0326     err = nitrox_register_interrupts(ndev);
0327     if (err)
0328         nitrox_common_sw_cleanup(ndev);
0329
0330     return err;
0331 }
0332
0333 static void nitrox_pf_sw_cleanup(struct nitrox_device *ndev)
0334 {
0335     nitrox_unregister_interrupts(ndev);
0336     nitrox_common_sw_cleanup(ndev);
0337 }
0338
0339 /**
0340  * nitrox_bist_check - Check NITROX BIST registers status
0341  * @ndev: NITROX device
0342  */
0343 static int nitrox_bist_check(struct nitrox_device *ndev)
0344 {
0345     u64 value = 0;
0346     int i;
0347
0348     for (i = 0; i < NR_CLUSTERS; i++) {
0349         value += nitrox_read_csr(ndev, EMU_BIST_STATUSX(i));
0350         value += nitrox_read_csr(ndev, EFL_CORE_BIST_REGX(i));
0351     }
0352     value += nitrox_read_csr(ndev, UCD_BIST_STATUS);
0353     value += nitrox_read_csr(ndev, NPS_CORE_BIST_REG);
0354     value += nitrox_read_csr(ndev, NPS_CORE_NPC_BIST_REG);
0355     value += nitrox_read_csr(ndev, NPS_PKT_SLC_BIST_REG);
0356     value += nitrox_read_csr(ndev, NPS_PKT_IN_BIST_REG);
0357     value += nitrox_read_csr(ndev, POM_BIST_REG);
0358     value += nitrox_read_csr(ndev, BMI_BIST_REG);
0359     value += nitrox_read_csr(ndev, EFL_TOP_BIST_STAT);
0360     value += nitrox_read_csr(ndev, BMO_BIST_REG);
0361     value += nitrox_read_csr(ndev, LBC_BIST_STATUS);
0362     value += nitrox_read_csr(ndev, PEM_BIST_STATUSX(0));
0363     if (value)
0364         return -EIO;
0365     return 0;
0366 }
0367
0368 static int nitrox_pf_hw_init(struct nitrox_device *ndev)
0369 {
0370     int err;
0371
0372     err = nitrox_bist_check(ndev);
0373     if (err) {
0374         dev_err(&ndev->pdev->dev, "BIST check failed\n");
0375         return err;
0376     }
0377     /* get cores information */
0378     nitrox_get_hwinfo(ndev);
0379
0380     nitrox_config_nps_core_unit(ndev);
0381     nitrox_config_aqm_unit(ndev);
0382     nitrox_config_nps_pkt_unit(ndev);
0383     nitrox_config_pom_unit(ndev);
0384     nitrox_config_efl_unit(ndev);
0385     /* configure IO units */
0386     nitrox_config_bmi_unit(ndev);
0387     nitrox_config_bmo_unit(ndev);
0388     /* configure Local Buffer Cache */
0389     nitrox_config_lbc_unit(ndev);
0390     nitrox_config_rand_unit(ndev);
0391
0392     /* load firmware on cores */
0393     err = nitrox_load_fw(ndev);
0394     if (err)
0395         return err;
0396
0397     nitrox_config_emu_unit(ndev);
0398
0399     return 0;
0400 }
0401
0402 /**
0403  * nitrox_probe - NITROX Initialization function.
0404  * @pdev: PCI device information struct
0405  * @id: entry in nitrox_pci_tbl
0406  *
0407  * Return: 0, if the driver is bound to the device, or
0408  *         a negative error if there is failure.
0409  */
0410 static int nitrox_probe(struct pci_dev *pdev,
0411             const struct pci_device_id *id)
0412 {
0413     struct nitrox_device *ndev;
0414     int err;
0415
0416     dev_info_once(&pdev->dev, "%s driver version %s\n",
0417               nitrox_driver_name, DRIVER_VERSION);
0418
0419     err = pci_enable_device_mem(pdev);
0420     if (err)
0421         return err;
0422
0423     /* do FLR */
0424     err = nitrox_device_flr(pdev);
0425     if (err) {
0426         dev_err(&pdev->dev, "FLR failed\n");
0427         goto flr_fail;
0428     }
0429
0430     if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
0431         dev_dbg(&pdev->dev, "DMA to 64-BIT address\n");
0432     } else {
0433         err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
0434         if (err) {
0435             dev_err(&pdev->dev, "DMA configuration failed\n");
0436             goto flr_fail;
0437         }
0438     }
0439
0440     err = pci_request_mem_regions(pdev, nitrox_driver_name);
0441     if (err)
0442         goto flr_fail;
0443     pci_set_master(pdev);
0444
0445     ndev = kzalloc(sizeof(*ndev), GFP_KERNEL);
0446     if (!ndev) {
0447         err = -ENOMEM;
0448         goto ndev_fail;
0449     }
0450
0451     pci_set_drvdata(pdev, ndev);
0452     ndev->pdev = pdev;
0453
0454     /* add to device list */
0455     nitrox_add_to_devlist(ndev);
0456
0457     ndev->hw.vendor_id = pdev->vendor;
0458     ndev->hw.device_id = pdev->device;
0459     ndev->hw.revision_id = pdev->revision;
0460     /* command timeout in jiffies */
0461     ndev->timeout = msecs_to_jiffies(CMD_TIMEOUT);
0462     ndev->node = dev_to_node(&pdev->dev);
0463     if (ndev->node == NUMA_NO_NODE)
0464         ndev->node = 0;
0465
0466     ndev->bar_addr = ioremap(pci_resource_start(pdev, 0),
0467                  pci_resource_len(pdev, 0));
0468     if (!ndev->bar_addr) {
0469         err = -EIO;
0470         goto ioremap_err;
0471     }
0472     /* allocate command queus based on cpus, max queues are 64 */
0473     ndev->nr_queues = min_t(u32, MAX_PF_QUEUES, num_online_cpus());
0474     ndev->qlen = qlen;
0475
0476     err = nitrox_pf_sw_init(ndev);
0477     if (err)
0478         goto pf_sw_fail;
0479
0480     err = nitrox_pf_hw_init(ndev);
0481     if (err)
0482         goto pf_hw_fail;
0483
0484     nitrox_debugfs_init(ndev);
0485
0486     /* clear the statistics */
0487     atomic64_set(&ndev->stats.posted, 0);
0488     atomic64_set(&ndev->stats.completed, 0);
0489     atomic64_set(&ndev->stats.dropped, 0);
0490
0491     atomic_set(&ndev->state, __NDEV_READY);
0492     /* barrier to sync with other cpus */
0493     smp_mb__after_atomic();
0494
0495     err = nitrox_crypto_register();
0496     if (err)
0497         goto crypto_fail;
0498
0499     return 0;
0500
0501 crypto_fail:
0502     nitrox_debugfs_exit(ndev);
0503     atomic_set(&ndev->state, __NDEV_NOT_READY);
0504     /* barrier to sync with other cpus */
0505     smp_mb__after_atomic();
0506 pf_hw_fail:
0507     nitrox_pf_sw_cleanup(ndev);
0508 pf_sw_fail:
0509     iounmap(ndev->bar_addr);
0510 ioremap_err:
0511     nitrox_remove_from_devlist(ndev);
0512     kfree(ndev);
0513     pci_set_drvdata(pdev, NULL);
0514 ndev_fail:
0515     pci_release_mem_regions(pdev);
0516 flr_fail:
0517     pci_disable_device(pdev);
0518     return err;
0519 }
0520
0521 /**
0522  * nitrox_remove - Unbind the driver from the device.
0523  * @pdev: PCI device information struct
0524  */
0525 static void nitrox_remove(struct pci_dev *pdev)
0526 {
0527     struct nitrox_device *ndev = pci_get_drvdata(pdev);
0528
0529     if (!ndev)
0530         return;
0531
0532     if (!refcount_dec_and_test(&ndev->refcnt)) {
0533         dev_err(DEV(ndev), "Device refcnt not zero (%d)\n",
0534             refcount_read(&ndev->refcnt));
0535         return;
0536     }
0537
0538     dev_info(DEV(ndev), "Removing Device %x:%x\n",
0539          ndev->hw.vendor_id, ndev->hw.device_id);
0540
0541     atomic_set(&ndev->state, __NDEV_NOT_READY);
0542     /* barrier to sync with other cpus */
0543     smp_mb__after_atomic();
0544
0545     nitrox_remove_from_devlist(ndev);
0546
0547     /* disable SR-IOV */
0548     nitrox_sriov_configure(pdev, 0);
0549     nitrox_crypto_unregister();
0550     nitrox_debugfs_exit(ndev);
0551     nitrox_pf_sw_cleanup(ndev);
0552
0553     iounmap(ndev->bar_addr);
0554     kfree(ndev);
0555
0556     pci_set_drvdata(pdev, NULL);
0557     pci_release_mem_regions(pdev);
0558     pci_disable_device(pdev);
0559 }
0560
0561 static void nitrox_shutdown(struct pci_dev *pdev)
0562 {
0563     pci_set_drvdata(pdev, NULL);
0564     pci_release_mem_regions(pdev);
0565     pci_disable_device(pdev);
0566 }
0567
0568 static struct pci_driver nitrox_driver = {
0569     .name = nitrox_driver_name,
0570     .id_table = nitrox_pci_tbl,
0571     .probe = nitrox_probe,
0572     .remove = nitrox_remove,
0573     .shutdown = nitrox_shutdown,
0574     .sriov_configure = nitrox_sriov_configure,
0575 };
0576
0577 module_pci_driver(nitrox_driver);
0578
0579 MODULE_AUTHOR("Srikanth Jampala <Jampala.Srikanth@cavium.com>");
0580 MODULE_DESCRIPTION("Cavium CNN55XX PF Driver" DRIVER_VERSION " ");
0581 MODULE_LICENSE("GPL");
0582 MODULE_VERSION(DRIVER_VERSION);
0583 MODULE_FIRMWARE(SE_FW);