Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Copyright (C) 2016 Cavium, Inc.
0004  */
0005 
0006 #include <linux/interrupt.h>
0007 #include <linux/module.h>
0008 
0009 #include "cptvf.h"
0010 
0011 #define DRV_NAME    "thunder-cptvf"
0012 #define DRV_VERSION "1.0"
0013 
0014 struct cptvf_wqe {
0015     struct tasklet_struct twork;
0016     void *cptvf;
0017     u32 qno;
0018 };
0019 
0020 struct cptvf_wqe_info {
0021     struct cptvf_wqe vq_wqe[CPT_NUM_QS_PER_VF];
0022 };
0023 
0024 static void vq_work_handler(unsigned long data)
0025 {
0026     struct cptvf_wqe_info *cwqe_info = (struct cptvf_wqe_info *)data;
0027     struct cptvf_wqe *cwqe = &cwqe_info->vq_wqe[0];
0028 
0029     vq_post_process(cwqe->cptvf, cwqe->qno);
0030 }
0031 
0032 static int init_worker_threads(struct cpt_vf *cptvf)
0033 {
0034     struct pci_dev *pdev = cptvf->pdev;
0035     struct cptvf_wqe_info *cwqe_info;
0036     int i;
0037 
0038     cwqe_info = kzalloc(sizeof(*cwqe_info), GFP_KERNEL);
0039     if (!cwqe_info)
0040         return -ENOMEM;
0041 
0042     if (cptvf->nr_queues) {
0043         dev_info(&pdev->dev, "Creating VQ worker threads (%d)\n",
0044              cptvf->nr_queues);
0045     }
0046 
0047     for (i = 0; i < cptvf->nr_queues; i++) {
0048         tasklet_init(&cwqe_info->vq_wqe[i].twork, vq_work_handler,
0049                  (u64)cwqe_info);
0050         cwqe_info->vq_wqe[i].qno = i;
0051         cwqe_info->vq_wqe[i].cptvf = cptvf;
0052     }
0053 
0054     cptvf->wqe_info = cwqe_info;
0055 
0056     return 0;
0057 }
0058 
0059 static void cleanup_worker_threads(struct cpt_vf *cptvf)
0060 {
0061     struct cptvf_wqe_info *cwqe_info;
0062     struct pci_dev *pdev = cptvf->pdev;
0063     int i;
0064 
0065     cwqe_info = (struct cptvf_wqe_info *)cptvf->wqe_info;
0066     if (!cwqe_info)
0067         return;
0068 
0069     if (cptvf->nr_queues) {
0070         dev_info(&pdev->dev, "Cleaning VQ worker threads (%u)\n",
0071              cptvf->nr_queues);
0072     }
0073 
0074     for (i = 0; i < cptvf->nr_queues; i++)
0075         tasklet_kill(&cwqe_info->vq_wqe[i].twork);
0076 
0077     kfree_sensitive(cwqe_info);
0078     cptvf->wqe_info = NULL;
0079 }
0080 
0081 static void free_pending_queues(struct pending_qinfo *pqinfo)
0082 {
0083     int i;
0084     struct pending_queue *queue;
0085 
0086     for_each_pending_queue(pqinfo, queue, i) {
0087         if (!queue->head)
0088             continue;
0089 
0090         /* free single queue */
0091         kfree_sensitive((queue->head));
0092 
0093         queue->front = 0;
0094         queue->rear = 0;
0095 
0096         return;
0097     }
0098 
0099     pqinfo->qlen = 0;
0100     pqinfo->nr_queues = 0;
0101 }
0102 
0103 static int alloc_pending_queues(struct pending_qinfo *pqinfo, u32 qlen,
0104                 u32 nr_queues)
0105 {
0106     u32 i;
0107     int ret;
0108     struct pending_queue *queue = NULL;
0109 
0110     pqinfo->nr_queues = nr_queues;
0111     pqinfo->qlen = qlen;
0112 
0113     for_each_pending_queue(pqinfo, queue, i) {
0114         queue->head = kcalloc(qlen, sizeof(*queue->head), GFP_KERNEL);
0115         if (!queue->head) {
0116             ret = -ENOMEM;
0117             goto pending_qfail;
0118         }
0119 
0120         queue->front = 0;
0121         queue->rear = 0;
0122         atomic64_set((&queue->pending_count), (0));
0123 
0124         /* init queue spin lock */
0125         spin_lock_init(&queue->lock);
0126     }
0127 
0128     return 0;
0129 
0130 pending_qfail:
0131     free_pending_queues(pqinfo);
0132 
0133     return ret;
0134 }
0135 
0136 static int init_pending_queues(struct cpt_vf *cptvf, u32 qlen, u32 nr_queues)
0137 {
0138     struct pci_dev *pdev = cptvf->pdev;
0139     int ret;
0140 
0141     if (!nr_queues)
0142         return 0;
0143 
0144     ret = alloc_pending_queues(&cptvf->pqinfo, qlen, nr_queues);
0145     if (ret) {
0146         dev_err(&pdev->dev, "failed to setup pending queues (%u)\n",
0147             nr_queues);
0148         return ret;
0149     }
0150 
0151     return 0;
0152 }
0153 
0154 static void cleanup_pending_queues(struct cpt_vf *cptvf)
0155 {
0156     struct pci_dev *pdev = cptvf->pdev;
0157 
0158     if (!cptvf->nr_queues)
0159         return;
0160 
0161     dev_info(&pdev->dev, "Cleaning VQ pending queue (%u)\n",
0162          cptvf->nr_queues);
0163     free_pending_queues(&cptvf->pqinfo);
0164 }
0165 
0166 static void free_command_queues(struct cpt_vf *cptvf,
0167                 struct command_qinfo *cqinfo)
0168 {
0169     int i;
0170     struct command_queue *queue = NULL;
0171     struct command_chunk *chunk = NULL;
0172     struct pci_dev *pdev = cptvf->pdev;
0173     struct hlist_node *node;
0174 
0175     /* clean up for each queue */
0176     for (i = 0; i < cptvf->nr_queues; i++) {
0177         queue = &cqinfo->queue[i];
0178         if (hlist_empty(&cqinfo->queue[i].chead))
0179             continue;
0180 
0181         hlist_for_each_entry_safe(chunk, node, &cqinfo->queue[i].chead,
0182                       nextchunk) {
0183             dma_free_coherent(&pdev->dev, chunk->size,
0184                       chunk->head,
0185                       chunk->dma_addr);
0186             chunk->head = NULL;
0187             chunk->dma_addr = 0;
0188             hlist_del(&chunk->nextchunk);
0189             kfree_sensitive(chunk);
0190         }
0191 
0192         queue->nchunks = 0;
0193         queue->idx = 0;
0194     }
0195 
0196     /* common cleanup */
0197     cqinfo->cmd_size = 0;
0198 }
0199 
0200 static int alloc_command_queues(struct cpt_vf *cptvf,
0201                 struct command_qinfo *cqinfo, size_t cmd_size,
0202                 u32 qlen)
0203 {
0204     int i;
0205     size_t q_size;
0206     struct command_queue *queue = NULL;
0207     struct pci_dev *pdev = cptvf->pdev;
0208 
0209     /* common init */
0210     cqinfo->cmd_size = cmd_size;
0211     /* Qsize in dwords, needed for SADDR config, 1-next chunk pointer */
0212     cptvf->qsize = min(qlen, cqinfo->qchunksize) *
0213             CPT_NEXT_CHUNK_PTR_SIZE + 1;
0214     /* Qsize in bytes to create space for alignment */
0215     q_size = qlen * cqinfo->cmd_size;
0216 
0217     /* per queue initialization */
0218     for (i = 0; i < cptvf->nr_queues; i++) {
0219         size_t c_size = 0;
0220         size_t rem_q_size = q_size;
0221         struct command_chunk *curr = NULL, *first = NULL, *last = NULL;
0222         u32 qcsize_bytes = cqinfo->qchunksize * cqinfo->cmd_size;
0223 
0224         queue = &cqinfo->queue[i];
0225         INIT_HLIST_HEAD(&cqinfo->queue[i].chead);
0226         do {
0227             curr = kzalloc(sizeof(*curr), GFP_KERNEL);
0228             if (!curr)
0229                 goto cmd_qfail;
0230 
0231             c_size = (rem_q_size > qcsize_bytes) ? qcsize_bytes :
0232                     rem_q_size;
0233             curr->head = dma_alloc_coherent(&pdev->dev,
0234                             c_size + CPT_NEXT_CHUNK_PTR_SIZE,
0235                             &curr->dma_addr,
0236                             GFP_KERNEL);
0237             if (!curr->head) {
0238                 dev_err(&pdev->dev, "Command Q (%d) chunk (%d) allocation failed\n",
0239                     i, queue->nchunks);
0240                 kfree(curr);
0241                 goto cmd_qfail;
0242             }
0243 
0244             curr->size = c_size;
0245             if (queue->nchunks == 0) {
0246                 hlist_add_head(&curr->nextchunk,
0247                            &cqinfo->queue[i].chead);
0248                 first = curr;
0249             } else {
0250                 hlist_add_behind(&curr->nextchunk,
0251                          &last->nextchunk);
0252             }
0253 
0254             queue->nchunks++;
0255             rem_q_size -= c_size;
0256             if (last)
0257                 *((u64 *)(&last->head[last->size])) = (u64)curr->dma_addr;
0258 
0259             last = curr;
0260         } while (rem_q_size);
0261 
0262         /* Make the queue circular */
0263         /* Tie back last chunk entry to head */
0264         curr = first;
0265         *((u64 *)(&last->head[last->size])) = (u64)curr->dma_addr;
0266         queue->qhead = curr;
0267         spin_lock_init(&queue->lock);
0268     }
0269     return 0;
0270 
0271 cmd_qfail:
0272     free_command_queues(cptvf, cqinfo);
0273     return -ENOMEM;
0274 }
0275 
0276 static int init_command_queues(struct cpt_vf *cptvf, u32 qlen)
0277 {
0278     struct pci_dev *pdev = cptvf->pdev;
0279     int ret;
0280 
0281     /* setup AE command queues */
0282     ret = alloc_command_queues(cptvf, &cptvf->cqinfo, CPT_INST_SIZE,
0283                    qlen);
0284     if (ret) {
0285         dev_err(&pdev->dev, "failed to allocate AE command queues (%u)\n",
0286             cptvf->nr_queues);
0287         return ret;
0288     }
0289 
0290     return ret;
0291 }
0292 
0293 static void cleanup_command_queues(struct cpt_vf *cptvf)
0294 {
0295     struct pci_dev *pdev = cptvf->pdev;
0296 
0297     if (!cptvf->nr_queues)
0298         return;
0299 
0300     dev_info(&pdev->dev, "Cleaning VQ command queue (%u)\n",
0301          cptvf->nr_queues);
0302     free_command_queues(cptvf, &cptvf->cqinfo);
0303 }
0304 
0305 static void cptvf_sw_cleanup(struct cpt_vf *cptvf)
0306 {
0307     cleanup_worker_threads(cptvf);
0308     cleanup_pending_queues(cptvf);
0309     cleanup_command_queues(cptvf);
0310 }
0311 
0312 static int cptvf_sw_init(struct cpt_vf *cptvf, u32 qlen, u32 nr_queues)
0313 {
0314     struct pci_dev *pdev = cptvf->pdev;
0315     int ret = 0;
0316     u32 max_dev_queues = 0;
0317 
0318     max_dev_queues = CPT_NUM_QS_PER_VF;
0319     /* possible cpus */
0320     nr_queues = min_t(u32, nr_queues, max_dev_queues);
0321     cptvf->nr_queues = nr_queues;
0322 
0323     ret = init_command_queues(cptvf, qlen);
0324     if (ret) {
0325         dev_err(&pdev->dev, "Failed to setup command queues (%u)\n",
0326             nr_queues);
0327         return ret;
0328     }
0329 
0330     ret = init_pending_queues(cptvf, qlen, nr_queues);
0331     if (ret) {
0332         dev_err(&pdev->dev, "Failed to setup pending queues (%u)\n",
0333             nr_queues);
0334         goto setup_pqfail;
0335     }
0336 
0337     /* Create worker threads for BH processing */
0338     ret = init_worker_threads(cptvf);
0339     if (ret) {
0340         dev_err(&pdev->dev, "Failed to setup worker threads\n");
0341         goto init_work_fail;
0342     }
0343 
0344     return 0;
0345 
0346 init_work_fail:
0347     cleanup_worker_threads(cptvf);
0348     cleanup_pending_queues(cptvf);
0349 
0350 setup_pqfail:
0351     cleanup_command_queues(cptvf);
0352 
0353     return ret;
0354 }
0355 
0356 static void cptvf_free_irq_affinity(struct cpt_vf *cptvf, int vec)
0357 {
0358     irq_set_affinity_hint(pci_irq_vector(cptvf->pdev, vec), NULL);
0359     free_cpumask_var(cptvf->affinity_mask[vec]);
0360 }
0361 
0362 static void cptvf_write_vq_ctl(struct cpt_vf *cptvf, bool val)
0363 {
0364     union cptx_vqx_ctl vqx_ctl;
0365 
0366     vqx_ctl.u = cpt_read_csr64(cptvf->reg_base, CPTX_VQX_CTL(0, 0));
0367     vqx_ctl.s.ena = val;
0368     cpt_write_csr64(cptvf->reg_base, CPTX_VQX_CTL(0, 0), vqx_ctl.u);
0369 }
0370 
0371 void cptvf_write_vq_doorbell(struct cpt_vf *cptvf, u32 val)
0372 {
0373     union cptx_vqx_doorbell vqx_dbell;
0374 
0375     vqx_dbell.u = cpt_read_csr64(cptvf->reg_base,
0376                      CPTX_VQX_DOORBELL(0, 0));
0377     vqx_dbell.s.dbell_cnt = val * 8; /* Num of Instructions * 8 words */
0378     cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DOORBELL(0, 0),
0379             vqx_dbell.u);
0380 }
0381 
0382 static void cptvf_write_vq_inprog(struct cpt_vf *cptvf, u8 val)
0383 {
0384     union cptx_vqx_inprog vqx_inprg;
0385 
0386     vqx_inprg.u = cpt_read_csr64(cptvf->reg_base, CPTX_VQX_INPROG(0, 0));
0387     vqx_inprg.s.inflight = val;
0388     cpt_write_csr64(cptvf->reg_base, CPTX_VQX_INPROG(0, 0), vqx_inprg.u);
0389 }
0390 
0391 static void cptvf_write_vq_done_numwait(struct cpt_vf *cptvf, u32 val)
0392 {
0393     union cptx_vqx_done_wait vqx_dwait;
0394 
0395     vqx_dwait.u = cpt_read_csr64(cptvf->reg_base,
0396                      CPTX_VQX_DONE_WAIT(0, 0));
0397     vqx_dwait.s.num_wait = val;
0398     cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_WAIT(0, 0),
0399             vqx_dwait.u);
0400 }
0401 
0402 static void cptvf_write_vq_done_timewait(struct cpt_vf *cptvf, u16 time)
0403 {
0404     union cptx_vqx_done_wait vqx_dwait;
0405 
0406     vqx_dwait.u = cpt_read_csr64(cptvf->reg_base,
0407                      CPTX_VQX_DONE_WAIT(0, 0));
0408     vqx_dwait.s.time_wait = time;
0409     cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_WAIT(0, 0),
0410             vqx_dwait.u);
0411 }
0412 
0413 static void cptvf_enable_swerr_interrupts(struct cpt_vf *cptvf)
0414 {
0415     union cptx_vqx_misc_ena_w1s vqx_misc_ena;
0416 
0417     vqx_misc_ena.u = cpt_read_csr64(cptvf->reg_base,
0418                     CPTX_VQX_MISC_ENA_W1S(0, 0));
0419     /* Set mbox(0) interupts for the requested vf */
0420     vqx_misc_ena.s.swerr = 1;
0421     cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_ENA_W1S(0, 0),
0422             vqx_misc_ena.u);
0423 }
0424 
0425 static void cptvf_enable_mbox_interrupts(struct cpt_vf *cptvf)
0426 {
0427     union cptx_vqx_misc_ena_w1s vqx_misc_ena;
0428 
0429     vqx_misc_ena.u = cpt_read_csr64(cptvf->reg_base,
0430                     CPTX_VQX_MISC_ENA_W1S(0, 0));
0431     /* Set mbox(0) interupts for the requested vf */
0432     vqx_misc_ena.s.mbox = 1;
0433     cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_ENA_W1S(0, 0),
0434             vqx_misc_ena.u);
0435 }
0436 
0437 static void cptvf_enable_done_interrupts(struct cpt_vf *cptvf)
0438 {
0439     union cptx_vqx_done_ena_w1s vqx_done_ena;
0440 
0441     vqx_done_ena.u = cpt_read_csr64(cptvf->reg_base,
0442                     CPTX_VQX_DONE_ENA_W1S(0, 0));
0443     /* Set DONE interrupt for the requested vf */
0444     vqx_done_ena.s.done = 1;
0445     cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_ENA_W1S(0, 0),
0446             vqx_done_ena.u);
0447 }
0448 
0449 static void cptvf_clear_dovf_intr(struct cpt_vf *cptvf)
0450 {
0451     union cptx_vqx_misc_int vqx_misc_int;
0452 
0453     vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base,
0454                     CPTX_VQX_MISC_INT(0, 0));
0455     /* W1C for the VF */
0456     vqx_misc_int.s.dovf = 1;
0457     cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0),
0458             vqx_misc_int.u);
0459 }
0460 
0461 static void cptvf_clear_irde_intr(struct cpt_vf *cptvf)
0462 {
0463     union cptx_vqx_misc_int vqx_misc_int;
0464 
0465     vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base,
0466                     CPTX_VQX_MISC_INT(0, 0));
0467     /* W1C for the VF */
0468     vqx_misc_int.s.irde = 1;
0469     cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0),
0470             vqx_misc_int.u);
0471 }
0472 
0473 static void cptvf_clear_nwrp_intr(struct cpt_vf *cptvf)
0474 {
0475     union cptx_vqx_misc_int vqx_misc_int;
0476 
0477     vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base,
0478                     CPTX_VQX_MISC_INT(0, 0));
0479     /* W1C for the VF */
0480     vqx_misc_int.s.nwrp = 1;
0481     cpt_write_csr64(cptvf->reg_base,
0482             CPTX_VQX_MISC_INT(0, 0), vqx_misc_int.u);
0483 }
0484 
0485 static void cptvf_clear_mbox_intr(struct cpt_vf *cptvf)
0486 {
0487     union cptx_vqx_misc_int vqx_misc_int;
0488 
0489     vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base,
0490                     CPTX_VQX_MISC_INT(0, 0));
0491     /* W1C for the VF */
0492     vqx_misc_int.s.mbox = 1;
0493     cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0),
0494             vqx_misc_int.u);
0495 }
0496 
0497 static void cptvf_clear_swerr_intr(struct cpt_vf *cptvf)
0498 {
0499     union cptx_vqx_misc_int vqx_misc_int;
0500 
0501     vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base,
0502                     CPTX_VQX_MISC_INT(0, 0));
0503     /* W1C for the VF */
0504     vqx_misc_int.s.swerr = 1;
0505     cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0),
0506             vqx_misc_int.u);
0507 }
0508 
0509 static u64 cptvf_read_vf_misc_intr_status(struct cpt_vf *cptvf)
0510 {
0511     return cpt_read_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0));
0512 }
0513 
0514 static irqreturn_t cptvf_misc_intr_handler(int irq, void *cptvf_irq)
0515 {
0516     struct cpt_vf *cptvf = (struct cpt_vf *)cptvf_irq;
0517     struct pci_dev *pdev = cptvf->pdev;
0518     u64 intr;
0519 
0520     intr = cptvf_read_vf_misc_intr_status(cptvf);
0521     /*Check for MISC interrupt types*/
0522     if (likely(intr & CPT_VF_INTR_MBOX_MASK)) {
0523         dev_dbg(&pdev->dev, "Mailbox interrupt 0x%llx on CPT VF %d\n",
0524             intr, cptvf->vfid);
0525         cptvf_handle_mbox_intr(cptvf);
0526         cptvf_clear_mbox_intr(cptvf);
0527     } else if (unlikely(intr & CPT_VF_INTR_DOVF_MASK)) {
0528         cptvf_clear_dovf_intr(cptvf);
0529         /*Clear doorbell count*/
0530         cptvf_write_vq_doorbell(cptvf, 0);
0531         dev_err(&pdev->dev, "Doorbell overflow error interrupt 0x%llx on CPT VF %d\n",
0532             intr, cptvf->vfid);
0533     } else if (unlikely(intr & CPT_VF_INTR_IRDE_MASK)) {
0534         cptvf_clear_irde_intr(cptvf);
0535         dev_err(&pdev->dev, "Instruction NCB read error interrupt 0x%llx on CPT VF %d\n",
0536             intr, cptvf->vfid);
0537     } else if (unlikely(intr & CPT_VF_INTR_NWRP_MASK)) {
0538         cptvf_clear_nwrp_intr(cptvf);
0539         dev_err(&pdev->dev, "NCB response write error interrupt 0x%llx on CPT VF %d\n",
0540             intr, cptvf->vfid);
0541     } else if (unlikely(intr & CPT_VF_INTR_SERR_MASK)) {
0542         cptvf_clear_swerr_intr(cptvf);
0543         dev_err(&pdev->dev, "Software error interrupt 0x%llx on CPT VF %d\n",
0544             intr, cptvf->vfid);
0545     } else {
0546         dev_err(&pdev->dev, "Unhandled interrupt in CPT VF %d\n",
0547             cptvf->vfid);
0548     }
0549 
0550     return IRQ_HANDLED;
0551 }
0552 
0553 static inline struct cptvf_wqe *get_cptvf_vq_wqe(struct cpt_vf *cptvf,
0554                          int qno)
0555 {
0556     struct cptvf_wqe_info *nwqe_info;
0557 
0558     if (unlikely(qno >= cptvf->nr_queues))
0559         return NULL;
0560     nwqe_info = (struct cptvf_wqe_info *)cptvf->wqe_info;
0561 
0562     return &nwqe_info->vq_wqe[qno];
0563 }
0564 
0565 static inline u32 cptvf_read_vq_done_count(struct cpt_vf *cptvf)
0566 {
0567     union cptx_vqx_done vqx_done;
0568 
0569     vqx_done.u = cpt_read_csr64(cptvf->reg_base, CPTX_VQX_DONE(0, 0));
0570     return vqx_done.s.done;
0571 }
0572 
0573 static inline void cptvf_write_vq_done_ack(struct cpt_vf *cptvf,
0574                        u32 ackcnt)
0575 {
0576     union cptx_vqx_done_ack vqx_dack_cnt;
0577 
0578     vqx_dack_cnt.u = cpt_read_csr64(cptvf->reg_base,
0579                     CPTX_VQX_DONE_ACK(0, 0));
0580     vqx_dack_cnt.s.done_ack = ackcnt;
0581     cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_ACK(0, 0),
0582             vqx_dack_cnt.u);
0583 }
0584 
0585 static irqreturn_t cptvf_done_intr_handler(int irq, void *cptvf_irq)
0586 {
0587     struct cpt_vf *cptvf = (struct cpt_vf *)cptvf_irq;
0588     struct pci_dev *pdev = cptvf->pdev;
0589     /* Read the number of completions */
0590     u32 intr = cptvf_read_vq_done_count(cptvf);
0591 
0592     if (intr) {
0593         struct cptvf_wqe *wqe;
0594 
0595         /* Acknowledge the number of
0596          * scheduled completions for processing
0597          */
0598         cptvf_write_vq_done_ack(cptvf, intr);
0599         wqe = get_cptvf_vq_wqe(cptvf, 0);
0600         if (unlikely(!wqe)) {
0601             dev_err(&pdev->dev, "No work to schedule for VF (%d)",
0602                 cptvf->vfid);
0603             return IRQ_NONE;
0604         }
0605         tasklet_hi_schedule(&wqe->twork);
0606     }
0607 
0608     return IRQ_HANDLED;
0609 }
0610 
0611 static void cptvf_set_irq_affinity(struct cpt_vf *cptvf, int vec)
0612 {
0613     struct pci_dev *pdev = cptvf->pdev;
0614     int cpu;
0615 
0616     if (!zalloc_cpumask_var(&cptvf->affinity_mask[vec],
0617                 GFP_KERNEL)) {
0618         dev_err(&pdev->dev, "Allocation failed for affinity_mask for VF %d",
0619             cptvf->vfid);
0620         return;
0621     }
0622 
0623     cpu = cptvf->vfid % num_online_cpus();
0624     cpumask_set_cpu(cpumask_local_spread(cpu, cptvf->node),
0625             cptvf->affinity_mask[vec]);
0626     irq_set_affinity_hint(pci_irq_vector(pdev, vec),
0627             cptvf->affinity_mask[vec]);
0628 }
0629 
0630 static void cptvf_write_vq_saddr(struct cpt_vf *cptvf, u64 val)
0631 {
0632     union cptx_vqx_saddr vqx_saddr;
0633 
0634     vqx_saddr.u = val;
0635     cpt_write_csr64(cptvf->reg_base, CPTX_VQX_SADDR(0, 0), vqx_saddr.u);
0636 }
0637 
0638 static void cptvf_device_init(struct cpt_vf *cptvf)
0639 {
0640     u64 base_addr = 0;
0641 
0642     /* Disable the VQ */
0643     cptvf_write_vq_ctl(cptvf, 0);
0644     /* Reset the doorbell */
0645     cptvf_write_vq_doorbell(cptvf, 0);
0646     /* Clear inflight */
0647     cptvf_write_vq_inprog(cptvf, 0);
0648     /* Write VQ SADDR */
0649     /* TODO: for now only one queue, so hard coded */
0650     base_addr = (u64)(cptvf->cqinfo.queue[0].qhead->dma_addr);
0651     cptvf_write_vq_saddr(cptvf, base_addr);
0652     /* Configure timerhold / coalescence */
0653     cptvf_write_vq_done_timewait(cptvf, CPT_TIMER_THOLD);
0654     cptvf_write_vq_done_numwait(cptvf, 1);
0655     /* Enable the VQ */
0656     cptvf_write_vq_ctl(cptvf, 1);
0657     /* Flag the VF ready */
0658     cptvf->flags |= CPT_FLAG_DEVICE_READY;
0659 }
0660 
0661 static int cptvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
0662 {
0663     struct device *dev = &pdev->dev;
0664     struct cpt_vf *cptvf;
0665     int    err;
0666 
0667     cptvf = devm_kzalloc(dev, sizeof(*cptvf), GFP_KERNEL);
0668     if (!cptvf)
0669         return -ENOMEM;
0670 
0671     pci_set_drvdata(pdev, cptvf);
0672     cptvf->pdev = pdev;
0673     err = pci_enable_device(pdev);
0674     if (err) {
0675         dev_err(dev, "Failed to enable PCI device\n");
0676         pci_set_drvdata(pdev, NULL);
0677         return err;
0678     }
0679 
0680     err = pci_request_regions(pdev, DRV_NAME);
0681     if (err) {
0682         dev_err(dev, "PCI request regions failed 0x%x\n", err);
0683         goto cptvf_err_disable_device;
0684     }
0685     /* Mark as VF driver */
0686     cptvf->flags |= CPT_FLAG_VF_DRIVER;
0687     err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48));
0688     if (err) {
0689         dev_err(dev, "Unable to get usable 48-bit DMA configuration\n");
0690         goto cptvf_err_release_regions;
0691     }
0692 
0693     /* MAP PF's configuration registers */
0694     cptvf->reg_base = pcim_iomap(pdev, 0, 0);
0695     if (!cptvf->reg_base) {
0696         dev_err(dev, "Cannot map config register space, aborting\n");
0697         err = -ENOMEM;
0698         goto cptvf_err_release_regions;
0699     }
0700 
0701     cptvf->node = dev_to_node(&pdev->dev);
0702     err = pci_alloc_irq_vectors(pdev, CPT_VF_MSIX_VECTORS,
0703             CPT_VF_MSIX_VECTORS, PCI_IRQ_MSIX);
0704     if (err < 0) {
0705         dev_err(dev, "Request for #%d msix vectors failed\n",
0706             CPT_VF_MSIX_VECTORS);
0707         goto cptvf_err_release_regions;
0708     }
0709 
0710     err = request_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_MISC),
0711               cptvf_misc_intr_handler, 0, "CPT VF misc intr",
0712               cptvf);
0713     if (err) {
0714         dev_err(dev, "Request misc irq failed");
0715         goto cptvf_free_vectors;
0716     }
0717 
0718     /* Enable mailbox interrupt */
0719     cptvf_enable_mbox_interrupts(cptvf);
0720     cptvf_enable_swerr_interrupts(cptvf);
0721 
0722     /* Check ready with PF */
0723     /* Gets chip ID / device Id from PF if ready */
0724     err = cptvf_check_pf_ready(cptvf);
0725     if (err) {
0726         dev_err(dev, "PF not responding to READY msg");
0727         goto cptvf_free_misc_irq;
0728     }
0729 
0730     /* CPT VF software resources initialization */
0731     cptvf->cqinfo.qchunksize = CPT_CMD_QCHUNK_SIZE;
0732     err = cptvf_sw_init(cptvf, CPT_CMD_QLEN, CPT_NUM_QS_PER_VF);
0733     if (err) {
0734         dev_err(dev, "cptvf_sw_init() failed");
0735         goto cptvf_free_misc_irq;
0736     }
0737     /* Convey VQ LEN to PF */
0738     err = cptvf_send_vq_size_msg(cptvf);
0739     if (err) {
0740         dev_err(dev, "PF not responding to QLEN msg");
0741         goto cptvf_free_misc_irq;
0742     }
0743 
0744     /* CPT VF device initialization */
0745     cptvf_device_init(cptvf);
0746     /* Send msg to PF to assign currnet Q to required group */
0747     cptvf->vfgrp = 1;
0748     err = cptvf_send_vf_to_grp_msg(cptvf);
0749     if (err) {
0750         dev_err(dev, "PF not responding to VF_GRP msg");
0751         goto cptvf_free_misc_irq;
0752     }
0753 
0754     cptvf->priority = 1;
0755     err = cptvf_send_vf_priority_msg(cptvf);
0756     if (err) {
0757         dev_err(dev, "PF not responding to VF_PRIO msg");
0758         goto cptvf_free_misc_irq;
0759     }
0760 
0761     err = request_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_DONE),
0762               cptvf_done_intr_handler, 0, "CPT VF done intr",
0763               cptvf);
0764     if (err) {
0765         dev_err(dev, "Request done irq failed\n");
0766         goto cptvf_free_misc_irq;
0767     }
0768 
0769     /* Enable mailbox interrupt */
0770     cptvf_enable_done_interrupts(cptvf);
0771 
0772     /* Set irq affinity masks */
0773     cptvf_set_irq_affinity(cptvf, CPT_VF_INT_VEC_E_MISC);
0774     cptvf_set_irq_affinity(cptvf, CPT_VF_INT_VEC_E_DONE);
0775 
0776     err = cptvf_send_vf_up(cptvf);
0777     if (err) {
0778         dev_err(dev, "PF not responding to UP msg");
0779         goto cptvf_free_irq_affinity;
0780     }
0781     err = cvm_crypto_init(cptvf);
0782     if (err) {
0783         dev_err(dev, "Algorithm register failed\n");
0784         goto cptvf_free_irq_affinity;
0785     }
0786     return 0;
0787 
0788 cptvf_free_irq_affinity:
0789     cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_DONE);
0790     cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_MISC);
0791 cptvf_free_misc_irq:
0792     free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_MISC), cptvf);
0793 cptvf_free_vectors:
0794     pci_free_irq_vectors(cptvf->pdev);
0795 cptvf_err_release_regions:
0796     pci_release_regions(pdev);
0797 cptvf_err_disable_device:
0798     pci_disable_device(pdev);
0799     pci_set_drvdata(pdev, NULL);
0800 
0801     return err;
0802 }
0803 
0804 static void cptvf_remove(struct pci_dev *pdev)
0805 {
0806     struct cpt_vf *cptvf = pci_get_drvdata(pdev);
0807 
0808     if (!cptvf) {
0809         dev_err(&pdev->dev, "Invalid CPT-VF device\n");
0810         return;
0811     }
0812 
0813     /* Convey DOWN to PF */
0814     if (cptvf_send_vf_down(cptvf)) {
0815         dev_err(&pdev->dev, "PF not responding to DOWN msg");
0816     } else {
0817         cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_DONE);
0818         cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_MISC);
0819         free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_DONE), cptvf);
0820         free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_MISC), cptvf);
0821         pci_free_irq_vectors(cptvf->pdev);
0822         cptvf_sw_cleanup(cptvf);
0823         pci_set_drvdata(pdev, NULL);
0824         pci_release_regions(pdev);
0825         pci_disable_device(pdev);
0826         cvm_crypto_exit();
0827     }
0828 }
0829 
0830 static void cptvf_shutdown(struct pci_dev *pdev)
0831 {
0832     cptvf_remove(pdev);
0833 }
0834 
0835 /* Supported devices */
0836 static const struct pci_device_id cptvf_id_table[] = {
0837     {PCI_VDEVICE(CAVIUM, CPT_81XX_PCI_VF_DEVICE_ID), 0},
0838     { 0, }  /* end of table */
0839 };
0840 
0841 static struct pci_driver cptvf_pci_driver = {
0842     .name = DRV_NAME,
0843     .id_table = cptvf_id_table,
0844     .probe = cptvf_probe,
0845     .remove = cptvf_remove,
0846     .shutdown = cptvf_shutdown,
0847 };
0848 
0849 module_pci_driver(cptvf_pci_driver);
0850 
0851 MODULE_AUTHOR("George Cherian <george.cherian@cavium.com>");
0852 MODULE_DESCRIPTION("Cavium Thunder CPT Virtual Function Driver");
0853 MODULE_LICENSE("GPL v2");
0854 MODULE_VERSION(DRV_VERSION);
0855 MODULE_DEVICE_TABLE(pci, cptvf_id_table);
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!