Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0
0002 /* Marvell OcteonTX CPT driver
0003  *
0004  * Copyright (C) 2019 Marvell International Ltd.
0005  *
0006  * This program is free software; you can redistribute it and/or modify
0007  * it under the terms of the GNU General Public License version 2 as
0008  * published by the Free Software Foundation.
0009  */
0010 
0011 #include "otx_cptvf.h"
0012 #include "otx_cptvf_algs.h"
0013 
0014 /* Completion code size and initial value */
0015 #define COMPLETION_CODE_SIZE    8
0016 #define COMPLETION_CODE_INIT    0
0017 
0018 /* SG list header size in bytes */
0019 #define SG_LIST_HDR_SIZE    8
0020 
0021 /* Default timeout when waiting for free pending entry in us */
0022 #define CPT_PENTRY_TIMEOUT  1000
0023 #define CPT_PENTRY_STEP     50
0024 
0025 /* Default threshold for stopping and resuming sender requests */
0026 #define CPT_IQ_STOP_MARGIN  128
0027 #define CPT_IQ_RESUME_MARGIN    512
0028 
0029 #define CPT_DMA_ALIGN       128
0030 
0031 void otx_cpt_dump_sg_list(struct pci_dev *pdev, struct otx_cpt_req_info *req)
0032 {
0033     int i;
0034 
0035     pr_debug("Gather list size %d\n", req->incnt);
0036     for (i = 0; i < req->incnt; i++) {
0037         pr_debug("Buffer %d size %d, vptr 0x%p, dmaptr 0x%p\n", i,
0038              req->in[i].size, req->in[i].vptr,
0039              (void *) req->in[i].dma_addr);
0040         pr_debug("Buffer hexdump (%d bytes)\n",
0041              req->in[i].size);
0042         print_hex_dump_debug("", DUMP_PREFIX_NONE, 16, 1,
0043                      req->in[i].vptr, req->in[i].size, false);
0044     }
0045 
0046     pr_debug("Scatter list size %d\n", req->outcnt);
0047     for (i = 0; i < req->outcnt; i++) {
0048         pr_debug("Buffer %d size %d, vptr 0x%p, dmaptr 0x%p\n", i,
0049              req->out[i].size, req->out[i].vptr,
0050              (void *) req->out[i].dma_addr);
0051         pr_debug("Buffer hexdump (%d bytes)\n", req->out[i].size);
0052         print_hex_dump_debug("", DUMP_PREFIX_NONE, 16, 1,
0053                      req->out[i].vptr, req->out[i].size, false);
0054     }
0055 }
0056 
0057 static inline struct otx_cpt_pending_entry *get_free_pending_entry(
0058                         struct otx_cpt_pending_queue *q,
0059                         int qlen)
0060 {
0061     struct otx_cpt_pending_entry *ent = NULL;
0062 
0063     ent = &q->head[q->rear];
0064     if (unlikely(ent->busy))
0065         return NULL;
0066 
0067     q->rear++;
0068     if (unlikely(q->rear == qlen))
0069         q->rear = 0;
0070 
0071     return ent;
0072 }
0073 
0074 static inline u32 modulo_inc(u32 index, u32 length, u32 inc)
0075 {
0076     if (WARN_ON(inc > length))
0077         inc = length;
0078 
0079     index += inc;
0080     if (unlikely(index >= length))
0081         index -= length;
0082 
0083     return index;
0084 }
0085 
0086 static inline void free_pentry(struct otx_cpt_pending_entry *pentry)
0087 {
0088     pentry->completion_addr = NULL;
0089     pentry->info = NULL;
0090     pentry->callback = NULL;
0091     pentry->areq = NULL;
0092     pentry->resume_sender = false;
0093     pentry->busy = false;
0094 }
0095 
0096 static inline int setup_sgio_components(struct pci_dev *pdev,
0097                     struct otx_cpt_buf_ptr *list,
0098                     int buf_count, u8 *buffer)
0099 {
0100     struct otx_cpt_sglist_component *sg_ptr = NULL;
0101     int ret = 0, i, j;
0102     int components;
0103 
0104     if (unlikely(!list)) {
0105         dev_err(&pdev->dev, "Input list pointer is NULL\n");
0106         return -EFAULT;
0107     }
0108 
0109     for (i = 0; i < buf_count; i++) {
0110         if (likely(list[i].vptr)) {
0111             list[i].dma_addr = dma_map_single(&pdev->dev,
0112                               list[i].vptr,
0113                               list[i].size,
0114                               DMA_BIDIRECTIONAL);
0115             if (unlikely(dma_mapping_error(&pdev->dev,
0116                                list[i].dma_addr))) {
0117                 dev_err(&pdev->dev, "Dma mapping failed\n");
0118                 ret = -EIO;
0119                 goto sg_cleanup;
0120             }
0121         }
0122     }
0123 
0124     components = buf_count / 4;
0125     sg_ptr = (struct otx_cpt_sglist_component *)buffer;
0126     for (i = 0; i < components; i++) {
0127         sg_ptr->u.s.len0 = cpu_to_be16(list[i * 4 + 0].size);
0128         sg_ptr->u.s.len1 = cpu_to_be16(list[i * 4 + 1].size);
0129         sg_ptr->u.s.len2 = cpu_to_be16(list[i * 4 + 2].size);
0130         sg_ptr->u.s.len3 = cpu_to_be16(list[i * 4 + 3].size);
0131         sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr);
0132         sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr);
0133         sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr);
0134         sg_ptr->ptr3 = cpu_to_be64(list[i * 4 + 3].dma_addr);
0135         sg_ptr++;
0136     }
0137     components = buf_count % 4;
0138 
0139     switch (components) {
0140     case 3:
0141         sg_ptr->u.s.len2 = cpu_to_be16(list[i * 4 + 2].size);
0142         sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr);
0143         fallthrough;
0144     case 2:
0145         sg_ptr->u.s.len1 = cpu_to_be16(list[i * 4 + 1].size);
0146         sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr);
0147         fallthrough;
0148     case 1:
0149         sg_ptr->u.s.len0 = cpu_to_be16(list[i * 4 + 0].size);
0150         sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr);
0151         break;
0152     default:
0153         break;
0154     }
0155     return ret;
0156 
0157 sg_cleanup:
0158     for (j = 0; j < i; j++) {
0159         if (list[j].dma_addr) {
0160             dma_unmap_single(&pdev->dev, list[i].dma_addr,
0161                      list[i].size, DMA_BIDIRECTIONAL);
0162         }
0163 
0164         list[j].dma_addr = 0;
0165     }
0166     return ret;
0167 }
0168 
0169 static inline int setup_sgio_list(struct pci_dev *pdev,
0170                   struct otx_cpt_info_buffer **pinfo,
0171                   struct otx_cpt_req_info *req, gfp_t gfp)
0172 {
0173     u32 dlen, align_dlen, info_len, rlen;
0174     struct otx_cpt_info_buffer *info;
0175     u16 g_sz_bytes, s_sz_bytes;
0176     int align = CPT_DMA_ALIGN;
0177     u32 total_mem_len;
0178 
0179     if (unlikely(req->incnt > OTX_CPT_MAX_SG_IN_CNT ||
0180              req->outcnt > OTX_CPT_MAX_SG_OUT_CNT)) {
0181         dev_err(&pdev->dev, "Error too many sg components\n");
0182         return -EINVAL;
0183     }
0184 
0185     g_sz_bytes = ((req->incnt + 3) / 4) *
0186               sizeof(struct otx_cpt_sglist_component);
0187     s_sz_bytes = ((req->outcnt + 3) / 4) *
0188               sizeof(struct otx_cpt_sglist_component);
0189 
0190     dlen = g_sz_bytes + s_sz_bytes + SG_LIST_HDR_SIZE;
0191     align_dlen = ALIGN(dlen, align);
0192     info_len = ALIGN(sizeof(*info), align);
0193     rlen = ALIGN(sizeof(union otx_cpt_res_s), align);
0194     total_mem_len = align_dlen + info_len + rlen + COMPLETION_CODE_SIZE;
0195 
0196     info = kzalloc(total_mem_len, gfp);
0197     if (unlikely(!info)) {
0198         dev_err(&pdev->dev, "Memory allocation failed\n");
0199         return -ENOMEM;
0200     }
0201     *pinfo = info;
0202     info->dlen = dlen;
0203     info->in_buffer = (u8 *)info + info_len;
0204 
0205     ((__be16 *)info->in_buffer)[0] = cpu_to_be16(req->outcnt);
0206     ((__be16 *)info->in_buffer)[1] = cpu_to_be16(req->incnt);
0207     ((u16 *)info->in_buffer)[2] = 0;
0208     ((u16 *)info->in_buffer)[3] = 0;
0209 
0210     /* Setup gather (input) components */
0211     if (setup_sgio_components(pdev, req->in, req->incnt,
0212                   &info->in_buffer[8])) {
0213         dev_err(&pdev->dev, "Failed to setup gather list\n");
0214         return -EFAULT;
0215     }
0216 
0217     if (setup_sgio_components(pdev, req->out, req->outcnt,
0218                   &info->in_buffer[8 + g_sz_bytes])) {
0219         dev_err(&pdev->dev, "Failed to setup scatter list\n");
0220         return -EFAULT;
0221     }
0222 
0223     info->dma_len = total_mem_len - info_len;
0224     info->dptr_baddr = dma_map_single(&pdev->dev, (void *)info->in_buffer,
0225                       info->dma_len, DMA_BIDIRECTIONAL);
0226     if (unlikely(dma_mapping_error(&pdev->dev, info->dptr_baddr))) {
0227         dev_err(&pdev->dev, "DMA Mapping failed for cpt req\n");
0228         return -EIO;
0229     }
0230     /*
0231      * Get buffer for union otx_cpt_res_s response
0232      * structure and its physical address
0233      */
0234     info->completion_addr = (u64 *)(info->in_buffer + align_dlen);
0235     info->comp_baddr = info->dptr_baddr + align_dlen;
0236 
0237     /* Create and initialize RPTR */
0238     info->out_buffer = (u8 *)info->completion_addr + rlen;
0239     info->rptr_baddr = info->comp_baddr + rlen;
0240 
0241     *((u64 *) info->out_buffer) = ~((u64) COMPLETION_CODE_INIT);
0242 
0243     return 0;
0244 }
0245 
0246 
0247 static void cpt_fill_inst(union otx_cpt_inst_s *inst,
0248               struct otx_cpt_info_buffer *info,
0249               struct otx_cpt_iq_cmd *cmd)
0250 {
0251     inst->u[0] = 0x0;
0252     inst->s.doneint = true;
0253     inst->s.res_addr = (u64)info->comp_baddr;
0254     inst->u[2] = 0x0;
0255     inst->s.wq_ptr = 0;
0256     inst->s.ei0 = cmd->cmd.u64;
0257     inst->s.ei1 = cmd->dptr;
0258     inst->s.ei2 = cmd->rptr;
0259     inst->s.ei3 = cmd->cptr.u64;
0260 }
0261 
0262 /*
0263  * On OcteonTX platform the parameter db_count is used as a count for ringing
0264  * door bell. The valid values for db_count are:
0265  * 0 - 1 CPT instruction will be enqueued however CPT will not be informed
0266  * 1 - 1 CPT instruction will be enqueued and CPT will be informed
0267  */
0268 static void cpt_send_cmd(union otx_cpt_inst_s *cptinst, struct otx_cptvf *cptvf)
0269 {
0270     struct otx_cpt_cmd_qinfo *qinfo = &cptvf->cqinfo;
0271     struct otx_cpt_cmd_queue *queue;
0272     struct otx_cpt_cmd_chunk *curr;
0273     u8 *ent;
0274 
0275     queue = &qinfo->queue[0];
0276     /*
0277      * cpt_send_cmd is currently called only from critical section
0278      * therefore no locking is required for accessing instruction queue
0279      */
0280     ent = &queue->qhead->head[queue->idx * OTX_CPT_INST_SIZE];
0281     memcpy(ent, (void *) cptinst, OTX_CPT_INST_SIZE);
0282 
0283     if (++queue->idx >= queue->qhead->size / 64) {
0284         curr = queue->qhead;
0285 
0286         if (list_is_last(&curr->nextchunk, &queue->chead))
0287             queue->qhead = queue->base;
0288         else
0289             queue->qhead = list_next_entry(queue->qhead, nextchunk);
0290         queue->idx = 0;
0291     }
0292     /* make sure all memory stores are done before ringing doorbell */
0293     smp_wmb();
0294     otx_cptvf_write_vq_doorbell(cptvf, 1);
0295 }
0296 
0297 static int process_request(struct pci_dev *pdev, struct otx_cpt_req_info *req,
0298                struct otx_cpt_pending_queue *pqueue,
0299                struct otx_cptvf *cptvf)
0300 {
0301     struct otx_cptvf_request *cpt_req = &req->req;
0302     struct otx_cpt_pending_entry *pentry = NULL;
0303     union otx_cpt_ctrl_info *ctrl = &req->ctrl;
0304     struct otx_cpt_info_buffer *info = NULL;
0305     union otx_cpt_res_s *result = NULL;
0306     struct otx_cpt_iq_cmd iq_cmd;
0307     union otx_cpt_inst_s cptinst;
0308     int retry, ret = 0;
0309     u8 resume_sender;
0310     gfp_t gfp;
0311 
0312     gfp = (req->areq->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL :
0313                                   GFP_ATOMIC;
0314     ret = setup_sgio_list(pdev, &info, req, gfp);
0315     if (unlikely(ret)) {
0316         dev_err(&pdev->dev, "Setting up SG list failed\n");
0317         goto request_cleanup;
0318     }
0319     cpt_req->dlen = info->dlen;
0320 
0321     result = (union otx_cpt_res_s *) info->completion_addr;
0322     result->s.compcode = COMPLETION_CODE_INIT;
0323 
0324     spin_lock_bh(&pqueue->lock);
0325     pentry = get_free_pending_entry(pqueue, pqueue->qlen);
0326     retry = CPT_PENTRY_TIMEOUT / CPT_PENTRY_STEP;
0327     while (unlikely(!pentry) && retry--) {
0328         spin_unlock_bh(&pqueue->lock);
0329         udelay(CPT_PENTRY_STEP);
0330         spin_lock_bh(&pqueue->lock);
0331         pentry = get_free_pending_entry(pqueue, pqueue->qlen);
0332     }
0333 
0334     if (unlikely(!pentry)) {
0335         ret = -ENOSPC;
0336         spin_unlock_bh(&pqueue->lock);
0337         goto request_cleanup;
0338     }
0339 
0340     /*
0341      * Check if we are close to filling in entire pending queue,
0342      * if so then tell the sender to stop/sleep by returning -EBUSY
0343      * We do it only for context which can sleep (GFP_KERNEL)
0344      */
0345     if (gfp == GFP_KERNEL &&
0346         pqueue->pending_count > (pqueue->qlen - CPT_IQ_STOP_MARGIN)) {
0347         pentry->resume_sender = true;
0348     } else
0349         pentry->resume_sender = false;
0350     resume_sender = pentry->resume_sender;
0351     pqueue->pending_count++;
0352 
0353     pentry->completion_addr = info->completion_addr;
0354     pentry->info = info;
0355     pentry->callback = req->callback;
0356     pentry->areq = req->areq;
0357     pentry->busy = true;
0358     info->pentry = pentry;
0359     info->time_in = jiffies;
0360     info->req = req;
0361 
0362     /* Fill in the command */
0363     iq_cmd.cmd.u64 = 0;
0364     iq_cmd.cmd.s.opcode = cpu_to_be16(cpt_req->opcode.flags);
0365     iq_cmd.cmd.s.param1 = cpu_to_be16(cpt_req->param1);
0366     iq_cmd.cmd.s.param2 = cpu_to_be16(cpt_req->param2);
0367     iq_cmd.cmd.s.dlen   = cpu_to_be16(cpt_req->dlen);
0368 
0369     iq_cmd.dptr = info->dptr_baddr;
0370     iq_cmd.rptr = info->rptr_baddr;
0371     iq_cmd.cptr.u64 = 0;
0372     iq_cmd.cptr.s.grp = ctrl->s.grp;
0373 
0374     /* Fill in the CPT_INST_S type command for HW interpretation */
0375     cpt_fill_inst(&cptinst, info, &iq_cmd);
0376 
0377     /* Print debug info if enabled */
0378     otx_cpt_dump_sg_list(pdev, req);
0379     pr_debug("Cpt_inst_s hexdump (%d bytes)\n", OTX_CPT_INST_SIZE);
0380     print_hex_dump_debug("", 0, 16, 1, &cptinst, OTX_CPT_INST_SIZE, false);
0381     pr_debug("Dptr hexdump (%d bytes)\n", cpt_req->dlen);
0382     print_hex_dump_debug("", 0, 16, 1, info->in_buffer,
0383                  cpt_req->dlen, false);
0384 
0385     /* Send CPT command */
0386     cpt_send_cmd(&cptinst, cptvf);
0387 
0388     /*
0389      * We allocate and prepare pending queue entry in critical section
0390      * together with submitting CPT instruction to CPT instruction queue
0391      * to make sure that order of CPT requests is the same in both
0392      * pending and instruction queues
0393      */
0394     spin_unlock_bh(&pqueue->lock);
0395 
0396     ret = resume_sender ? -EBUSY : -EINPROGRESS;
0397     return ret;
0398 
0399 request_cleanup:
0400     do_request_cleanup(pdev, info);
0401     return ret;
0402 }
0403 
0404 int otx_cpt_do_request(struct pci_dev *pdev, struct otx_cpt_req_info *req,
0405                int cpu_num)
0406 {
0407     struct otx_cptvf *cptvf = pci_get_drvdata(pdev);
0408 
0409     if (!otx_cpt_device_ready(cptvf)) {
0410         dev_err(&pdev->dev, "CPT Device is not ready\n");
0411         return -ENODEV;
0412     }
0413 
0414     if ((cptvf->vftype == OTX_CPT_SE_TYPES) && (!req->ctrl.s.se_req)) {
0415         dev_err(&pdev->dev, "CPTVF-%d of SE TYPE got AE request\n",
0416             cptvf->vfid);
0417         return -EINVAL;
0418     } else if ((cptvf->vftype == OTX_CPT_AE_TYPES) &&
0419            (req->ctrl.s.se_req)) {
0420         dev_err(&pdev->dev, "CPTVF-%d of AE TYPE got SE request\n",
0421             cptvf->vfid);
0422         return -EINVAL;
0423     }
0424 
0425     return process_request(pdev, req, &cptvf->pqinfo.queue[0], cptvf);
0426 }
0427 
0428 static int cpt_process_ccode(struct pci_dev *pdev,
0429                  union otx_cpt_res_s *cpt_status,
0430                  struct otx_cpt_info_buffer *cpt_info,
0431                  struct otx_cpt_req_info *req, u32 *res_code)
0432 {
0433     u8 ccode = cpt_status->s.compcode;
0434     union otx_cpt_error_code ecode;
0435 
0436     ecode.u = be64_to_cpup((__be64 *)cpt_info->out_buffer);
0437     switch (ccode) {
0438     case CPT_COMP_E_FAULT:
0439         dev_err(&pdev->dev,
0440             "Request failed with DMA fault\n");
0441         otx_cpt_dump_sg_list(pdev, req);
0442         break;
0443 
0444     case CPT_COMP_E_SWERR:
0445         dev_err(&pdev->dev,
0446             "Request failed with software error code %d\n",
0447             ecode.s.ccode);
0448         otx_cpt_dump_sg_list(pdev, req);
0449         break;
0450 
0451     case CPT_COMP_E_HWERR:
0452         dev_err(&pdev->dev,
0453             "Request failed with hardware error\n");
0454         otx_cpt_dump_sg_list(pdev, req);
0455         break;
0456 
0457     case COMPLETION_CODE_INIT:
0458         /* check for timeout */
0459         if (time_after_eq(jiffies, cpt_info->time_in +
0460                   OTX_CPT_COMMAND_TIMEOUT * HZ))
0461             dev_warn(&pdev->dev, "Request timed out 0x%p\n", req);
0462         else if (cpt_info->extra_time < OTX_CPT_TIME_IN_RESET_COUNT) {
0463             cpt_info->time_in = jiffies;
0464             cpt_info->extra_time++;
0465         }
0466         return 1;
0467 
0468     case CPT_COMP_E_GOOD:
0469         /* Check microcode completion code */
0470         if (ecode.s.ccode) {
0471             /*
0472              * If requested hmac is truncated and ucode returns
0473              * s/g write length error then we report success
0474              * because ucode writes as many bytes of calculated
0475              * hmac as available in gather buffer and reports
0476              * s/g write length error if number of bytes in gather
0477              * buffer is less than full hmac size.
0478              */
0479             if (req->is_trunc_hmac &&
0480                 ecode.s.ccode == ERR_SCATTER_GATHER_WRITE_LENGTH) {
0481                 *res_code = 0;
0482                 break;
0483             }
0484 
0485             dev_err(&pdev->dev,
0486                 "Request failed with software error code 0x%x\n",
0487                 ecode.s.ccode);
0488             otx_cpt_dump_sg_list(pdev, req);
0489             break;
0490         }
0491 
0492         /* Request has been processed with success */
0493         *res_code = 0;
0494         break;
0495 
0496     default:
0497         dev_err(&pdev->dev, "Request returned invalid status\n");
0498         break;
0499     }
0500 
0501     return 0;
0502 }
0503 
0504 static inline void process_pending_queue(struct pci_dev *pdev,
0505                      struct otx_cpt_pending_queue *pqueue)
0506 {
0507     void (*callback)(int status, void *arg1, void *arg2);
0508     struct otx_cpt_pending_entry *resume_pentry = NULL;
0509     struct otx_cpt_pending_entry *pentry = NULL;
0510     struct otx_cpt_info_buffer *cpt_info = NULL;
0511     union otx_cpt_res_s *cpt_status = NULL;
0512     struct otx_cpt_req_info *req = NULL;
0513     struct crypto_async_request *areq;
0514     u32 res_code, resume_index;
0515 
0516     while (1) {
0517         spin_lock_bh(&pqueue->lock);
0518         pentry = &pqueue->head[pqueue->front];
0519 
0520         if (WARN_ON(!pentry)) {
0521             spin_unlock_bh(&pqueue->lock);
0522             break;
0523         }
0524 
0525         res_code = -EINVAL;
0526         if (unlikely(!pentry->busy)) {
0527             spin_unlock_bh(&pqueue->lock);
0528             break;
0529         }
0530 
0531         if (unlikely(!pentry->callback)) {
0532             dev_err(&pdev->dev, "Callback NULL\n");
0533             goto process_pentry;
0534         }
0535 
0536         cpt_info = pentry->info;
0537         if (unlikely(!cpt_info)) {
0538             dev_err(&pdev->dev, "Pending entry post arg NULL\n");
0539             goto process_pentry;
0540         }
0541 
0542         req = cpt_info->req;
0543         if (unlikely(!req)) {
0544             dev_err(&pdev->dev, "Request NULL\n");
0545             goto process_pentry;
0546         }
0547 
0548         cpt_status = (union otx_cpt_res_s *) pentry->completion_addr;
0549         if (unlikely(!cpt_status)) {
0550             dev_err(&pdev->dev, "Completion address NULL\n");
0551             goto process_pentry;
0552         }
0553 
0554         if (cpt_process_ccode(pdev, cpt_status, cpt_info, req,
0555                       &res_code)) {
0556             spin_unlock_bh(&pqueue->lock);
0557             return;
0558         }
0559         cpt_info->pdev = pdev;
0560 
0561 process_pentry:
0562         /*
0563          * Check if we should inform sending side to resume
0564          * We do it CPT_IQ_RESUME_MARGIN elements in advance before
0565          * pending queue becomes empty
0566          */
0567         resume_index = modulo_inc(pqueue->front, pqueue->qlen,
0568                       CPT_IQ_RESUME_MARGIN);
0569         resume_pentry = &pqueue->head[resume_index];
0570         if (resume_pentry &&
0571             resume_pentry->resume_sender) {
0572             resume_pentry->resume_sender = false;
0573             callback = resume_pentry->callback;
0574             areq = resume_pentry->areq;
0575 
0576             if (callback) {
0577                 spin_unlock_bh(&pqueue->lock);
0578 
0579                 /*
0580                  * EINPROGRESS is an indication for sending
0581                  * side that it can resume sending requests
0582                  */
0583                 callback(-EINPROGRESS, areq, cpt_info);
0584                 spin_lock_bh(&pqueue->lock);
0585             }
0586         }
0587 
0588         callback = pentry->callback;
0589         areq = pentry->areq;
0590         free_pentry(pentry);
0591 
0592         pqueue->pending_count--;
0593         pqueue->front = modulo_inc(pqueue->front, pqueue->qlen, 1);
0594         spin_unlock_bh(&pqueue->lock);
0595 
0596         /*
0597          * Call callback after current pending entry has been
0598          * processed, we don't do it if the callback pointer is
0599          * invalid.
0600          */
0601         if (callback)
0602             callback(res_code, areq, cpt_info);
0603     }
0604 }
0605 
0606 void otx_cpt_post_process(struct otx_cptvf_wqe *wqe)
0607 {
0608     process_pending_queue(wqe->cptvf->pdev, &wqe->cptvf->pqinfo.queue[0]);
0609 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!