Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /* Copyright (C) 2020 Marvell. */
0003 
0004 #include "otx2_cptvf.h"
0005 #include "otx2_cpt_common.h"
0006 
0007 /* SG list header size in bytes */
0008 #define SG_LIST_HDR_SIZE    8
0009 
0010 /* Default timeout when waiting for free pending entry in us */
0011 #define CPT_PENTRY_TIMEOUT  1000
0012 #define CPT_PENTRY_STEP     50
0013 
0014 /* Default threshold for stopping and resuming sender requests */
0015 #define CPT_IQ_STOP_MARGIN  128
0016 #define CPT_IQ_RESUME_MARGIN    512
0017 
0018 /* Default command timeout in seconds */
0019 #define CPT_COMMAND_TIMEOUT 4
0020 #define CPT_TIME_IN_RESET_COUNT 5
0021 
0022 static void otx2_cpt_dump_sg_list(struct pci_dev *pdev,
0023                   struct otx2_cpt_req_info *req)
0024 {
0025     int i;
0026 
0027     pr_debug("Gather list size %d\n", req->in_cnt);
0028     for (i = 0; i < req->in_cnt; i++) {
0029         pr_debug("Buffer %d size %d, vptr 0x%p, dmaptr 0x%p\n", i,
0030              req->in[i].size, req->in[i].vptr,
0031              (void *) req->in[i].dma_addr);
0032         pr_debug("Buffer hexdump (%d bytes)\n",
0033              req->in[i].size);
0034         print_hex_dump_debug("", DUMP_PREFIX_NONE, 16, 1,
0035                      req->in[i].vptr, req->in[i].size, false);
0036     }
0037     pr_debug("Scatter list size %d\n", req->out_cnt);
0038     for (i = 0; i < req->out_cnt; i++) {
0039         pr_debug("Buffer %d size %d, vptr 0x%p, dmaptr 0x%p\n", i,
0040              req->out[i].size, req->out[i].vptr,
0041              (void *) req->out[i].dma_addr);
0042         pr_debug("Buffer hexdump (%d bytes)\n", req->out[i].size);
0043         print_hex_dump_debug("", DUMP_PREFIX_NONE, 16, 1,
0044                      req->out[i].vptr, req->out[i].size, false);
0045     }
0046 }
0047 
0048 static inline struct otx2_cpt_pending_entry *get_free_pending_entry(
0049                     struct otx2_cpt_pending_queue *q,
0050                     int qlen)
0051 {
0052     struct otx2_cpt_pending_entry *ent = NULL;
0053 
0054     ent = &q->head[q->rear];
0055     if (unlikely(ent->busy))
0056         return NULL;
0057 
0058     q->rear++;
0059     if (unlikely(q->rear == qlen))
0060         q->rear = 0;
0061 
0062     return ent;
0063 }
0064 
0065 static inline u32 modulo_inc(u32 index, u32 length, u32 inc)
0066 {
0067     if (WARN_ON(inc > length))
0068         inc = length;
0069 
0070     index += inc;
0071     if (unlikely(index >= length))
0072         index -= length;
0073 
0074     return index;
0075 }
0076 
0077 static inline void free_pentry(struct otx2_cpt_pending_entry *pentry)
0078 {
0079     pentry->completion_addr = NULL;
0080     pentry->info = NULL;
0081     pentry->callback = NULL;
0082     pentry->areq = NULL;
0083     pentry->resume_sender = false;
0084     pentry->busy = false;
0085 }
0086 
0087 static inline int setup_sgio_components(struct pci_dev *pdev,
0088                     struct otx2_cpt_buf_ptr *list,
0089                     int buf_count, u8 *buffer)
0090 {
0091     struct otx2_cpt_sglist_component *sg_ptr = NULL;
0092     int ret = 0, i, j;
0093     int components;
0094 
0095     if (unlikely(!list)) {
0096         dev_err(&pdev->dev, "Input list pointer is NULL\n");
0097         return -EFAULT;
0098     }
0099 
0100     for (i = 0; i < buf_count; i++) {
0101         if (unlikely(!list[i].vptr))
0102             continue;
0103         list[i].dma_addr = dma_map_single(&pdev->dev, list[i].vptr,
0104                           list[i].size,
0105                           DMA_BIDIRECTIONAL);
0106         if (unlikely(dma_mapping_error(&pdev->dev, list[i].dma_addr))) {
0107             dev_err(&pdev->dev, "Dma mapping failed\n");
0108             ret = -EIO;
0109             goto sg_cleanup;
0110         }
0111     }
0112     components = buf_count / 4;
0113     sg_ptr = (struct otx2_cpt_sglist_component *)buffer;
0114     for (i = 0; i < components; i++) {
0115         sg_ptr->len0 = cpu_to_be16(list[i * 4 + 0].size);
0116         sg_ptr->len1 = cpu_to_be16(list[i * 4 + 1].size);
0117         sg_ptr->len2 = cpu_to_be16(list[i * 4 + 2].size);
0118         sg_ptr->len3 = cpu_to_be16(list[i * 4 + 3].size);
0119         sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr);
0120         sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr);
0121         sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr);
0122         sg_ptr->ptr3 = cpu_to_be64(list[i * 4 + 3].dma_addr);
0123         sg_ptr++;
0124     }
0125     components = buf_count % 4;
0126 
0127     switch (components) {
0128     case 3:
0129         sg_ptr->len2 = cpu_to_be16(list[i * 4 + 2].size);
0130         sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr);
0131         fallthrough;
0132     case 2:
0133         sg_ptr->len1 = cpu_to_be16(list[i * 4 + 1].size);
0134         sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr);
0135         fallthrough;
0136     case 1:
0137         sg_ptr->len0 = cpu_to_be16(list[i * 4 + 0].size);
0138         sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr);
0139         break;
0140     default:
0141         break;
0142     }
0143     return ret;
0144 
0145 sg_cleanup:
0146     for (j = 0; j < i; j++) {
0147         if (list[j].dma_addr) {
0148             dma_unmap_single(&pdev->dev, list[j].dma_addr,
0149                      list[j].size, DMA_BIDIRECTIONAL);
0150         }
0151 
0152         list[j].dma_addr = 0;
0153     }
0154     return ret;
0155 }
0156 
0157 static inline struct otx2_cpt_inst_info *info_create(struct pci_dev *pdev,
0158                           struct otx2_cpt_req_info *req,
0159                           gfp_t gfp)
0160 {
0161     int align = OTX2_CPT_DMA_MINALIGN;
0162     struct otx2_cpt_inst_info *info;
0163     u32 dlen, align_dlen, info_len;
0164     u16 g_sz_bytes, s_sz_bytes;
0165     u32 total_mem_len;
0166 
0167     if (unlikely(req->in_cnt > OTX2_CPT_MAX_SG_IN_CNT ||
0168              req->out_cnt > OTX2_CPT_MAX_SG_OUT_CNT)) {
0169         dev_err(&pdev->dev, "Error too many sg components\n");
0170         return NULL;
0171     }
0172 
0173     g_sz_bytes = ((req->in_cnt + 3) / 4) *
0174               sizeof(struct otx2_cpt_sglist_component);
0175     s_sz_bytes = ((req->out_cnt + 3) / 4) *
0176               sizeof(struct otx2_cpt_sglist_component);
0177 
0178     dlen = g_sz_bytes + s_sz_bytes + SG_LIST_HDR_SIZE;
0179     align_dlen = ALIGN(dlen, align);
0180     info_len = ALIGN(sizeof(*info), align);
0181     total_mem_len = align_dlen + info_len + sizeof(union otx2_cpt_res_s);
0182 
0183     info = kzalloc(total_mem_len, gfp);
0184     if (unlikely(!info))
0185         return NULL;
0186 
0187     info->dlen = dlen;
0188     info->in_buffer = (u8 *)info + info_len;
0189 
0190     ((u16 *)info->in_buffer)[0] = req->out_cnt;
0191     ((u16 *)info->in_buffer)[1] = req->in_cnt;
0192     ((u16 *)info->in_buffer)[2] = 0;
0193     ((u16 *)info->in_buffer)[3] = 0;
0194     cpu_to_be64s((u64 *)info->in_buffer);
0195 
0196     /* Setup gather (input) components */
0197     if (setup_sgio_components(pdev, req->in, req->in_cnt,
0198                   &info->in_buffer[8])) {
0199         dev_err(&pdev->dev, "Failed to setup gather list\n");
0200         goto destroy_info;
0201     }
0202 
0203     if (setup_sgio_components(pdev, req->out, req->out_cnt,
0204                   &info->in_buffer[8 + g_sz_bytes])) {
0205         dev_err(&pdev->dev, "Failed to setup scatter list\n");
0206         goto destroy_info;
0207     }
0208 
0209     info->dma_len = total_mem_len - info_len;
0210     info->dptr_baddr = dma_map_single(&pdev->dev, info->in_buffer,
0211                       info->dma_len, DMA_BIDIRECTIONAL);
0212     if (unlikely(dma_mapping_error(&pdev->dev, info->dptr_baddr))) {
0213         dev_err(&pdev->dev, "DMA Mapping failed for cpt req\n");
0214         goto destroy_info;
0215     }
0216     /*
0217      * Get buffer for union otx2_cpt_res_s response
0218      * structure and its physical address
0219      */
0220     info->completion_addr = info->in_buffer + align_dlen;
0221     info->comp_baddr = info->dptr_baddr + align_dlen;
0222 
0223     return info;
0224 
0225 destroy_info:
0226     otx2_cpt_info_destroy(pdev, info);
0227     return NULL;
0228 }
0229 
0230 static int process_request(struct pci_dev *pdev, struct otx2_cpt_req_info *req,
0231                struct otx2_cpt_pending_queue *pqueue,
0232                struct otx2_cptlf_info *lf)
0233 {
0234     struct otx2_cptvf_request *cpt_req = &req->req;
0235     struct otx2_cpt_pending_entry *pentry = NULL;
0236     union otx2_cpt_ctrl_info *ctrl = &req->ctrl;
0237     struct otx2_cpt_inst_info *info = NULL;
0238     union otx2_cpt_res_s *result = NULL;
0239     struct otx2_cpt_iq_command iq_cmd;
0240     union otx2_cpt_inst_s cptinst;
0241     int retry, ret = 0;
0242     u8 resume_sender;
0243     gfp_t gfp;
0244 
0245     gfp = (req->areq->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL :
0246                                   GFP_ATOMIC;
0247     if (unlikely(!otx2_cptlf_started(lf->lfs)))
0248         return -ENODEV;
0249 
0250     info = info_create(pdev, req, gfp);
0251     if (unlikely(!info)) {
0252         dev_err(&pdev->dev, "Setting up cpt inst info failed");
0253         return -ENOMEM;
0254     }
0255     cpt_req->dlen = info->dlen;
0256 
0257     result = info->completion_addr;
0258     result->s.compcode = OTX2_CPT_COMPLETION_CODE_INIT;
0259 
0260     spin_lock_bh(&pqueue->lock);
0261     pentry = get_free_pending_entry(pqueue, pqueue->qlen);
0262     retry = CPT_PENTRY_TIMEOUT / CPT_PENTRY_STEP;
0263     while (unlikely(!pentry) && retry--) {
0264         spin_unlock_bh(&pqueue->lock);
0265         udelay(CPT_PENTRY_STEP);
0266         spin_lock_bh(&pqueue->lock);
0267         pentry = get_free_pending_entry(pqueue, pqueue->qlen);
0268     }
0269 
0270     if (unlikely(!pentry)) {
0271         ret = -ENOSPC;
0272         goto destroy_info;
0273     }
0274 
0275     /*
0276      * Check if we are close to filling in entire pending queue,
0277      * if so then tell the sender to stop/sleep by returning -EBUSY
0278      * We do it only for context which can sleep (GFP_KERNEL)
0279      */
0280     if (gfp == GFP_KERNEL &&
0281         pqueue->pending_count > (pqueue->qlen - CPT_IQ_STOP_MARGIN)) {
0282         pentry->resume_sender = true;
0283     } else
0284         pentry->resume_sender = false;
0285     resume_sender = pentry->resume_sender;
0286     pqueue->pending_count++;
0287 
0288     pentry->completion_addr = info->completion_addr;
0289     pentry->info = info;
0290     pentry->callback = req->callback;
0291     pentry->areq = req->areq;
0292     pentry->busy = true;
0293     info->pentry = pentry;
0294     info->time_in = jiffies;
0295     info->req = req;
0296 
0297     /* Fill in the command */
0298     iq_cmd.cmd.u = 0;
0299     iq_cmd.cmd.s.opcode = cpu_to_be16(cpt_req->opcode.flags);
0300     iq_cmd.cmd.s.param1 = cpu_to_be16(cpt_req->param1);
0301     iq_cmd.cmd.s.param2 = cpu_to_be16(cpt_req->param2);
0302     iq_cmd.cmd.s.dlen   = cpu_to_be16(cpt_req->dlen);
0303 
0304     /* 64-bit swap for microcode data reads, not needed for addresses*/
0305     cpu_to_be64s(&iq_cmd.cmd.u);
0306     iq_cmd.dptr = info->dptr_baddr;
0307     iq_cmd.rptr = 0;
0308     iq_cmd.cptr.u = 0;
0309     iq_cmd.cptr.s.grp = ctrl->s.grp;
0310 
0311     /* Fill in the CPT_INST_S type command for HW interpretation */
0312     otx2_cpt_fill_inst(&cptinst, &iq_cmd, info->comp_baddr);
0313 
0314     /* Print debug info if enabled */
0315     otx2_cpt_dump_sg_list(pdev, req);
0316     pr_debug("Cpt_inst_s hexdump (%d bytes)\n", OTX2_CPT_INST_SIZE);
0317     print_hex_dump_debug("", 0, 16, 1, &cptinst, OTX2_CPT_INST_SIZE, false);
0318     pr_debug("Dptr hexdump (%d bytes)\n", cpt_req->dlen);
0319     print_hex_dump_debug("", 0, 16, 1, info->in_buffer,
0320                  cpt_req->dlen, false);
0321 
0322     /* Send CPT command */
0323     lf->lfs->ops->send_cmd(&cptinst, 1, lf);
0324 
0325     /*
0326      * We allocate and prepare pending queue entry in critical section
0327      * together with submitting CPT instruction to CPT instruction queue
0328      * to make sure that order of CPT requests is the same in both
0329      * pending and instruction queues
0330      */
0331     spin_unlock_bh(&pqueue->lock);
0332 
0333     ret = resume_sender ? -EBUSY : -EINPROGRESS;
0334     return ret;
0335 
0336 destroy_info:
0337     spin_unlock_bh(&pqueue->lock);
0338     otx2_cpt_info_destroy(pdev, info);
0339     return ret;
0340 }
0341 
0342 int otx2_cpt_do_request(struct pci_dev *pdev, struct otx2_cpt_req_info *req,
0343             int cpu_num)
0344 {
0345     struct otx2_cptvf_dev *cptvf = pci_get_drvdata(pdev);
0346     struct otx2_cptlfs_info *lfs = &cptvf->lfs;
0347 
0348     return process_request(lfs->pdev, req, &lfs->lf[cpu_num].pqueue,
0349                    &lfs->lf[cpu_num]);
0350 }
0351 
0352 static int cpt_process_ccode(struct otx2_cptlfs_info *lfs,
0353                  union otx2_cpt_res_s *cpt_status,
0354                  struct otx2_cpt_inst_info *info,
0355                  u32 *res_code)
0356 {
0357     u8 uc_ccode = lfs->ops->cpt_get_uc_compcode(cpt_status);
0358     u8 ccode = lfs->ops->cpt_get_compcode(cpt_status);
0359     struct pci_dev *pdev = lfs->pdev;
0360 
0361     switch (ccode) {
0362     case OTX2_CPT_COMP_E_FAULT:
0363         dev_err(&pdev->dev,
0364             "Request failed with DMA fault\n");
0365         otx2_cpt_dump_sg_list(pdev, info->req);
0366         break;
0367 
0368     case OTX2_CPT_COMP_E_HWERR:
0369         dev_err(&pdev->dev,
0370             "Request failed with hardware error\n");
0371         otx2_cpt_dump_sg_list(pdev, info->req);
0372         break;
0373 
0374     case OTX2_CPT_COMP_E_INSTERR:
0375         dev_err(&pdev->dev,
0376             "Request failed with instruction error\n");
0377         otx2_cpt_dump_sg_list(pdev, info->req);
0378         break;
0379 
0380     case OTX2_CPT_COMP_E_NOTDONE:
0381         /* check for timeout */
0382         if (time_after_eq(jiffies, info->time_in +
0383                   CPT_COMMAND_TIMEOUT * HZ))
0384             dev_warn(&pdev->dev,
0385                  "Request timed out 0x%p", info->req);
0386         else if (info->extra_time < CPT_TIME_IN_RESET_COUNT) {
0387             info->time_in = jiffies;
0388             info->extra_time++;
0389         }
0390         return 1;
0391 
0392     case OTX2_CPT_COMP_E_GOOD:
0393     case OTX2_CPT_COMP_E_WARN:
0394         /*
0395          * Check microcode completion code, it is only valid
0396          * when completion code is CPT_COMP_E::GOOD
0397          */
0398         if (uc_ccode != OTX2_CPT_UCC_SUCCESS) {
0399             /*
0400              * If requested hmac is truncated and ucode returns
0401              * s/g write length error then we report success
0402              * because ucode writes as many bytes of calculated
0403              * hmac as available in gather buffer and reports
0404              * s/g write length error if number of bytes in gather
0405              * buffer is less than full hmac size.
0406              */
0407             if (info->req->is_trunc_hmac &&
0408                 uc_ccode == OTX2_CPT_UCC_SG_WRITE_LENGTH) {
0409                 *res_code = 0;
0410                 break;
0411             }
0412 
0413             dev_err(&pdev->dev,
0414                 "Request failed with software error code 0x%x\n",
0415                 cpt_status->s.uc_compcode);
0416             otx2_cpt_dump_sg_list(pdev, info->req);
0417             break;
0418         }
0419         /* Request has been processed with success */
0420         *res_code = 0;
0421         break;
0422 
0423     default:
0424         dev_err(&pdev->dev,
0425             "Request returned invalid status %d\n", ccode);
0426         break;
0427     }
0428     return 0;
0429 }
0430 
0431 static inline void process_pending_queue(struct otx2_cptlfs_info *lfs,
0432                      struct otx2_cpt_pending_queue *pqueue)
0433 {
0434     struct otx2_cpt_pending_entry *resume_pentry = NULL;
0435     void (*callback)(int status, void *arg, void *req);
0436     struct otx2_cpt_pending_entry *pentry = NULL;
0437     union otx2_cpt_res_s *cpt_status = NULL;
0438     struct otx2_cpt_inst_info *info = NULL;
0439     struct otx2_cpt_req_info *req = NULL;
0440     struct crypto_async_request *areq;
0441     struct pci_dev *pdev = lfs->pdev;
0442     u32 res_code, resume_index;
0443 
0444     while (1) {
0445         spin_lock_bh(&pqueue->lock);
0446         pentry = &pqueue->head[pqueue->front];
0447 
0448         if (WARN_ON(!pentry)) {
0449             spin_unlock_bh(&pqueue->lock);
0450             break;
0451         }
0452 
0453         res_code = -EINVAL;
0454         if (unlikely(!pentry->busy)) {
0455             spin_unlock_bh(&pqueue->lock);
0456             break;
0457         }
0458 
0459         if (unlikely(!pentry->callback)) {
0460             dev_err(&pdev->dev, "Callback NULL\n");
0461             goto process_pentry;
0462         }
0463 
0464         info = pentry->info;
0465         if (unlikely(!info)) {
0466             dev_err(&pdev->dev, "Pending entry post arg NULL\n");
0467             goto process_pentry;
0468         }
0469 
0470         req = info->req;
0471         if (unlikely(!req)) {
0472             dev_err(&pdev->dev, "Request NULL\n");
0473             goto process_pentry;
0474         }
0475 
0476         cpt_status = pentry->completion_addr;
0477         if (unlikely(!cpt_status)) {
0478             dev_err(&pdev->dev, "Completion address NULL\n");
0479             goto process_pentry;
0480         }
0481 
0482         if (cpt_process_ccode(lfs, cpt_status, info, &res_code)) {
0483             spin_unlock_bh(&pqueue->lock);
0484             return;
0485         }
0486         info->pdev = pdev;
0487 
0488 process_pentry:
0489         /*
0490          * Check if we should inform sending side to resume
0491          * We do it CPT_IQ_RESUME_MARGIN elements in advance before
0492          * pending queue becomes empty
0493          */
0494         resume_index = modulo_inc(pqueue->front, pqueue->qlen,
0495                       CPT_IQ_RESUME_MARGIN);
0496         resume_pentry = &pqueue->head[resume_index];
0497         if (resume_pentry &&
0498             resume_pentry->resume_sender) {
0499             resume_pentry->resume_sender = false;
0500             callback = resume_pentry->callback;
0501             areq = resume_pentry->areq;
0502 
0503             if (callback) {
0504                 spin_unlock_bh(&pqueue->lock);
0505 
0506                 /*
0507                  * EINPROGRESS is an indication for sending
0508                  * side that it can resume sending requests
0509                  */
0510                 callback(-EINPROGRESS, areq, info);
0511                 spin_lock_bh(&pqueue->lock);
0512             }
0513         }
0514 
0515         callback = pentry->callback;
0516         areq = pentry->areq;
0517         free_pentry(pentry);
0518 
0519         pqueue->pending_count--;
0520         pqueue->front = modulo_inc(pqueue->front, pqueue->qlen, 1);
0521         spin_unlock_bh(&pqueue->lock);
0522 
0523         /*
0524          * Call callback after current pending entry has been
0525          * processed, we don't do it if the callback pointer is
0526          * invalid.
0527          */
0528         if (callback)
0529             callback(res_code, areq, info);
0530     }
0531 }
0532 
0533 void otx2_cpt_post_process(struct otx2_cptlf_wqe *wqe)
0534 {
0535     process_pending_queue(wqe->lfs,
0536                   &wqe->lfs->lf[wqe->lf_num].pqueue);
0537 }
0538 
0539 int otx2_cpt_get_kcrypto_eng_grp_num(struct pci_dev *pdev)
0540 {
0541     struct otx2_cptvf_dev *cptvf = pci_get_drvdata(pdev);
0542 
0543     return cptvf->lfs.kcrypto_eng_grp_num;
0544 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!