cavium/nitrox/nitrox_reqmgr.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 #include <linux/gfp.h>
0003 #include <linux/workqueue.h>
0004 #include <crypto/internal/skcipher.h>
0005
0006 #include "nitrox_common.h"
0007 #include "nitrox_dev.h"
0008 #include "nitrox_req.h"
0009 #include "nitrox_csr.h"
0010
0011 /* SLC_STORE_INFO */
0012 #define MIN_UDD_LEN 16
0013 /* PKT_IN_HDR + SLC_STORE_INFO */
0014 #define FDATA_SIZE 32
0015 /* Base destination port for the solicited requests */
0016 #define SOLICIT_BASE_DPORT 256
0017
0018 #define REQ_NOT_POSTED 1
0019 #define REQ_BACKLOG    2
0020 #define REQ_POSTED     3
0021
0022 /*
0023  * Response codes from SE microcode
0024  * 0x00 - Success
0025  *   Completion with no error
0026  * 0x43 - ERR_GC_DATA_LEN_INVALID
0027  *   Invalid Data length if Encryption Data length is
0028  *   less than 16 bytes for AES-XTS and AES-CTS.
0029  * 0x45 - ERR_GC_CTX_LEN_INVALID
0030  *   Invalid context length: CTXL != 23 words.
0031  * 0x4F - ERR_GC_DOCSIS_CIPHER_INVALID
0032  *   DOCSIS support is enabled with other than
0033  *   AES/DES-CBC mode encryption.
0034  * 0x50 - ERR_GC_DOCSIS_OFFSET_INVALID
0035  *   Authentication offset is other than 0 with
0036  *   Encryption IV source = 0.
0037  *   Authentication offset is other than 8 (DES)/16 (AES)
0038  *   with Encryption IV source = 1
0039  * 0x51 - ERR_GC_CRC32_INVALID_SELECTION
0040  *   CRC32 is enabled for other than DOCSIS encryption.
0041  * 0x52 - ERR_GC_AES_CCM_FLAG_INVALID
0042  *   Invalid flag options in AES-CCM IV.
0043  */
0044
0045 static inline int incr_index(int index, int count, int max)
0046 {
0047     if ((index + count) >= max)
0048         index = index + count - max;
0049     else
0050         index += count;
0051
0052     return index;
0053 }
0054
0055 static void softreq_unmap_sgbufs(struct nitrox_softreq *sr)
0056 {
0057     struct nitrox_device *ndev = sr->ndev;
0058     struct device *dev = DEV(ndev);
0059
0060
0061     dma_unmap_sg(dev, sr->in.sg, sg_nents(sr->in.sg),
0062              DMA_BIDIRECTIONAL);
0063     dma_unmap_single(dev, sr->in.sgcomp_dma, sr->in.sgcomp_len,
0064              DMA_TO_DEVICE);
0065     kfree(sr->in.sgcomp);
0066     sr->in.sg = NULL;
0067     sr->in.sgmap_cnt = 0;
0068
0069     dma_unmap_sg(dev, sr->out.sg, sg_nents(sr->out.sg),
0070              DMA_BIDIRECTIONAL);
0071     dma_unmap_single(dev, sr->out.sgcomp_dma, sr->out.sgcomp_len,
0072              DMA_TO_DEVICE);
0073     kfree(sr->out.sgcomp);
0074     sr->out.sg = NULL;
0075     sr->out.sgmap_cnt = 0;
0076 }
0077
0078 static void softreq_destroy(struct nitrox_softreq *sr)
0079 {
0080     softreq_unmap_sgbufs(sr);
0081     kfree(sr);
0082 }
0083
0084 /**
0085  * create_sg_component - create SG componets for N5 device.
0086  * @sr: Request structure
0087  * @sgtbl: SG table
0088  * @map_nents: number of dma mapped entries
0089  *
0090  * Component structure
0091  *
0092  *   63     48 47     32 31    16 15      0
0093  *   --------------------------------------
0094  *   |   LEN0  |  LEN1  |  LEN2  |  LEN3  |
0095  *   |-------------------------------------
0096  *   |               PTR0                 |
0097  *   --------------------------------------
0098  *   |               PTR1                 |
0099  *   --------------------------------------
0100  *   |               PTR2                 |
0101  *   --------------------------------------
0102  *   |               PTR3                 |
0103  *   --------------------------------------
0104  *
0105  *   Returns 0 if success or a negative errno code on error.
0106  */
0107 static int create_sg_component(struct nitrox_softreq *sr,
0108                    struct nitrox_sgtable *sgtbl, int map_nents)
0109 {
0110     struct nitrox_device *ndev = sr->ndev;
0111     struct nitrox_sgcomp *sgcomp;
0112     struct scatterlist *sg;
0113     dma_addr_t dma;
0114     size_t sz_comp;
0115     int i, j, nr_sgcomp;
0116
0117     nr_sgcomp = roundup(map_nents, 4) / 4;
0118
0119     /* each component holds 4 dma pointers */
0120     sz_comp = nr_sgcomp * sizeof(*sgcomp);
0121     sgcomp = kzalloc(sz_comp, sr->gfp);
0122     if (!sgcomp)
0123         return -ENOMEM;
0124
0125     sgtbl->sgcomp = sgcomp;
0126
0127     sg = sgtbl->sg;
0128     /* populate device sg component */
0129     for (i = 0; i < nr_sgcomp; i++) {
0130         for (j = 0; j < 4 && sg; j++) {
0131             sgcomp[i].len[j] = cpu_to_be16(sg_dma_len(sg));
0132             sgcomp[i].dma[j] = cpu_to_be64(sg_dma_address(sg));
0133             sg = sg_next(sg);
0134         }
0135     }
0136     /* map the device sg component */
0137     dma = dma_map_single(DEV(ndev), sgtbl->sgcomp, sz_comp, DMA_TO_DEVICE);
0138     if (dma_mapping_error(DEV(ndev), dma)) {
0139         kfree(sgtbl->sgcomp);
0140         sgtbl->sgcomp = NULL;
0141         return -ENOMEM;
0142     }
0143
0144     sgtbl->sgcomp_dma = dma;
0145     sgtbl->sgcomp_len = sz_comp;
0146
0147     return 0;
0148 }
0149
0150 /**
0151  * dma_map_inbufs - DMA map input sglist and creates sglist component
0152  *                  for N5 device.
0153  * @sr: Request structure
0154  * @req: Crypto request structre
0155  *
0156  * Returns 0 if successful or a negative errno code on error.
0157  */
0158 static int dma_map_inbufs(struct nitrox_softreq *sr,
0159               struct se_crypto_request *req)
0160 {
0161     struct device *dev = DEV(sr->ndev);
0162     struct scatterlist *sg;
0163     int i, nents, ret = 0;
0164
0165     nents = dma_map_sg(dev, req->src, sg_nents(req->src),
0166                DMA_BIDIRECTIONAL);
0167     if (!nents)
0168         return -EINVAL;
0169
0170     for_each_sg(req->src, sg, nents, i)
0171         sr->in.total_bytes += sg_dma_len(sg);
0172
0173     sr->in.sg = req->src;
0174     sr->in.sgmap_cnt = nents;
0175     ret = create_sg_component(sr, &sr->in, sr->in.sgmap_cnt);
0176     if (ret)
0177         goto incomp_err;
0178
0179     return 0;
0180
0181 incomp_err:
0182     dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_BIDIRECTIONAL);
0183     sr->in.sgmap_cnt = 0;
0184     return ret;
0185 }
0186
0187 static int dma_map_outbufs(struct nitrox_softreq *sr,
0188                struct se_crypto_request *req)
0189 {
0190     struct device *dev = DEV(sr->ndev);
0191     int nents, ret = 0;
0192
0193     nents = dma_map_sg(dev, req->dst, sg_nents(req->dst),
0194                DMA_BIDIRECTIONAL);
0195     if (!nents)
0196         return -EINVAL;
0197
0198     sr->out.sg = req->dst;
0199     sr->out.sgmap_cnt = nents;
0200     ret = create_sg_component(sr, &sr->out, sr->out.sgmap_cnt);
0201     if (ret)
0202         goto outcomp_map_err;
0203
0204     return 0;
0205
0206 outcomp_map_err:
0207     dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_BIDIRECTIONAL);
0208     sr->out.sgmap_cnt = 0;
0209     sr->out.sg = NULL;
0210     return ret;
0211 }
0212
0213 static inline int softreq_map_iobuf(struct nitrox_softreq *sr,
0214                     struct se_crypto_request *creq)
0215 {
0216     int ret;
0217
0218     ret = dma_map_inbufs(sr, creq);
0219     if (ret)
0220         return ret;
0221
0222     ret = dma_map_outbufs(sr, creq);
0223     if (ret)
0224         softreq_unmap_sgbufs(sr);
0225
0226     return ret;
0227 }
0228
0229 static inline void backlog_list_add(struct nitrox_softreq *sr,
0230                     struct nitrox_cmdq *cmdq)
0231 {
0232     INIT_LIST_HEAD(&sr->backlog);
0233
0234     spin_lock_bh(&cmdq->backlog_qlock);
0235     list_add_tail(&sr->backlog, &cmdq->backlog_head);
0236     atomic_inc(&cmdq->backlog_count);
0237     atomic_set(&sr->status, REQ_BACKLOG);
0238     spin_unlock_bh(&cmdq->backlog_qlock);
0239 }
0240
0241 static inline void response_list_add(struct nitrox_softreq *sr,
0242                      struct nitrox_cmdq *cmdq)
0243 {
0244     INIT_LIST_HEAD(&sr->response);
0245
0246     spin_lock_bh(&cmdq->resp_qlock);
0247     list_add_tail(&sr->response, &cmdq->response_head);
0248     spin_unlock_bh(&cmdq->resp_qlock);
0249 }
0250
0251 static inline void response_list_del(struct nitrox_softreq *sr,
0252                      struct nitrox_cmdq *cmdq)
0253 {
0254     spin_lock_bh(&cmdq->resp_qlock);
0255     list_del(&sr->response);
0256     spin_unlock_bh(&cmdq->resp_qlock);
0257 }
0258
0259 static struct nitrox_softreq *
0260 get_first_response_entry(struct nitrox_cmdq *cmdq)
0261 {
0262     return list_first_entry_or_null(&cmdq->response_head,
0263                     struct nitrox_softreq, response);
0264 }
0265
0266 static inline bool cmdq_full(struct nitrox_cmdq *cmdq, int qlen)
0267 {
0268     if (atomic_inc_return(&cmdq->pending_count) > qlen) {
0269         atomic_dec(&cmdq->pending_count);
0270         /* sync with other cpus */
0271         smp_mb__after_atomic();
0272         return true;
0273     }
0274     /* sync with other cpus */
0275     smp_mb__after_atomic();
0276     return false;
0277 }
0278
0279 /**
0280  * post_se_instr - Post SE instruction to Packet Input ring
0281  * @sr: Request structure
0282  * @cmdq: Command queue structure
0283  *
0284  * Returns 0 if successful or a negative error code,
0285  * if no space in ring.
0286  */
0287 static void post_se_instr(struct nitrox_softreq *sr,
0288               struct nitrox_cmdq *cmdq)
0289 {
0290     struct nitrox_device *ndev = sr->ndev;
0291     int idx;
0292     u8 *ent;
0293
0294     spin_lock_bh(&cmdq->cmd_qlock);
0295
0296     idx = cmdq->write_idx;
0297     /* copy the instruction */
0298     ent = cmdq->base + (idx * cmdq->instr_size);
0299     memcpy(ent, &sr->instr, cmdq->instr_size);
0300
0301     atomic_set(&sr->status, REQ_POSTED);
0302     response_list_add(sr, cmdq);
0303     sr->tstamp = jiffies;
0304     /* flush the command queue updates */
0305     dma_wmb();
0306
0307     /* Ring doorbell with count 1 */
0308     writeq(1, cmdq->dbell_csr_addr);
0309
0310     cmdq->write_idx = incr_index(idx, 1, ndev->qlen);
0311
0312     spin_unlock_bh(&cmdq->cmd_qlock);
0313
0314     /* increment the posted command count */
0315     atomic64_inc(&ndev->stats.posted);
0316 }
0317
0318 static int post_backlog_cmds(struct nitrox_cmdq *cmdq)
0319 {
0320     struct nitrox_device *ndev = cmdq->ndev;
0321     struct nitrox_softreq *sr, *tmp;
0322     int ret = 0;
0323
0324     if (!atomic_read(&cmdq->backlog_count))
0325         return 0;
0326
0327     spin_lock_bh(&cmdq->backlog_qlock);
0328
0329     list_for_each_entry_safe(sr, tmp, &cmdq->backlog_head, backlog) {
0330         /* submit until space available */
0331         if (unlikely(cmdq_full(cmdq, ndev->qlen))) {
0332             ret = -ENOSPC;
0333             break;
0334         }
0335         /* delete from backlog list */
0336         list_del(&sr->backlog);
0337         atomic_dec(&cmdq->backlog_count);
0338         /* sync with other cpus */
0339         smp_mb__after_atomic();
0340
0341         /* post the command */
0342         post_se_instr(sr, cmdq);
0343     }
0344     spin_unlock_bh(&cmdq->backlog_qlock);
0345
0346     return ret;
0347 }
0348
0349 static int nitrox_enqueue_request(struct nitrox_softreq *sr)
0350 {
0351     struct nitrox_cmdq *cmdq = sr->cmdq;
0352     struct nitrox_device *ndev = sr->ndev;
0353
0354     /* try to post backlog requests */
0355     post_backlog_cmds(cmdq);
0356
0357     if (unlikely(cmdq_full(cmdq, ndev->qlen))) {
0358         if (!(sr->flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
0359             /* increment drop count */
0360             atomic64_inc(&ndev->stats.dropped);
0361             return -ENOSPC;
0362         }
0363         /* add to backlog list */
0364         backlog_list_add(sr, cmdq);
0365         return -EINPROGRESS;
0366     }
0367     post_se_instr(sr, cmdq);
0368
0369     return -EINPROGRESS;
0370 }
0371
0372 /**
0373  * nitrox_process_se_request - Send request to SE core
0374  * @ndev: NITROX device
0375  * @req: Crypto request
0376  * @callback: Completion callback
0377  * @cb_arg: Completion callback arguments
0378  *
0379  * Returns 0 on success, or a negative error code.
0380  */
0381 int nitrox_process_se_request(struct nitrox_device *ndev,
0382                   struct se_crypto_request *req,
0383                   completion_t callback,
0384                   void *cb_arg)
0385 {
0386     struct nitrox_softreq *sr;
0387     dma_addr_t ctx_handle = 0;
0388     int qno, ret = 0;
0389
0390     if (!nitrox_ready(ndev))
0391         return -ENODEV;
0392
0393     sr = kzalloc(sizeof(*sr), req->gfp);
0394     if (!sr)
0395         return -ENOMEM;
0396
0397     sr->ndev = ndev;
0398     sr->flags = req->flags;
0399     sr->gfp = req->gfp;
0400     sr->callback = callback;
0401     sr->cb_arg = cb_arg;
0402
0403     atomic_set(&sr->status, REQ_NOT_POSTED);
0404
0405     sr->resp.orh = req->orh;
0406     sr->resp.completion = req->comp;
0407
0408     ret = softreq_map_iobuf(sr, req);
0409     if (ret) {
0410         kfree(sr);
0411         return ret;
0412     }
0413
0414     /* get the context handle */
0415     if (req->ctx_handle) {
0416         struct ctx_hdr *hdr;
0417         u8 *ctx_ptr;
0418
0419         ctx_ptr = (u8 *)(uintptr_t)req->ctx_handle;
0420         hdr = (struct ctx_hdr *)(ctx_ptr - sizeof(struct ctx_hdr));
0421         ctx_handle = hdr->ctx_dma;
0422     }
0423
0424     /* select the queue */
0425     qno = smp_processor_id() % ndev->nr_queues;
0426
0427     sr->cmdq = &ndev->pkt_inq[qno];
0428
0429     /*
0430      * 64-Byte Instruction Format
0431      *
0432      *  ----------------------
0433      *  |      DPTR0         | 8 bytes
0434      *  ----------------------
0435      *  |  PKT_IN_INSTR_HDR  | 8 bytes
0436      *  ----------------------
0437      *  |    PKT_IN_HDR      | 16 bytes
0438      *  ----------------------
0439      *  |    SLC_INFO        | 16 bytes
0440      *  ----------------------
0441      *  |   Front data       | 16 bytes
0442      *  ----------------------
0443      */
0444
0445     /* fill the packet instruction */
0446     /* word 0 */
0447     sr->instr.dptr0 = cpu_to_be64(sr->in.sgcomp_dma);
0448
0449     /* word 1 */
0450     sr->instr.ih.value = 0;
0451     sr->instr.ih.s.g = 1;
0452     sr->instr.ih.s.gsz = sr->in.sgmap_cnt;
0453     sr->instr.ih.s.ssz = sr->out.sgmap_cnt;
0454     sr->instr.ih.s.fsz = FDATA_SIZE + sizeof(struct gphdr);
0455     sr->instr.ih.s.tlen = sr->instr.ih.s.fsz + sr->in.total_bytes;
0456     sr->instr.ih.bev = cpu_to_be64(sr->instr.ih.value);
0457
0458     /* word 2 */
0459     sr->instr.irh.value[0] = 0;
0460     sr->instr.irh.s.uddl = MIN_UDD_LEN;
0461     /* context length in 64-bit words */
0462     sr->instr.irh.s.ctxl = (req->ctrl.s.ctxl / 8);
0463     /* offset from solicit base port 256 */
0464     sr->instr.irh.s.destport = SOLICIT_BASE_DPORT + qno;
0465     sr->instr.irh.s.ctxc = req->ctrl.s.ctxc;
0466     sr->instr.irh.s.arg = req->ctrl.s.arg;
0467     sr->instr.irh.s.opcode = req->opcode;
0468     sr->instr.irh.bev[0] = cpu_to_be64(sr->instr.irh.value[0]);
0469
0470     /* word 3 */
0471     sr->instr.irh.s.ctxp = cpu_to_be64(ctx_handle);
0472
0473     /* word 4 */
0474     sr->instr.slc.value[0] = 0;
0475     sr->instr.slc.s.ssz = sr->out.sgmap_cnt;
0476     sr->instr.slc.bev[0] = cpu_to_be64(sr->instr.slc.value[0]);
0477
0478     /* word 5 */
0479     sr->instr.slc.s.rptr = cpu_to_be64(sr->out.sgcomp_dma);
0480
0481     /*
0482      * No conversion for front data,
0483      * It goes into payload
0484      * put GP Header in front data
0485      */
0486     sr->instr.fdata[0] = *((u64 *)&req->gph);
0487     sr->instr.fdata[1] = 0;
0488
0489     ret = nitrox_enqueue_request(sr);
0490     if (ret == -ENOSPC)
0491         goto send_fail;
0492
0493     return ret;
0494
0495 send_fail:
0496     softreq_destroy(sr);
0497     return ret;
0498 }
0499
0500 static inline int cmd_timeout(unsigned long tstamp, unsigned long timeout)
0501 {
0502     return time_after_eq(jiffies, (tstamp + timeout));
0503 }
0504
0505 void backlog_qflush_work(struct work_struct *work)
0506 {
0507     struct nitrox_cmdq *cmdq;
0508
0509     cmdq = container_of(work, struct nitrox_cmdq, backlog_qflush);
0510     post_backlog_cmds(cmdq);
0511 }
0512
0513 static bool sr_completed(struct nitrox_softreq *sr)
0514 {
0515     u64 orh = READ_ONCE(*sr->resp.orh);
0516     unsigned long timeout = jiffies + msecs_to_jiffies(1);
0517
0518     if ((orh != PENDING_SIG) && (orh & 0xff))
0519         return true;
0520
0521     while (READ_ONCE(*sr->resp.completion) == PENDING_SIG) {
0522         if (time_after(jiffies, timeout)) {
0523             pr_err("comp not done\n");
0524             return false;
0525         }
0526     }
0527
0528     return true;
0529 }
0530
0531 /**
0532  * process_response_list - process completed requests
0533  * @cmdq: Command queue structure
0534  *
0535  * Returns the number of responses processed.
0536  */
0537 static void process_response_list(struct nitrox_cmdq *cmdq)
0538 {
0539     struct nitrox_device *ndev = cmdq->ndev;
0540     struct nitrox_softreq *sr;
0541     int req_completed = 0, err = 0, budget;
0542     completion_t callback;
0543     void *cb_arg;
0544
0545     /* check all pending requests */
0546     budget = atomic_read(&cmdq->pending_count);
0547
0548     while (req_completed < budget) {
0549         sr = get_first_response_entry(cmdq);
0550         if (!sr)
0551             break;
0552
0553         if (atomic_read(&sr->status) != REQ_POSTED)
0554             break;
0555
0556         /* check orh and completion bytes updates */
0557         if (!sr_completed(sr)) {
0558             /* request not completed, check for timeout */
0559             if (!cmd_timeout(sr->tstamp, ndev->timeout))
0560                 break;
0561             dev_err_ratelimited(DEV(ndev),
0562                         "Request timeout, orh 0x%016llx\n",
0563                         READ_ONCE(*sr->resp.orh));
0564         }
0565         atomic_dec(&cmdq->pending_count);
0566         atomic64_inc(&ndev->stats.completed);
0567         /* sync with other cpus */
0568         smp_mb__after_atomic();
0569         /* remove from response list */
0570         response_list_del(sr, cmdq);
0571         /* ORH error code */
0572         err = READ_ONCE(*sr->resp.orh) & 0xff;
0573         callback = sr->callback;
0574         cb_arg = sr->cb_arg;
0575         softreq_destroy(sr);
0576         if (callback)
0577             callback(cb_arg, err);
0578
0579         req_completed++;
0580     }
0581 }
0582
0583 /*
0584  * pkt_slc_resp_tasklet - post processing of SE responses
0585  */
0586 void pkt_slc_resp_tasklet(unsigned long data)
0587 {
0588     struct nitrox_q_vector *qvec = (void *)(uintptr_t)(data);
0589     struct nitrox_cmdq *cmdq = qvec->cmdq;
0590     union nps_pkt_slc_cnts slc_cnts;
0591
0592     /* read completion count */
0593     slc_cnts.value = readq(cmdq->compl_cnt_csr_addr);
0594     /* resend the interrupt if more work to do */
0595     slc_cnts.s.resend = 1;
0596
0597     process_response_list(cmdq);
0598
0599     /*
0600      * clear the interrupt with resend bit enabled,
0601      * MSI-X interrupt generates if Completion count > Threshold
0602      */
0603     writeq(slc_cnts.value, cmdq->compl_cnt_csr_addr);
0604
0605     if (atomic_read(&cmdq->backlog_count))
0606         schedule_work(&cmdq->backlog_qflush);
0607 }