Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * IBM Accelerator Family 'GenWQE'
0004  *
0005  * (C) Copyright IBM Corp. 2013
0006  *
0007  * Author: Frank Haverkamp <haver@linux.vnet.ibm.com>
0008  * Author: Joerg-Stephan Vogt <jsvogt@de.ibm.com>
0009  * Author: Michael Jung <mijung@gmx.net>
0010  * Author: Michael Ruettger <michael@ibmra.de>
0011  */
0012 
0013 /*
0014  * Device Driver Control Block (DDCB) queue support. Definition of
0015  * interrupt handlers for queue support as well as triggering the
0016  * health monitor code in case of problems. The current hardware uses
0017  * an MSI interrupt which is shared between error handling and
0018  * functional code.
0019  */
0020 
0021 #include <linux/types.h>
0022 #include <linux/sched.h>
0023 #include <linux/wait.h>
0024 #include <linux/pci.h>
0025 #include <linux/string.h>
0026 #include <linux/dma-mapping.h>
0027 #include <linux/delay.h>
0028 #include <linux/module.h>
0029 #include <linux/interrupt.h>
0030 #include <linux/crc-itu-t.h>
0031 
0032 #include "card_base.h"
0033 #include "card_ddcb.h"
0034 
0035 /*
0036  * N: next DDCB, this is where the next DDCB will be put.
0037  * A: active DDCB, this is where the code will look for the next completion.
0038  * x: DDCB is enqueued, we are waiting for its completion.
0039 
0040  * Situation (1): Empty queue
0041  *  +---+---+---+---+---+---+---+---+
0042  *  | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
0043  *  |   |   |   |   |   |   |   |   |
0044  *  +---+---+---+---+---+---+---+---+
0045  *           A/N
0046  *  enqueued_ddcbs = A - N = 2 - 2 = 0
0047  *
0048  * Situation (2): Wrapped, N > A
0049  *  +---+---+---+---+---+---+---+---+
0050  *  | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
0051  *  |   |   | x | x |   |   |   |   |
0052  *  +---+---+---+---+---+---+---+---+
0053  *            A       N
0054  *  enqueued_ddcbs = N - A = 4 - 2 = 2
0055  *
0056  * Situation (3): Queue wrapped, A > N
0057  *  +---+---+---+---+---+---+---+---+
0058  *  | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
0059  *  | x | x |   |   | x | x | x | x |
0060  *  +---+---+---+---+---+---+---+---+
0061  *            N       A
0062  *  enqueued_ddcbs = queue_max  - (A - N) = 8 - (4 - 2) = 6
0063  *
0064  * Situation (4a): Queue full N > A
0065  *  +---+---+---+---+---+---+---+---+
0066  *  | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
0067  *  | x | x | x | x | x | x | x |   |
0068  *  +---+---+---+---+---+---+---+---+
0069  *    A                           N
0070  *
0071  *  enqueued_ddcbs = N - A = 7 - 0 = 7
0072  *
0073  * Situation (4a): Queue full A > N
0074  *  +---+---+---+---+---+---+---+---+
0075  *  | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
0076  *  | x | x | x |   | x | x | x | x |
0077  *  +---+---+---+---+---+---+---+---+
0078  *                N   A
0079  *  enqueued_ddcbs = queue_max - (A - N) = 8 - (4 - 3) = 7
0080  */
0081 
0082 static int queue_empty(struct ddcb_queue *queue)
0083 {
0084     return queue->ddcb_next == queue->ddcb_act;
0085 }
0086 
0087 static int queue_enqueued_ddcbs(struct ddcb_queue *queue)
0088 {
0089     if (queue->ddcb_next >= queue->ddcb_act)
0090         return queue->ddcb_next - queue->ddcb_act;
0091 
0092     return queue->ddcb_max - (queue->ddcb_act - queue->ddcb_next);
0093 }
0094 
0095 static int queue_free_ddcbs(struct ddcb_queue *queue)
0096 {
0097     int free_ddcbs = queue->ddcb_max - queue_enqueued_ddcbs(queue) - 1;
0098 
0099     if (WARN_ON_ONCE(free_ddcbs < 0)) { /* must never ever happen! */
0100         return 0;
0101     }
0102     return free_ddcbs;
0103 }
0104 
0105 /*
0106  * Use of the PRIV field in the DDCB for queue debugging:
0107  *
0108  * (1) Trying to get rid of a DDCB which saw a timeout:
0109  *     pddcb->priv[6] = 0xcc;   # cleared
0110  *
0111  * (2) Append a DDCB via NEXT bit:
0112  *     pddcb->priv[7] = 0xaa;   # appended
0113  *
0114  * (3) DDCB needed tapping:
0115  *     pddcb->priv[7] = 0xbb;   # tapped
0116  *
0117  * (4) DDCB marked as correctly finished:
0118  *     pddcb->priv[6] = 0xff;   # finished
0119  */
0120 
0121 static inline void ddcb_mark_tapped(struct ddcb *pddcb)
0122 {
0123     pddcb->priv[7] = 0xbb;  /* tapped */
0124 }
0125 
0126 static inline void ddcb_mark_appended(struct ddcb *pddcb)
0127 {
0128     pddcb->priv[7] = 0xaa;  /* appended */
0129 }
0130 
0131 static inline void ddcb_mark_cleared(struct ddcb *pddcb)
0132 {
0133     pddcb->priv[6] = 0xcc; /* cleared */
0134 }
0135 
0136 static inline void ddcb_mark_finished(struct ddcb *pddcb)
0137 {
0138     pddcb->priv[6] = 0xff;  /* finished */
0139 }
0140 
0141 static inline void ddcb_mark_unused(struct ddcb *pddcb)
0142 {
0143     pddcb->priv_64 = cpu_to_be64(0); /* not tapped */
0144 }
0145 
0146 /**
0147  * genwqe_crc16() - Generate 16-bit crc as required for DDCBs
0148  * @buff:       pointer to data buffer
0149  * @len:        length of data for calculation
0150  * @init:       initial crc (0xffff at start)
0151  *
0152  * Polynomial = x^16 + x^12 + x^5 + 1   (0x1021)
0153  * Example: 4 bytes 0x01 0x02 0x03 0x04 with init = 0xffff
0154  *          should result in a crc16 of 0x89c3
0155  *
0156  * Return: crc16 checksum in big endian format !
0157  */
0158 static inline u16 genwqe_crc16(const u8 *buff, size_t len, u16 init)
0159 {
0160     return crc_itu_t(init, buff, len);
0161 }
0162 
0163 static void print_ddcb_info(struct genwqe_dev *cd, struct ddcb_queue *queue)
0164 {
0165     int i;
0166     struct ddcb *pddcb;
0167     unsigned long flags;
0168     struct pci_dev *pci_dev = cd->pci_dev;
0169 
0170     spin_lock_irqsave(&cd->print_lock, flags);
0171 
0172     dev_info(&pci_dev->dev,
0173          "DDCB list for card #%d (ddcb_act=%d / ddcb_next=%d):\n",
0174          cd->card_idx, queue->ddcb_act, queue->ddcb_next);
0175 
0176     pddcb = queue->ddcb_vaddr;
0177     for (i = 0; i < queue->ddcb_max; i++) {
0178         dev_err(&pci_dev->dev,
0179             "  %c %-3d: RETC=%03x SEQ=%04x HSI=%02X SHI=%02x PRIV=%06llx CMD=%03x\n",
0180             i == queue->ddcb_act ? '>' : ' ',
0181             i,
0182             be16_to_cpu(pddcb->retc_16),
0183             be16_to_cpu(pddcb->seqnum_16),
0184             pddcb->hsi,
0185             pddcb->shi,
0186             be64_to_cpu(pddcb->priv_64),
0187             pddcb->cmd);
0188         pddcb++;
0189     }
0190     spin_unlock_irqrestore(&cd->print_lock, flags);
0191 }
0192 
0193 struct genwqe_ddcb_cmd *ddcb_requ_alloc(void)
0194 {
0195     struct ddcb_requ *req;
0196 
0197     req = kzalloc(sizeof(*req), GFP_KERNEL);
0198     if (!req)
0199         return NULL;
0200 
0201     return &req->cmd;
0202 }
0203 
0204 void ddcb_requ_free(struct genwqe_ddcb_cmd *cmd)
0205 {
0206     struct ddcb_requ *req = container_of(cmd, struct ddcb_requ, cmd);
0207 
0208     kfree(req);
0209 }
0210 
0211 static inline enum genwqe_requ_state ddcb_requ_get_state(struct ddcb_requ *req)
0212 {
0213     return req->req_state;
0214 }
0215 
0216 static inline void ddcb_requ_set_state(struct ddcb_requ *req,
0217                        enum genwqe_requ_state new_state)
0218 {
0219     req->req_state = new_state;
0220 }
0221 
0222 static inline int ddcb_requ_collect_debug_data(struct ddcb_requ *req)
0223 {
0224     return req->cmd.ddata_addr != 0x0;
0225 }
0226 
0227 /**
0228  * ddcb_requ_finished() - Returns the hardware state of the associated DDCB
0229  * @cd:          pointer to genwqe device descriptor
0230  * @req:         DDCB work request
0231  *
0232  * Status of ddcb_requ mirrors this hardware state, but is copied in
0233  * the ddcb_requ on interrupt/polling function. The lowlevel code
0234  * should check the hardware state directly, the higher level code
0235  * should check the copy.
0236  *
0237  * This function will also return true if the state of the queue is
0238  * not GENWQE_CARD_USED. This enables us to purge all DDCBs in the
0239  * shutdown case.
0240  */
0241 static int ddcb_requ_finished(struct genwqe_dev *cd, struct ddcb_requ *req)
0242 {
0243     return (ddcb_requ_get_state(req) == GENWQE_REQU_FINISHED) ||
0244         (cd->card_state != GENWQE_CARD_USED);
0245 }
0246 
0247 #define RET_DDCB_APPENDED 1
0248 #define RET_DDCB_TAPPED   2
0249 /**
0250  * enqueue_ddcb() - Enqueue a DDCB
0251  * @cd:         pointer to genwqe device descriptor
0252  * @queue:  queue this operation should be done on
0253  * @pddcb:      pointer to ddcb structure
0254  * @ddcb_no:    pointer to ddcb number being tapped
0255  *
0256  * Start execution of DDCB by tapping or append to queue via NEXT
0257  * bit. This is done by an atomic 'compare and swap' instruction and
0258  * checking SHI and HSI of the previous DDCB.
0259  *
0260  * This function must only be called with ddcb_lock held.
0261  *
0262  * Return: 1 if new DDCB is appended to previous
0263  *         2 if DDCB queue is tapped via register/simulation
0264  */
0265 static int enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_queue *queue,
0266             struct ddcb *pddcb, int ddcb_no)
0267 {
0268     unsigned int try;
0269     int prev_no;
0270     struct ddcb *prev_ddcb;
0271     __be32 old, new, icrc_hsi_shi;
0272     u64 num;
0273 
0274     /*
0275      * For performance checks a Dispatch Timestamp can be put into
0276      * DDCB It is supposed to use the SLU's free running counter,
0277      * but this requires PCIe cycles.
0278      */
0279     ddcb_mark_unused(pddcb);
0280 
0281     /* check previous DDCB if already fetched */
0282     prev_no = (ddcb_no == 0) ? queue->ddcb_max - 1 : ddcb_no - 1;
0283     prev_ddcb = &queue->ddcb_vaddr[prev_no];
0284 
0285     /*
0286      * It might have happened that the HSI.FETCHED bit is
0287      * set. Retry in this case. Therefore I expect maximum 2 times
0288      * trying.
0289      */
0290     ddcb_mark_appended(pddcb);
0291     for (try = 0; try < 2; try++) {
0292         old = prev_ddcb->icrc_hsi_shi_32; /* read SHI/HSI in BE32 */
0293 
0294         /* try to append via NEXT bit if prev DDCB is not completed */
0295         if ((old & DDCB_COMPLETED_BE32) != 0x00000000)
0296             break;
0297 
0298         new = (old | DDCB_NEXT_BE32);
0299 
0300         wmb();      /* need to ensure write ordering */
0301         icrc_hsi_shi = cmpxchg(&prev_ddcb->icrc_hsi_shi_32, old, new);
0302 
0303         if (icrc_hsi_shi == old)
0304             return RET_DDCB_APPENDED; /* appended to queue */
0305     }
0306 
0307     /* Queue must be re-started by updating QUEUE_OFFSET */
0308     ddcb_mark_tapped(pddcb);
0309     num = (u64)ddcb_no << 8;
0310 
0311     wmb();          /* need to ensure write ordering */
0312     __genwqe_writeq(cd, queue->IO_QUEUE_OFFSET, num); /* start queue */
0313 
0314     return RET_DDCB_TAPPED;
0315 }
0316 
0317 /**
0318  * copy_ddcb_results() - Copy output state from real DDCB to request
0319  * @req:        pointer to requested DDCB parameters
0320  * @ddcb_no:    pointer to ddcb number being tapped
0321  *
0322  * Copy DDCB ASV to request struct. There is no endian
0323  * conversion made, since data structure in ASV is still
0324  * unknown here.
0325  *
0326  * This is needed by:
0327  *   - genwqe_purge_ddcb()
0328  *   - genwqe_check_ddcb_queue()
0329  */
0330 static void copy_ddcb_results(struct ddcb_requ *req, int ddcb_no)
0331 {
0332     struct ddcb_queue *queue = req->queue;
0333     struct ddcb *pddcb = &queue->ddcb_vaddr[req->num];
0334 
0335     memcpy(&req->cmd.asv[0], &pddcb->asv[0], DDCB_ASV_LENGTH);
0336 
0337     /* copy status flags of the variant part */
0338     req->cmd.vcrc     = be16_to_cpu(pddcb->vcrc_16);
0339     req->cmd.deque_ts = be64_to_cpu(pddcb->deque_ts_64);
0340     req->cmd.cmplt_ts = be64_to_cpu(pddcb->cmplt_ts_64);
0341 
0342     req->cmd.attn     = be16_to_cpu(pddcb->attn_16);
0343     req->cmd.progress = be32_to_cpu(pddcb->progress_32);
0344     req->cmd.retc     = be16_to_cpu(pddcb->retc_16);
0345 
0346     if (ddcb_requ_collect_debug_data(req)) {
0347         int prev_no = (ddcb_no == 0) ?
0348             queue->ddcb_max - 1 : ddcb_no - 1;
0349         struct ddcb *prev_pddcb = &queue->ddcb_vaddr[prev_no];
0350 
0351         memcpy(&req->debug_data.ddcb_finished, pddcb,
0352                sizeof(req->debug_data.ddcb_finished));
0353         memcpy(&req->debug_data.ddcb_prev, prev_pddcb,
0354                sizeof(req->debug_data.ddcb_prev));
0355     }
0356 }
0357 
0358 /**
0359  * genwqe_check_ddcb_queue() - Checks DDCB queue for completed work requests.
0360  * @cd:         pointer to genwqe device descriptor
0361  * @queue:  queue to be checked
0362  *
0363  * Return: Number of DDCBs which were finished
0364  */
0365 static int genwqe_check_ddcb_queue(struct genwqe_dev *cd,
0366                    struct ddcb_queue *queue)
0367 {
0368     unsigned long flags;
0369     int ddcbs_finished = 0;
0370     struct pci_dev *pci_dev = cd->pci_dev;
0371 
0372     spin_lock_irqsave(&queue->ddcb_lock, flags);
0373 
0374     /* FIXME avoid soft locking CPU */
0375     while (!queue_empty(queue) && (ddcbs_finished < queue->ddcb_max)) {
0376 
0377         struct ddcb *pddcb;
0378         struct ddcb_requ *req;
0379         u16 vcrc, vcrc_16, retc_16;
0380 
0381         pddcb = &queue->ddcb_vaddr[queue->ddcb_act];
0382 
0383         if ((pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) ==
0384             0x00000000)
0385             goto go_home; /* not completed, continue waiting */
0386 
0387         wmb();  /*  Add sync to decouple prev. read operations */
0388 
0389         /* Note: DDCB could be purged */
0390         req = queue->ddcb_req[queue->ddcb_act];
0391         if (req == NULL) {
0392             /* this occurs if DDCB is purged, not an error */
0393             /* Move active DDCB further; Nothing to do anymore. */
0394             goto pick_next_one;
0395         }
0396 
0397         /*
0398          * HSI=0x44 (fetched and completed), but RETC is
0399          * 0x101, or even worse 0x000.
0400          *
0401          * In case of seeing the queue in inconsistent state
0402          * we read the errcnts and the queue status to provide
0403          * a trigger for our PCIe analyzer stop capturing.
0404          */
0405         retc_16 = be16_to_cpu(pddcb->retc_16);
0406         if ((pddcb->hsi == 0x44) && (retc_16 <= 0x101)) {
0407             u64 errcnts, status;
0408             u64 ddcb_offs = (u64)pddcb - (u64)queue->ddcb_vaddr;
0409 
0410             errcnts = __genwqe_readq(cd, queue->IO_QUEUE_ERRCNTS);
0411             status  = __genwqe_readq(cd, queue->IO_QUEUE_STATUS);
0412 
0413             dev_err(&pci_dev->dev,
0414                 "[%s] SEQN=%04x HSI=%02x RETC=%03x Q_ERRCNTS=%016llx Q_STATUS=%016llx DDCB_DMA_ADDR=%016llx\n",
0415                 __func__, be16_to_cpu(pddcb->seqnum_16),
0416                 pddcb->hsi, retc_16, errcnts, status,
0417                 queue->ddcb_daddr + ddcb_offs);
0418         }
0419 
0420         copy_ddcb_results(req, queue->ddcb_act);
0421         queue->ddcb_req[queue->ddcb_act] = NULL; /* take from queue */
0422 
0423         dev_dbg(&pci_dev->dev, "FINISHED DDCB#%d\n", req->num);
0424         genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
0425 
0426         ddcb_mark_finished(pddcb);
0427 
0428         /* calculate CRC_16 to see if VCRC is correct */
0429         vcrc = genwqe_crc16(pddcb->asv,
0430                    VCRC_LENGTH(req->cmd.asv_length),
0431                    0xffff);
0432         vcrc_16 = be16_to_cpu(pddcb->vcrc_16);
0433         if (vcrc != vcrc_16) {
0434             printk_ratelimited(KERN_ERR
0435                 "%s %s: err: wrong VCRC pre=%02x vcrc_len=%d bytes vcrc_data=%04x is not vcrc_card=%04x\n",
0436                 GENWQE_DEVNAME, dev_name(&pci_dev->dev),
0437                 pddcb->pre, VCRC_LENGTH(req->cmd.asv_length),
0438                 vcrc, vcrc_16);
0439         }
0440 
0441         ddcb_requ_set_state(req, GENWQE_REQU_FINISHED);
0442         queue->ddcbs_completed++;
0443         queue->ddcbs_in_flight--;
0444 
0445         /* wake up process waiting for this DDCB, and
0446                    processes on the busy queue */
0447         wake_up_interruptible(&queue->ddcb_waitqs[queue->ddcb_act]);
0448         wake_up_interruptible(&queue->busy_waitq);
0449 
0450 pick_next_one:
0451         queue->ddcb_act = (queue->ddcb_act + 1) % queue->ddcb_max;
0452         ddcbs_finished++;
0453     }
0454 
0455  go_home:
0456     spin_unlock_irqrestore(&queue->ddcb_lock, flags);
0457     return ddcbs_finished;
0458 }
0459 
0460 /**
0461  * __genwqe_wait_ddcb(): Waits until DDCB is completed
0462  * @cd:         pointer to genwqe device descriptor
0463  * @req:        pointer to requsted DDCB parameters
0464  *
0465  * The Service Layer will update the RETC in DDCB when processing is
0466  * pending or done.
0467  *
0468  * Return: > 0 remaining jiffies, DDCB completed
0469  *           -ETIMEDOUT when timeout
0470  *           -ERESTARTSYS when ^C
0471  *           -EINVAL when unknown error condition
0472  *
0473  * When an error is returned the called needs to ensure that
0474  * purge_ddcb() is being called to get the &req removed from the
0475  * queue.
0476  */
0477 int __genwqe_wait_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req)
0478 {
0479     int rc;
0480     unsigned int ddcb_no;
0481     struct ddcb_queue *queue;
0482     struct pci_dev *pci_dev = cd->pci_dev;
0483 
0484     if (req == NULL)
0485         return -EINVAL;
0486 
0487     queue = req->queue;
0488     if (queue == NULL)
0489         return -EINVAL;
0490 
0491     ddcb_no = req->num;
0492     if (ddcb_no >= queue->ddcb_max)
0493         return -EINVAL;
0494 
0495     rc = wait_event_interruptible_timeout(queue->ddcb_waitqs[ddcb_no],
0496                 ddcb_requ_finished(cd, req),
0497                 GENWQE_DDCB_SOFTWARE_TIMEOUT * HZ);
0498 
0499     /*
0500      * We need to distinguish 3 cases here:
0501      *   1. rc == 0              timeout occurred
0502      *   2. rc == -ERESTARTSYS   signal received
0503      *   3. rc > 0               remaining jiffies condition is true
0504      */
0505     if (rc == 0) {
0506         struct ddcb_queue *queue = req->queue;
0507         struct ddcb *pddcb;
0508 
0509         /*
0510          * Timeout may be caused by long task switching time.
0511          * When timeout happens, check if the request has
0512          * meanwhile completed.
0513          */
0514         genwqe_check_ddcb_queue(cd, req->queue);
0515         if (ddcb_requ_finished(cd, req))
0516             return rc;
0517 
0518         dev_err(&pci_dev->dev,
0519             "[%s] err: DDCB#%d timeout rc=%d state=%d req @ %p\n",
0520             __func__, req->num, rc, ddcb_requ_get_state(req),
0521             req);
0522         dev_err(&pci_dev->dev,
0523             "[%s]      IO_QUEUE_STATUS=0x%016llx\n", __func__,
0524             __genwqe_readq(cd, queue->IO_QUEUE_STATUS));
0525 
0526         pddcb = &queue->ddcb_vaddr[req->num];
0527         genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
0528 
0529         print_ddcb_info(cd, req->queue);
0530         return -ETIMEDOUT;
0531 
0532     } else if (rc == -ERESTARTSYS) {
0533         return rc;
0534         /*
0535          * EINTR:       Stops the application
0536          * ERESTARTSYS: Restartable systemcall; called again
0537          */
0538 
0539     } else if (rc < 0) {
0540         dev_err(&pci_dev->dev,
0541             "[%s] err: DDCB#%d unknown result (rc=%d) %d!\n",
0542             __func__, req->num, rc, ddcb_requ_get_state(req));
0543         return -EINVAL;
0544     }
0545 
0546     /* Severe error occured. Driver is forced to stop operation */
0547     if (cd->card_state != GENWQE_CARD_USED) {
0548         dev_err(&pci_dev->dev,
0549             "[%s] err: DDCB#%d forced to stop (rc=%d)\n",
0550             __func__, req->num, rc);
0551         return -EIO;
0552     }
0553     return rc;
0554 }
0555 
0556 /**
0557  * get_next_ddcb() - Get next available DDCB
0558  * @cd:         pointer to genwqe device descriptor
0559  * @queue:      DDCB queue
0560  * @num:        internal DDCB number
0561  *
0562  * DDCB's content is completely cleared but presets for PRE and
0563  * SEQNUM. This function must only be called when ddcb_lock is held.
0564  *
0565  * Return: NULL if no empty DDCB available otherwise ptr to next DDCB.
0566  */
0567 static struct ddcb *get_next_ddcb(struct genwqe_dev *cd,
0568                   struct ddcb_queue *queue,
0569                   int *num)
0570 {
0571     u64 *pu64;
0572     struct ddcb *pddcb;
0573 
0574     if (queue_free_ddcbs(queue) == 0) /* queue is  full */
0575         return NULL;
0576 
0577     /* find new ddcb */
0578     pddcb = &queue->ddcb_vaddr[queue->ddcb_next];
0579 
0580     /* if it is not completed, we are not allowed to use it */
0581     /* barrier(); */
0582     if ((pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) == 0x00000000)
0583         return NULL;
0584 
0585     *num = queue->ddcb_next;    /* internal DDCB number */
0586     queue->ddcb_next = (queue->ddcb_next + 1) % queue->ddcb_max;
0587 
0588     /* clear important DDCB fields */
0589     pu64 = (u64 *)pddcb;
0590     pu64[0] = 0ULL;     /* offs 0x00 (ICRC,HSI,SHI,...) */
0591     pu64[1] = 0ULL;     /* offs 0x01 (ACFUNC,CMD...) */
0592 
0593     /* destroy previous results in ASV */
0594     pu64[0x80/8] = 0ULL;    /* offs 0x80 (ASV + 0) */
0595     pu64[0x88/8] = 0ULL;    /* offs 0x88 (ASV + 0x08) */
0596     pu64[0x90/8] = 0ULL;    /* offs 0x90 (ASV + 0x10) */
0597     pu64[0x98/8] = 0ULL;    /* offs 0x98 (ASV + 0x18) */
0598     pu64[0xd0/8] = 0ULL;    /* offs 0xd0 (RETC,ATTN...) */
0599 
0600     pddcb->pre = DDCB_PRESET_PRE; /* 128 */
0601     pddcb->seqnum_16 = cpu_to_be16(queue->ddcb_seq++);
0602     return pddcb;
0603 }
0604 
0605 /**
0606  * __genwqe_purge_ddcb() - Remove a DDCB from the workqueue
0607  * @cd:         genwqe device descriptor
0608  * @req:        DDCB request
0609  *
0610  * This will fail when the request was already FETCHED. In this case
0611  * we need to wait until it is finished. Else the DDCB can be
0612  * reused. This function also ensures that the request data structure
0613  * is removed from ddcb_req[].
0614  *
0615  * Do not forget to call this function when genwqe_wait_ddcb() fails,
0616  * such that the request gets really removed from ddcb_req[].
0617  *
0618  * Return: 0 success
0619  */
0620 int __genwqe_purge_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req)
0621 {
0622     struct ddcb *pddcb = NULL;
0623     unsigned int t;
0624     unsigned long flags;
0625     struct ddcb_queue *queue = req->queue;
0626     struct pci_dev *pci_dev = cd->pci_dev;
0627     u64 queue_status;
0628     __be32 icrc_hsi_shi = 0x0000;
0629     __be32 old, new;
0630 
0631     /* unsigned long flags; */
0632     if (GENWQE_DDCB_SOFTWARE_TIMEOUT <= 0) {
0633         dev_err(&pci_dev->dev,
0634             "[%s] err: software timeout is not set!\n", __func__);
0635         return -EFAULT;
0636     }
0637 
0638     pddcb = &queue->ddcb_vaddr[req->num];
0639 
0640     for (t = 0; t < GENWQE_DDCB_SOFTWARE_TIMEOUT * 10; t++) {
0641 
0642         spin_lock_irqsave(&queue->ddcb_lock, flags);
0643 
0644         /* Check if req was meanwhile finished */
0645         if (ddcb_requ_get_state(req) == GENWQE_REQU_FINISHED)
0646             goto go_home;
0647 
0648         /* try to set PURGE bit if FETCHED/COMPLETED are not set */
0649         old = pddcb->icrc_hsi_shi_32;   /* read SHI/HSI in BE32 */
0650         if ((old & DDCB_FETCHED_BE32) == 0x00000000) {
0651 
0652             new = (old | DDCB_PURGE_BE32);
0653             icrc_hsi_shi = cmpxchg(&pddcb->icrc_hsi_shi_32,
0654                            old, new);
0655             if (icrc_hsi_shi == old)
0656                 goto finish_ddcb;
0657         }
0658 
0659         /* normal finish with HSI bit */
0660         barrier();
0661         icrc_hsi_shi = pddcb->icrc_hsi_shi_32;
0662         if (icrc_hsi_shi & DDCB_COMPLETED_BE32)
0663             goto finish_ddcb;
0664 
0665         spin_unlock_irqrestore(&queue->ddcb_lock, flags);
0666 
0667         /*
0668          * Here the check_ddcb() function will most likely
0669          * discover this DDCB to be finished some point in
0670          * time. It will mark the req finished and free it up
0671          * in the list.
0672          */
0673 
0674         copy_ddcb_results(req, req->num); /* for the failing case */
0675         msleep(100); /* sleep for 1/10 second and try again */
0676         continue;
0677 
0678 finish_ddcb:
0679         copy_ddcb_results(req, req->num);
0680         ddcb_requ_set_state(req, GENWQE_REQU_FINISHED);
0681         queue->ddcbs_in_flight--;
0682         queue->ddcb_req[req->num] = NULL; /* delete from array */
0683         ddcb_mark_cleared(pddcb);
0684 
0685         /* Move active DDCB further; Nothing to do here anymore. */
0686 
0687         /*
0688          * We need to ensure that there is at least one free
0689          * DDCB in the queue. To do that, we must update
0690          * ddcb_act only if the COMPLETED bit is set for the
0691          * DDCB we are working on else we treat that DDCB even
0692          * if we PURGED it as occupied (hardware is supposed
0693          * to set the COMPLETED bit yet!).
0694          */
0695         icrc_hsi_shi = pddcb->icrc_hsi_shi_32;
0696         if ((icrc_hsi_shi & DDCB_COMPLETED_BE32) &&
0697             (queue->ddcb_act == req->num)) {
0698             queue->ddcb_act = ((queue->ddcb_act + 1) %
0699                        queue->ddcb_max);
0700         }
0701 go_home:
0702         spin_unlock_irqrestore(&queue->ddcb_lock, flags);
0703         return 0;
0704     }
0705 
0706     /*
0707      * If the card is dead and the queue is forced to stop, we
0708      * might see this in the queue status register.
0709      */
0710     queue_status = __genwqe_readq(cd, queue->IO_QUEUE_STATUS);
0711 
0712     dev_dbg(&pci_dev->dev, "UN/FINISHED DDCB#%d\n", req->num);
0713     genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
0714 
0715     dev_err(&pci_dev->dev,
0716         "[%s] err: DDCB#%d not purged and not completed after %d seconds QSTAT=%016llx!!\n",
0717         __func__, req->num, GENWQE_DDCB_SOFTWARE_TIMEOUT,
0718         queue_status);
0719 
0720     print_ddcb_info(cd, req->queue);
0721 
0722     return -EFAULT;
0723 }
0724 
0725 int genwqe_init_debug_data(struct genwqe_dev *cd, struct genwqe_debug_data *d)
0726 {
0727     int len;
0728     struct pci_dev *pci_dev = cd->pci_dev;
0729 
0730     if (d == NULL) {
0731         dev_err(&pci_dev->dev,
0732             "[%s] err: invalid memory for debug data!\n",
0733             __func__);
0734         return -EFAULT;
0735     }
0736 
0737     len  = sizeof(d->driver_version);
0738     snprintf(d->driver_version, len, "%s", DRV_VERSION);
0739     d->slu_unitcfg = cd->slu_unitcfg;
0740     d->app_unitcfg = cd->app_unitcfg;
0741     return 0;
0742 }
0743 
0744 /**
0745  * __genwqe_enqueue_ddcb() - Enqueue a DDCB
0746  * @cd:         pointer to genwqe device descriptor
0747  * @req:        pointer to DDCB execution request
0748  * @f_flags:    file mode: blocking, non-blocking
0749  *
0750  * Return: 0 if enqueuing succeeded
0751  *         -EIO if card is unusable/PCIe problems
0752  *         -EBUSY if enqueuing failed
0753  */
0754 int __genwqe_enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req,
0755               unsigned int f_flags)
0756 {
0757     struct ddcb *pddcb;
0758     unsigned long flags;
0759     struct ddcb_queue *queue;
0760     struct pci_dev *pci_dev = cd->pci_dev;
0761     u16 icrc;
0762 
0763  retry:
0764     if (cd->card_state != GENWQE_CARD_USED) {
0765         printk_ratelimited(KERN_ERR
0766             "%s %s: [%s] Card is unusable/PCIe problem Req#%d\n",
0767             GENWQE_DEVNAME, dev_name(&pci_dev->dev),
0768             __func__, req->num);
0769         return -EIO;
0770     }
0771 
0772     queue = req->queue = &cd->queue;
0773 
0774     /* FIXME circumvention to improve performance when no irq is
0775      * there.
0776      */
0777     if (GENWQE_POLLING_ENABLED)
0778         genwqe_check_ddcb_queue(cd, queue);
0779 
0780     /*
0781      * It must be ensured to process all DDCBs in successive
0782      * order. Use a lock here in order to prevent nested DDCB
0783      * enqueuing.
0784      */
0785     spin_lock_irqsave(&queue->ddcb_lock, flags);
0786 
0787     pddcb = get_next_ddcb(cd, queue, &req->num);    /* get ptr and num */
0788     if (pddcb == NULL) {
0789         int rc;
0790 
0791         spin_unlock_irqrestore(&queue->ddcb_lock, flags);
0792 
0793         if (f_flags & O_NONBLOCK) {
0794             queue->return_on_busy++;
0795             return -EBUSY;
0796         }
0797 
0798         queue->wait_on_busy++;
0799         rc = wait_event_interruptible(queue->busy_waitq,
0800                           queue_free_ddcbs(queue) != 0);
0801         dev_dbg(&pci_dev->dev, "[%s] waiting for free DDCB: rc=%d\n",
0802             __func__, rc);
0803         if (rc == -ERESTARTSYS)
0804             return rc;  /* interrupted by a signal */
0805 
0806         goto retry;
0807     }
0808 
0809     if (queue->ddcb_req[req->num] != NULL) {
0810         spin_unlock_irqrestore(&queue->ddcb_lock, flags);
0811 
0812         dev_err(&pci_dev->dev,
0813             "[%s] picked DDCB %d with req=%p still in use!!\n",
0814             __func__, req->num, req);
0815         return -EFAULT;
0816     }
0817     ddcb_requ_set_state(req, GENWQE_REQU_ENQUEUED);
0818     queue->ddcb_req[req->num] = req;
0819 
0820     pddcb->cmdopts_16 = cpu_to_be16(req->cmd.cmdopts);
0821     pddcb->cmd = req->cmd.cmd;
0822     pddcb->acfunc = req->cmd.acfunc;    /* functional unit */
0823 
0824     /*
0825      * We know that we can get retc 0x104 with CRC error, do not
0826      * stop the queue in those cases for this command. XDIR = 1
0827      * does not work for old SLU versions.
0828      *
0829      * Last bitstream with the old XDIR behavior had SLU_ID
0830      * 0x34199.
0831      */
0832     if ((cd->slu_unitcfg & 0xFFFF0ull) > 0x34199ull)
0833         pddcb->xdir = 0x1;
0834     else
0835         pddcb->xdir = 0x0;
0836 
0837 
0838     pddcb->psp = (((req->cmd.asiv_length / 8) << 4) |
0839               ((req->cmd.asv_length  / 8)));
0840     pddcb->disp_ts_64 = cpu_to_be64(req->cmd.disp_ts);
0841 
0842     /*
0843      * If copying the whole DDCB_ASIV_LENGTH is impacting
0844      * performance we need to change it to
0845      * req->cmd.asiv_length. But simulation benefits from some
0846      * non-architectured bits behind the architectured content.
0847      *
0848      * How much data is copied depends on the availability of the
0849      * ATS field, which was introduced late. If the ATS field is
0850      * supported ASIV is 8 bytes shorter than it used to be. Since
0851      * the ATS field is copied too, the code should do exactly
0852      * what it did before, but I wanted to make copying of the ATS
0853      * field very explicit.
0854      */
0855     if (genwqe_get_slu_id(cd) <= 0x2) {
0856         memcpy(&pddcb->__asiv[0],   /* destination */
0857                &req->cmd.__asiv[0], /* source */
0858                DDCB_ASIV_LENGTH);   /* req->cmd.asiv_length */
0859     } else {
0860         pddcb->n.ats_64 = cpu_to_be64(req->cmd.ats);
0861         memcpy(&pddcb->n.asiv[0],   /* destination */
0862             &req->cmd.asiv[0],  /* source */
0863             DDCB_ASIV_LENGTH_ATS);  /* req->cmd.asiv_length */
0864     }
0865 
0866     pddcb->icrc_hsi_shi_32 = cpu_to_be32(0x00000000); /* for crc */
0867 
0868     /*
0869      * Calculate CRC_16 for corresponding range PSP(7:4). Include
0870      * empty 4 bytes prior to the data.
0871      */
0872     icrc = genwqe_crc16((const u8 *)pddcb,
0873                ICRC_LENGTH(req->cmd.asiv_length), 0xffff);
0874     pddcb->icrc_hsi_shi_32 = cpu_to_be32((u32)icrc << 16);
0875 
0876     /* enable DDCB completion irq */
0877     if (!GENWQE_POLLING_ENABLED)
0878         pddcb->icrc_hsi_shi_32 |= DDCB_INTR_BE32;
0879 
0880     dev_dbg(&pci_dev->dev, "INPUT DDCB#%d\n", req->num);
0881     genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
0882 
0883     if (ddcb_requ_collect_debug_data(req)) {
0884         /* use the kernel copy of debug data. copying back to
0885            user buffer happens later */
0886 
0887         genwqe_init_debug_data(cd, &req->debug_data);
0888         memcpy(&req->debug_data.ddcb_before, pddcb,
0889                sizeof(req->debug_data.ddcb_before));
0890     }
0891 
0892     enqueue_ddcb(cd, queue, pddcb, req->num);
0893     queue->ddcbs_in_flight++;
0894 
0895     if (queue->ddcbs_in_flight > queue->ddcbs_max_in_flight)
0896         queue->ddcbs_max_in_flight = queue->ddcbs_in_flight;
0897 
0898     ddcb_requ_set_state(req, GENWQE_REQU_TAPPED);
0899     spin_unlock_irqrestore(&queue->ddcb_lock, flags);
0900     wake_up_interruptible(&cd->queue_waitq);
0901 
0902     return 0;
0903 }
0904 
0905 /**
0906  * __genwqe_execute_raw_ddcb() - Setup and execute DDCB
0907  * @cd:         pointer to genwqe device descriptor
0908  * @cmd:        user provided DDCB command
0909  * @f_flags:    file mode: blocking, non-blocking
0910  */
0911 int __genwqe_execute_raw_ddcb(struct genwqe_dev *cd,
0912                   struct genwqe_ddcb_cmd *cmd,
0913                   unsigned int f_flags)
0914 {
0915     int rc = 0;
0916     struct pci_dev *pci_dev = cd->pci_dev;
0917     struct ddcb_requ *req = container_of(cmd, struct ddcb_requ, cmd);
0918 
0919     if (cmd->asiv_length > DDCB_ASIV_LENGTH) {
0920         dev_err(&pci_dev->dev, "[%s] err: wrong asiv_length of %d\n",
0921             __func__, cmd->asiv_length);
0922         return -EINVAL;
0923     }
0924     if (cmd->asv_length > DDCB_ASV_LENGTH) {
0925         dev_err(&pci_dev->dev, "[%s] err: wrong asv_length of %d\n",
0926             __func__, cmd->asiv_length);
0927         return -EINVAL;
0928     }
0929     rc = __genwqe_enqueue_ddcb(cd, req, f_flags);
0930     if (rc != 0)
0931         return rc;
0932 
0933     rc = __genwqe_wait_ddcb(cd, req);
0934     if (rc < 0)     /* error or signal interrupt */
0935         goto err_exit;
0936 
0937     if (ddcb_requ_collect_debug_data(req)) {
0938         if (copy_to_user((struct genwqe_debug_data __user *)
0939                  (unsigned long)cmd->ddata_addr,
0940                  &req->debug_data,
0941                  sizeof(struct genwqe_debug_data)))
0942             return -EFAULT;
0943     }
0944 
0945     /*
0946      * Higher values than 0x102 indicate completion with faults,
0947      * lower values than 0x102 indicate processing faults. Note
0948      * that DDCB might have been purged. E.g. Cntl+C.
0949      */
0950     if (cmd->retc != DDCB_RETC_COMPLETE) {
0951         /* This might happen e.g. flash read, and needs to be
0952            handled by the upper layer code. */
0953         rc = -EBADMSG;  /* not processed/error retc */
0954     }
0955 
0956     return rc;
0957 
0958  err_exit:
0959     __genwqe_purge_ddcb(cd, req);
0960 
0961     if (ddcb_requ_collect_debug_data(req)) {
0962         if (copy_to_user((struct genwqe_debug_data __user *)
0963                  (unsigned long)cmd->ddata_addr,
0964                  &req->debug_data,
0965                  sizeof(struct genwqe_debug_data)))
0966             return -EFAULT;
0967     }
0968     return rc;
0969 }
0970 
0971 /**
0972  * genwqe_next_ddcb_ready() - Figure out if the next DDCB is already finished
0973  * @cd:         pointer to genwqe device descriptor
0974  *
0975  * We use this as condition for our wait-queue code.
0976  */
0977 static int genwqe_next_ddcb_ready(struct genwqe_dev *cd)
0978 {
0979     unsigned long flags;
0980     struct ddcb *pddcb;
0981     struct ddcb_queue *queue = &cd->queue;
0982 
0983     spin_lock_irqsave(&queue->ddcb_lock, flags);
0984 
0985     if (queue_empty(queue)) { /* empty queue */
0986         spin_unlock_irqrestore(&queue->ddcb_lock, flags);
0987         return 0;
0988     }
0989 
0990     pddcb = &queue->ddcb_vaddr[queue->ddcb_act];
0991     if (pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) { /* ddcb ready */
0992         spin_unlock_irqrestore(&queue->ddcb_lock, flags);
0993         return 1;
0994     }
0995 
0996     spin_unlock_irqrestore(&queue->ddcb_lock, flags);
0997     return 0;
0998 }
0999 
1000 /**
1001  * genwqe_ddcbs_in_flight() - Check how many DDCBs are in flight
1002  * @cd:         pointer to genwqe device descriptor
1003  *
1004  * Keep track on the number of DDCBs which ware currently in the
1005  * queue. This is needed for statistics as well as condition if we want
1006  * to wait or better do polling in case of no interrupts available.
1007  */
1008 int genwqe_ddcbs_in_flight(struct genwqe_dev *cd)
1009 {
1010     unsigned long flags;
1011     int ddcbs_in_flight = 0;
1012     struct ddcb_queue *queue = &cd->queue;
1013 
1014     spin_lock_irqsave(&queue->ddcb_lock, flags);
1015     ddcbs_in_flight += queue->ddcbs_in_flight;
1016     spin_unlock_irqrestore(&queue->ddcb_lock, flags);
1017 
1018     return ddcbs_in_flight;
1019 }
1020 
1021 static int setup_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue)
1022 {
1023     int rc, i;
1024     struct ddcb *pddcb;
1025     u64 val64;
1026     unsigned int queue_size;
1027     struct pci_dev *pci_dev = cd->pci_dev;
1028 
1029     if (GENWQE_DDCB_MAX < 2)
1030         return -EINVAL;
1031 
1032     queue_size = roundup(GENWQE_DDCB_MAX * sizeof(struct ddcb), PAGE_SIZE);
1033 
1034     queue->ddcbs_in_flight = 0;  /* statistics */
1035     queue->ddcbs_max_in_flight = 0;
1036     queue->ddcbs_completed = 0;
1037     queue->return_on_busy = 0;
1038     queue->wait_on_busy = 0;
1039 
1040     queue->ddcb_seq   = 0x100; /* start sequence number */
1041     queue->ddcb_max   = GENWQE_DDCB_MAX;
1042     queue->ddcb_vaddr = __genwqe_alloc_consistent(cd, queue_size,
1043                         &queue->ddcb_daddr);
1044     if (queue->ddcb_vaddr == NULL) {
1045         dev_err(&pci_dev->dev,
1046             "[%s] **err: could not allocate DDCB **\n", __func__);
1047         return -ENOMEM;
1048     }
1049     queue->ddcb_req = kcalloc(queue->ddcb_max, sizeof(struct ddcb_requ *),
1050                   GFP_KERNEL);
1051     if (!queue->ddcb_req) {
1052         rc = -ENOMEM;
1053         goto free_ddcbs;
1054     }
1055 
1056     queue->ddcb_waitqs = kcalloc(queue->ddcb_max,
1057                      sizeof(wait_queue_head_t),
1058                      GFP_KERNEL);
1059     if (!queue->ddcb_waitqs) {
1060         rc = -ENOMEM;
1061         goto free_requs;
1062     }
1063 
1064     for (i = 0; i < queue->ddcb_max; i++) {
1065         pddcb = &queue->ddcb_vaddr[i];           /* DDCBs */
1066         pddcb->icrc_hsi_shi_32 = DDCB_COMPLETED_BE32;
1067         pddcb->retc_16 = cpu_to_be16(0xfff);
1068 
1069         queue->ddcb_req[i] = NULL;           /* requests */
1070         init_waitqueue_head(&queue->ddcb_waitqs[i]); /* waitqueues */
1071     }
1072 
1073     queue->ddcb_act  = 0;
1074     queue->ddcb_next = 0;   /* queue is empty */
1075 
1076     spin_lock_init(&queue->ddcb_lock);
1077     init_waitqueue_head(&queue->busy_waitq);
1078 
1079     val64 = ((u64)(queue->ddcb_max - 1) <<  8); /* lastptr */
1080     __genwqe_writeq(cd, queue->IO_QUEUE_CONFIG,  0x07);  /* iCRC/vCRC */
1081     __genwqe_writeq(cd, queue->IO_QUEUE_SEGMENT, queue->ddcb_daddr);
1082     __genwqe_writeq(cd, queue->IO_QUEUE_INITSQN, queue->ddcb_seq);
1083     __genwqe_writeq(cd, queue->IO_QUEUE_WRAP,    val64);
1084     return 0;
1085 
1086  free_requs:
1087     kfree(queue->ddcb_req);
1088     queue->ddcb_req = NULL;
1089  free_ddcbs:
1090     __genwqe_free_consistent(cd, queue_size, queue->ddcb_vaddr,
1091                 queue->ddcb_daddr);
1092     queue->ddcb_vaddr = NULL;
1093     queue->ddcb_daddr = 0ull;
1094     return rc;
1095 
1096 }
1097 
1098 static int ddcb_queue_initialized(struct ddcb_queue *queue)
1099 {
1100     return queue->ddcb_vaddr != NULL;
1101 }
1102 
1103 static void free_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue)
1104 {
1105     unsigned int queue_size;
1106 
1107     queue_size = roundup(queue->ddcb_max * sizeof(struct ddcb), PAGE_SIZE);
1108 
1109     kfree(queue->ddcb_req);
1110     queue->ddcb_req = NULL;
1111 
1112     if (queue->ddcb_vaddr) {
1113         __genwqe_free_consistent(cd, queue_size, queue->ddcb_vaddr,
1114                     queue->ddcb_daddr);
1115         queue->ddcb_vaddr = NULL;
1116         queue->ddcb_daddr = 0ull;
1117     }
1118 }
1119 
1120 static irqreturn_t genwqe_pf_isr(int irq, void *dev_id)
1121 {
1122     u64 gfir;
1123     struct genwqe_dev *cd = (struct genwqe_dev *)dev_id;
1124     struct pci_dev *pci_dev = cd->pci_dev;
1125 
1126     /*
1127      * In case of fatal FIR error the queue is stopped, such that
1128      * we can safely check it without risking anything.
1129      */
1130     cd->irqs_processed++;
1131     wake_up_interruptible(&cd->queue_waitq);
1132 
1133     /*
1134      * Checking for errors before kicking the queue might be
1135      * safer, but slower for the good-case ... See above.
1136      */
1137     gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR);
1138     if (((gfir & GFIR_ERR_TRIGGER) != 0x0) &&
1139         !pci_channel_offline(pci_dev)) {
1140 
1141         if (cd->use_platform_recovery) {
1142             /*
1143              * Since we use raw accessors, EEH errors won't be
1144              * detected by the platform until we do a non-raw
1145              * MMIO or config space read
1146              */
1147             readq(cd->mmio + IO_SLC_CFGREG_GFIR);
1148 
1149             /* Don't do anything if the PCI channel is frozen */
1150             if (pci_channel_offline(pci_dev))
1151                 goto exit;
1152         }
1153 
1154         wake_up_interruptible(&cd->health_waitq);
1155 
1156         /*
1157          * By default GFIRs causes recovery actions. This
1158          * count is just for debug when recovery is masked.
1159          */
1160         dev_err_ratelimited(&pci_dev->dev,
1161                     "[%s] GFIR=%016llx\n",
1162                     __func__, gfir);
1163     }
1164 
1165  exit:
1166     return IRQ_HANDLED;
1167 }
1168 
1169 static irqreturn_t genwqe_vf_isr(int irq, void *dev_id)
1170 {
1171     struct genwqe_dev *cd = (struct genwqe_dev *)dev_id;
1172 
1173     cd->irqs_processed++;
1174     wake_up_interruptible(&cd->queue_waitq);
1175 
1176     return IRQ_HANDLED;
1177 }
1178 
1179 /**
1180  * genwqe_card_thread() - Work thread for the DDCB queue
1181  * @data:         pointer to genwqe device descriptor
1182  *
1183  * The idea is to check if there are DDCBs in processing. If there are
1184  * some finished DDCBs, we process them and wakeup the
1185  * requestors. Otherwise we give other processes time using
1186  * cond_resched().
1187  */
1188 static int genwqe_card_thread(void *data)
1189 {
1190     int should_stop = 0;
1191     struct genwqe_dev *cd = (struct genwqe_dev *)data;
1192 
1193     while (!kthread_should_stop()) {
1194 
1195         genwqe_check_ddcb_queue(cd, &cd->queue);
1196 
1197         if (GENWQE_POLLING_ENABLED) {
1198             wait_event_interruptible_timeout(
1199                 cd->queue_waitq,
1200                 genwqe_ddcbs_in_flight(cd) ||
1201                 (should_stop = kthread_should_stop()), 1);
1202         } else {
1203             wait_event_interruptible_timeout(
1204                 cd->queue_waitq,
1205                 genwqe_next_ddcb_ready(cd) ||
1206                 (should_stop = kthread_should_stop()), HZ);
1207         }
1208         if (should_stop)
1209             break;
1210 
1211         /*
1212          * Avoid soft lockups on heavy loads; we do not want
1213          * to disable our interrupts.
1214          */
1215         cond_resched();
1216     }
1217     return 0;
1218 }
1219 
1220 /**
1221  * genwqe_setup_service_layer() - Setup DDCB queue
1222  * @cd:         pointer to genwqe device descriptor
1223  *
1224  * Allocate DDCBs. Configure Service Layer Controller (SLC).
1225  *
1226  * Return: 0 success
1227  */
1228 int genwqe_setup_service_layer(struct genwqe_dev *cd)
1229 {
1230     int rc;
1231     struct ddcb_queue *queue;
1232     struct pci_dev *pci_dev = cd->pci_dev;
1233 
1234     if (genwqe_is_privileged(cd)) {
1235         rc = genwqe_card_reset(cd);
1236         if (rc < 0) {
1237             dev_err(&pci_dev->dev,
1238                 "[%s] err: reset failed.\n", __func__);
1239             return rc;
1240         }
1241         genwqe_read_softreset(cd);
1242     }
1243 
1244     queue = &cd->queue;
1245     queue->IO_QUEUE_CONFIG  = IO_SLC_QUEUE_CONFIG;
1246     queue->IO_QUEUE_STATUS  = IO_SLC_QUEUE_STATUS;
1247     queue->IO_QUEUE_SEGMENT = IO_SLC_QUEUE_SEGMENT;
1248     queue->IO_QUEUE_INITSQN = IO_SLC_QUEUE_INITSQN;
1249     queue->IO_QUEUE_OFFSET  = IO_SLC_QUEUE_OFFSET;
1250     queue->IO_QUEUE_WRAP    = IO_SLC_QUEUE_WRAP;
1251     queue->IO_QUEUE_WTIME   = IO_SLC_QUEUE_WTIME;
1252     queue->IO_QUEUE_ERRCNTS = IO_SLC_QUEUE_ERRCNTS;
1253     queue->IO_QUEUE_LRW     = IO_SLC_QUEUE_LRW;
1254 
1255     rc = setup_ddcb_queue(cd, queue);
1256     if (rc != 0) {
1257         rc = -ENODEV;
1258         goto err_out;
1259     }
1260 
1261     init_waitqueue_head(&cd->queue_waitq);
1262     cd->card_thread = kthread_run(genwqe_card_thread, cd,
1263                       GENWQE_DEVNAME "%d_thread",
1264                       cd->card_idx);
1265     if (IS_ERR(cd->card_thread)) {
1266         rc = PTR_ERR(cd->card_thread);
1267         cd->card_thread = NULL;
1268         goto stop_free_queue;
1269     }
1270 
1271     rc = genwqe_set_interrupt_capability(cd, GENWQE_MSI_IRQS);
1272     if (rc)
1273         goto stop_kthread;
1274 
1275     /*
1276      * We must have all wait-queues initialized when we enable the
1277      * interrupts. Otherwise we might crash if we get an early
1278      * irq.
1279      */
1280     init_waitqueue_head(&cd->health_waitq);
1281 
1282     if (genwqe_is_privileged(cd)) {
1283         rc = request_irq(pci_dev->irq, genwqe_pf_isr, IRQF_SHARED,
1284                  GENWQE_DEVNAME, cd);
1285     } else {
1286         rc = request_irq(pci_dev->irq, genwqe_vf_isr, IRQF_SHARED,
1287                  GENWQE_DEVNAME, cd);
1288     }
1289     if (rc < 0) {
1290         dev_err(&pci_dev->dev, "irq %d not free.\n", pci_dev->irq);
1291         goto stop_irq_cap;
1292     }
1293 
1294     cd->card_state = GENWQE_CARD_USED;
1295     return 0;
1296 
1297  stop_irq_cap:
1298     genwqe_reset_interrupt_capability(cd);
1299  stop_kthread:
1300     kthread_stop(cd->card_thread);
1301     cd->card_thread = NULL;
1302  stop_free_queue:
1303     free_ddcb_queue(cd, queue);
1304  err_out:
1305     return rc;
1306 }
1307 
1308 /**
1309  * queue_wake_up_all() - Handles fatal error case
1310  * @cd:         pointer to genwqe device descriptor
1311  *
1312  * The PCI device got unusable and we have to stop all pending
1313  * requests as fast as we can. The code after this must purge the
1314  * DDCBs in question and ensure that all mappings are freed.
1315  */
1316 static int queue_wake_up_all(struct genwqe_dev *cd)
1317 {
1318     unsigned int i;
1319     unsigned long flags;
1320     struct ddcb_queue *queue = &cd->queue;
1321 
1322     spin_lock_irqsave(&queue->ddcb_lock, flags);
1323 
1324     for (i = 0; i < queue->ddcb_max; i++)
1325         wake_up_interruptible(&queue->ddcb_waitqs[queue->ddcb_act]);
1326 
1327     wake_up_interruptible(&queue->busy_waitq);
1328     spin_unlock_irqrestore(&queue->ddcb_lock, flags);
1329 
1330     return 0;
1331 }
1332 
1333 /**
1334  * genwqe_finish_queue() - Remove any genwqe devices and user-interfaces
1335  * @cd:         pointer to genwqe device descriptor
1336  *
1337  * Relies on the pre-condition that there are no users of the card
1338  * device anymore e.g. with open file-descriptors.
1339  *
1340  * This function must be robust enough to be called twice.
1341  */
1342 int genwqe_finish_queue(struct genwqe_dev *cd)
1343 {
1344     int i, rc = 0, in_flight;
1345     int waitmax = GENWQE_DDCB_SOFTWARE_TIMEOUT;
1346     struct pci_dev *pci_dev = cd->pci_dev;
1347     struct ddcb_queue *queue = &cd->queue;
1348 
1349     if (!ddcb_queue_initialized(queue))
1350         return 0;
1351 
1352     /* Do not wipe out the error state. */
1353     if (cd->card_state == GENWQE_CARD_USED)
1354         cd->card_state = GENWQE_CARD_UNUSED;
1355 
1356     /* Wake up all requests in the DDCB queue such that they
1357        should be removed nicely. */
1358     queue_wake_up_all(cd);
1359 
1360     /* We must wait to get rid of the DDCBs in flight */
1361     for (i = 0; i < waitmax; i++) {
1362         in_flight = genwqe_ddcbs_in_flight(cd);
1363 
1364         if (in_flight == 0)
1365             break;
1366 
1367         dev_dbg(&pci_dev->dev,
1368             "  DEBUG [%d/%d] waiting for queue to get empty: %d requests!\n",
1369             i, waitmax, in_flight);
1370 
1371         /*
1372          * Severe severe error situation: The card itself has
1373          * 16 DDCB queues, each queue has e.g. 32 entries,
1374          * each DDBC has a hardware timeout of currently 250
1375          * msec but the PFs have a hardware timeout of 8 sec
1376          * ... so I take something large.
1377          */
1378         msleep(1000);
1379     }
1380     if (i == waitmax) {
1381         dev_err(&pci_dev->dev, "  [%s] err: queue is not empty!!\n",
1382             __func__);
1383         rc = -EIO;
1384     }
1385     return rc;
1386 }
1387 
1388 /**
1389  * genwqe_release_service_layer() - Shutdown DDCB queue
1390  * @cd:       genwqe device descriptor
1391  *
1392  * This function must be robust enough to be called twice.
1393  */
1394 int genwqe_release_service_layer(struct genwqe_dev *cd)
1395 {
1396     struct pci_dev *pci_dev = cd->pci_dev;
1397 
1398     if (!ddcb_queue_initialized(&cd->queue))
1399         return 1;
1400 
1401     free_irq(pci_dev->irq, cd);
1402     genwqe_reset_interrupt_capability(cd);
1403 
1404     if (cd->card_thread != NULL) {
1405         kthread_stop(cd->card_thread);
1406         cd->card_thread = NULL;
1407     }
1408 
1409     free_ddcb_queue(cd, &cd->queue);
1410     return 0;
1411 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!