Back to home page

OSCL-LXR
 
 

     

0001 /*
0002  * Copyright (c) 2012 - 2018 Intel Corporation.  All rights reserved.
0003  * Copyright (c) 2006 - 2012 QLogic Corporation. All rights reserved.
0004  * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved.
0005  *
0006  * This software is available to you under a choice of one of two
0007  * licenses.  You may choose to be licensed under the terms of the GNU
0008  * General Public License (GPL) Version 2, available from the file
0009  * COPYING in the main directory of this source tree, or the
0010  * OpenIB.org BSD license below:
0011  *
0012  *     Redistribution and use in source and binary forms, with or
0013  *     without modification, are permitted provided that the following
0014  *     conditions are met:
0015  *
0016  *      - Redistributions of source code must retain the above
0017  *        copyright notice, this list of conditions and the following
0018  *        disclaimer.
0019  *
0020  *      - Redistributions in binary form must reproduce the above
0021  *        copyright notice, this list of conditions and the following
0022  *        disclaimer in the documentation and/or other materials
0023  *        provided with the distribution.
0024  *
0025  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
0026  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
0027  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
0028  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
0029  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
0030  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
0031  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
0032  * SOFTWARE.
0033  */
0034 
0035 #include <rdma/ib_mad.h>
0036 #include <rdma/ib_user_verbs.h>
0037 #include <linux/io.h>
0038 #include <linux/module.h>
0039 #include <linux/utsname.h>
0040 #include <linux/rculist.h>
0041 #include <linux/mm.h>
0042 #include <linux/vmalloc.h>
0043 #include <rdma/rdma_vt.h>
0044 
0045 #include "qib.h"
0046 #include "qib_common.h"
0047 
0048 static unsigned int ib_qib_qp_table_size = 256;
0049 module_param_named(qp_table_size, ib_qib_qp_table_size, uint, S_IRUGO);
0050 MODULE_PARM_DESC(qp_table_size, "QP table size");
0051 
0052 static unsigned int qib_lkey_table_size = 16;
0053 module_param_named(lkey_table_size, qib_lkey_table_size, uint,
0054            S_IRUGO);
0055 MODULE_PARM_DESC(lkey_table_size,
0056          "LKEY table size in bits (2^n, 1 <= n <= 23)");
0057 
0058 static unsigned int ib_qib_max_pds = 0xFFFF;
0059 module_param_named(max_pds, ib_qib_max_pds, uint, S_IRUGO);
0060 MODULE_PARM_DESC(max_pds,
0061          "Maximum number of protection domains to support");
0062 
0063 static unsigned int ib_qib_max_ahs = 0xFFFF;
0064 module_param_named(max_ahs, ib_qib_max_ahs, uint, S_IRUGO);
0065 MODULE_PARM_DESC(max_ahs, "Maximum number of address handles to support");
0066 
0067 unsigned int ib_qib_max_cqes = 0x2FFFF;
0068 module_param_named(max_cqes, ib_qib_max_cqes, uint, S_IRUGO);
0069 MODULE_PARM_DESC(max_cqes,
0070          "Maximum number of completion queue entries to support");
0071 
0072 unsigned int ib_qib_max_cqs = 0x1FFFF;
0073 module_param_named(max_cqs, ib_qib_max_cqs, uint, S_IRUGO);
0074 MODULE_PARM_DESC(max_cqs, "Maximum number of completion queues to support");
0075 
0076 unsigned int ib_qib_max_qp_wrs = 0x3FFF;
0077 module_param_named(max_qp_wrs, ib_qib_max_qp_wrs, uint, S_IRUGO);
0078 MODULE_PARM_DESC(max_qp_wrs, "Maximum number of QP WRs to support");
0079 
0080 unsigned int ib_qib_max_qps = 16384;
0081 module_param_named(max_qps, ib_qib_max_qps, uint, S_IRUGO);
0082 MODULE_PARM_DESC(max_qps, "Maximum number of QPs to support");
0083 
0084 unsigned int ib_qib_max_sges = 0x60;
0085 module_param_named(max_sges, ib_qib_max_sges, uint, S_IRUGO);
0086 MODULE_PARM_DESC(max_sges, "Maximum number of SGEs to support");
0087 
0088 unsigned int ib_qib_max_mcast_grps = 16384;
0089 module_param_named(max_mcast_grps, ib_qib_max_mcast_grps, uint, S_IRUGO);
0090 MODULE_PARM_DESC(max_mcast_grps,
0091          "Maximum number of multicast groups to support");
0092 
0093 unsigned int ib_qib_max_mcast_qp_attached = 16;
0094 module_param_named(max_mcast_qp_attached, ib_qib_max_mcast_qp_attached,
0095            uint, S_IRUGO);
0096 MODULE_PARM_DESC(max_mcast_qp_attached,
0097          "Maximum number of attached QPs to support");
0098 
0099 unsigned int ib_qib_max_srqs = 1024;
0100 module_param_named(max_srqs, ib_qib_max_srqs, uint, S_IRUGO);
0101 MODULE_PARM_DESC(max_srqs, "Maximum number of SRQs to support");
0102 
0103 unsigned int ib_qib_max_srq_sges = 128;
0104 module_param_named(max_srq_sges, ib_qib_max_srq_sges, uint, S_IRUGO);
0105 MODULE_PARM_DESC(max_srq_sges, "Maximum number of SRQ SGEs to support");
0106 
0107 unsigned int ib_qib_max_srq_wrs = 0x1FFFF;
0108 module_param_named(max_srq_wrs, ib_qib_max_srq_wrs, uint, S_IRUGO);
0109 MODULE_PARM_DESC(max_srq_wrs, "Maximum number of SRQ WRs support");
0110 
0111 static unsigned int ib_qib_disable_sma;
0112 module_param_named(disable_sma, ib_qib_disable_sma, uint, S_IWUSR | S_IRUGO);
0113 MODULE_PARM_DESC(disable_sma, "Disable the SMA");
0114 
0115 /*
0116  * Translate ib_wr_opcode into ib_wc_opcode.
0117  */
0118 const enum ib_wc_opcode ib_qib_wc_opcode[] = {
0119     [IB_WR_RDMA_WRITE] = IB_WC_RDMA_WRITE,
0120     [IB_WR_RDMA_WRITE_WITH_IMM] = IB_WC_RDMA_WRITE,
0121     [IB_WR_SEND] = IB_WC_SEND,
0122     [IB_WR_SEND_WITH_IMM] = IB_WC_SEND,
0123     [IB_WR_RDMA_READ] = IB_WC_RDMA_READ,
0124     [IB_WR_ATOMIC_CMP_AND_SWP] = IB_WC_COMP_SWAP,
0125     [IB_WR_ATOMIC_FETCH_AND_ADD] = IB_WC_FETCH_ADD
0126 };
0127 
0128 /*
0129  * System image GUID.
0130  */
0131 __be64 ib_qib_sys_image_guid;
0132 
0133 /*
0134  * Count the number of DMA descriptors needed to send length bytes of data.
0135  * Don't modify the qib_sge_state to get the count.
0136  * Return zero if any of the segments is not aligned.
0137  */
0138 static u32 qib_count_sge(struct rvt_sge_state *ss, u32 length)
0139 {
0140     struct rvt_sge *sg_list = ss->sg_list;
0141     struct rvt_sge sge = ss->sge;
0142     u8 num_sge = ss->num_sge;
0143     u32 ndesc = 1;  /* count the header */
0144 
0145     while (length) {
0146         u32 len = rvt_get_sge_length(&sge, length);
0147 
0148         if (((long) sge.vaddr & (sizeof(u32) - 1)) ||
0149             (len != length && (len & (sizeof(u32) - 1)))) {
0150             ndesc = 0;
0151             break;
0152         }
0153         ndesc++;
0154         sge.vaddr += len;
0155         sge.length -= len;
0156         sge.sge_length -= len;
0157         if (sge.sge_length == 0) {
0158             if (--num_sge)
0159                 sge = *sg_list++;
0160         } else if (sge.length == 0 && sge.mr->lkey) {
0161             if (++sge.n >= RVT_SEGSZ) {
0162                 if (++sge.m >= sge.mr->mapsz)
0163                     break;
0164                 sge.n = 0;
0165             }
0166             sge.vaddr =
0167                 sge.mr->map[sge.m]->segs[sge.n].vaddr;
0168             sge.length =
0169                 sge.mr->map[sge.m]->segs[sge.n].length;
0170         }
0171         length -= len;
0172     }
0173     return ndesc;
0174 }
0175 
0176 /*
0177  * Copy from the SGEs to the data buffer.
0178  */
0179 static void qib_copy_from_sge(void *data, struct rvt_sge_state *ss, u32 length)
0180 {
0181     struct rvt_sge *sge = &ss->sge;
0182 
0183     while (length) {
0184         u32 len = rvt_get_sge_length(sge, length);
0185 
0186         memcpy(data, sge->vaddr, len);
0187         sge->vaddr += len;
0188         sge->length -= len;
0189         sge->sge_length -= len;
0190         if (sge->sge_length == 0) {
0191             if (--ss->num_sge)
0192                 *sge = *ss->sg_list++;
0193         } else if (sge->length == 0 && sge->mr->lkey) {
0194             if (++sge->n >= RVT_SEGSZ) {
0195                 if (++sge->m >= sge->mr->mapsz)
0196                     break;
0197                 sge->n = 0;
0198             }
0199             sge->vaddr =
0200                 sge->mr->map[sge->m]->segs[sge->n].vaddr;
0201             sge->length =
0202                 sge->mr->map[sge->m]->segs[sge->n].length;
0203         }
0204         data += len;
0205         length -= len;
0206     }
0207 }
0208 
0209 /**
0210  * qib_qp_rcv - processing an incoming packet on a QP
0211  * @rcd: the context pointer
0212  * @hdr: the packet header
0213  * @has_grh: true if the packet has a GRH
0214  * @data: the packet data
0215  * @tlen: the packet length
0216  * @qp: the QP the packet came on
0217  *
0218  * This is called from qib_ib_rcv() to process an incoming packet
0219  * for the given QP.
0220  * Called at interrupt level.
0221  */
0222 static void qib_qp_rcv(struct qib_ctxtdata *rcd, struct ib_header *hdr,
0223                int has_grh, void *data, u32 tlen, struct rvt_qp *qp)
0224 {
0225     struct qib_ibport *ibp = &rcd->ppd->ibport_data;
0226 
0227     spin_lock(&qp->r_lock);
0228 
0229     /* Check for valid receive state. */
0230     if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) {
0231         ibp->rvp.n_pkt_drops++;
0232         goto unlock;
0233     }
0234 
0235     switch (qp->ibqp.qp_type) {
0236     case IB_QPT_SMI:
0237     case IB_QPT_GSI:
0238         if (ib_qib_disable_sma)
0239             break;
0240         fallthrough;
0241     case IB_QPT_UD:
0242         qib_ud_rcv(ibp, hdr, has_grh, data, tlen, qp);
0243         break;
0244 
0245     case IB_QPT_RC:
0246         qib_rc_rcv(rcd, hdr, has_grh, data, tlen, qp);
0247         break;
0248 
0249     case IB_QPT_UC:
0250         qib_uc_rcv(ibp, hdr, has_grh, data, tlen, qp);
0251         break;
0252 
0253     default:
0254         break;
0255     }
0256 
0257 unlock:
0258     spin_unlock(&qp->r_lock);
0259 }
0260 
0261 /**
0262  * qib_ib_rcv - process an incoming packet
0263  * @rcd: the context pointer
0264  * @rhdr: the header of the packet
0265  * @data: the packet payload
0266  * @tlen: the packet length
0267  *
0268  * This is called from qib_kreceive() to process an incoming packet at
0269  * interrupt level. Tlen is the length of the header + data + CRC in bytes.
0270  */
0271 void qib_ib_rcv(struct qib_ctxtdata *rcd, void *rhdr, void *data, u32 tlen)
0272 {
0273     struct qib_pportdata *ppd = rcd->ppd;
0274     struct qib_ibport *ibp = &ppd->ibport_data;
0275     struct ib_header *hdr = rhdr;
0276     struct qib_devdata *dd = ppd->dd;
0277     struct rvt_dev_info *rdi = &dd->verbs_dev.rdi;
0278     struct ib_other_headers *ohdr;
0279     struct rvt_qp *qp;
0280     u32 qp_num;
0281     int lnh;
0282     u8 opcode;
0283     u16 lid;
0284 
0285     /* 24 == LRH+BTH+CRC */
0286     if (unlikely(tlen < 24))
0287         goto drop;
0288 
0289     /* Check for a valid destination LID (see ch. 7.11.1). */
0290     lid = be16_to_cpu(hdr->lrh[1]);
0291     if (lid < be16_to_cpu(IB_MULTICAST_LID_BASE)) {
0292         lid &= ~((1 << ppd->lmc) - 1);
0293         if (unlikely(lid != ppd->lid))
0294             goto drop;
0295     }
0296 
0297     /* Check for GRH */
0298     lnh = be16_to_cpu(hdr->lrh[0]) & 3;
0299     if (lnh == QIB_LRH_BTH)
0300         ohdr = &hdr->u.oth;
0301     else if (lnh == QIB_LRH_GRH) {
0302         u32 vtf;
0303 
0304         ohdr = &hdr->u.l.oth;
0305         if (hdr->u.l.grh.next_hdr != IB_GRH_NEXT_HDR)
0306             goto drop;
0307         vtf = be32_to_cpu(hdr->u.l.grh.version_tclass_flow);
0308         if ((vtf >> IB_GRH_VERSION_SHIFT) != IB_GRH_VERSION)
0309             goto drop;
0310     } else
0311         goto drop;
0312 
0313     opcode = (be32_to_cpu(ohdr->bth[0]) >> 24) & 0x7f;
0314 #ifdef CONFIG_DEBUG_FS
0315     rcd->opstats->stats[opcode].n_bytes += tlen;
0316     rcd->opstats->stats[opcode].n_packets++;
0317 #endif
0318 
0319     /* Get the destination QP number. */
0320     qp_num = be32_to_cpu(ohdr->bth[1]) & RVT_QPN_MASK;
0321     if (qp_num == QIB_MULTICAST_QPN) {
0322         struct rvt_mcast *mcast;
0323         struct rvt_mcast_qp *p;
0324 
0325         if (lnh != QIB_LRH_GRH)
0326             goto drop;
0327         mcast = rvt_mcast_find(&ibp->rvp, &hdr->u.l.grh.dgid, lid);
0328         if (mcast == NULL)
0329             goto drop;
0330         this_cpu_inc(ibp->pmastats->n_multicast_rcv);
0331         rcu_read_lock();
0332         list_for_each_entry_rcu(p, &mcast->qp_list, list)
0333             qib_qp_rcv(rcd, hdr, 1, data, tlen, p->qp);
0334         rcu_read_unlock();
0335         /*
0336          * Notify rvt_multicast_detach() if it is waiting for us
0337          * to finish.
0338          */
0339         if (atomic_dec_return(&mcast->refcount) <= 1)
0340             wake_up(&mcast->wait);
0341     } else {
0342         rcu_read_lock();
0343         qp = rvt_lookup_qpn(rdi, &ibp->rvp, qp_num);
0344         if (!qp) {
0345             rcu_read_unlock();
0346             goto drop;
0347         }
0348         this_cpu_inc(ibp->pmastats->n_unicast_rcv);
0349         qib_qp_rcv(rcd, hdr, lnh == QIB_LRH_GRH, data, tlen, qp);
0350         rcu_read_unlock();
0351     }
0352     return;
0353 
0354 drop:
0355     ibp->rvp.n_pkt_drops++;
0356 }
0357 
0358 /*
0359  * This is called from a timer to check for QPs
0360  * which need kernel memory in order to send a packet.
0361  */
0362 static void mem_timer(struct timer_list *t)
0363 {
0364     struct qib_ibdev *dev = from_timer(dev, t, mem_timer);
0365     struct list_head *list = &dev->memwait;
0366     struct rvt_qp *qp = NULL;
0367     struct qib_qp_priv *priv = NULL;
0368     unsigned long flags;
0369 
0370     spin_lock_irqsave(&dev->rdi.pending_lock, flags);
0371     if (!list_empty(list)) {
0372         priv = list_entry(list->next, struct qib_qp_priv, iowait);
0373         qp = priv->owner;
0374         list_del_init(&priv->iowait);
0375         rvt_get_qp(qp);
0376         if (!list_empty(list))
0377             mod_timer(&dev->mem_timer, jiffies + 1);
0378     }
0379     spin_unlock_irqrestore(&dev->rdi.pending_lock, flags);
0380 
0381     if (qp) {
0382         spin_lock_irqsave(&qp->s_lock, flags);
0383         if (qp->s_flags & RVT_S_WAIT_KMEM) {
0384             qp->s_flags &= ~RVT_S_WAIT_KMEM;
0385             qib_schedule_send(qp);
0386         }
0387         spin_unlock_irqrestore(&qp->s_lock, flags);
0388         rvt_put_qp(qp);
0389     }
0390 }
0391 
0392 #ifdef __LITTLE_ENDIAN
0393 static inline u32 get_upper_bits(u32 data, u32 shift)
0394 {
0395     return data >> shift;
0396 }
0397 
0398 static inline u32 set_upper_bits(u32 data, u32 shift)
0399 {
0400     return data << shift;
0401 }
0402 
0403 static inline u32 clear_upper_bytes(u32 data, u32 n, u32 off)
0404 {
0405     data <<= ((sizeof(u32) - n) * BITS_PER_BYTE);
0406     data >>= ((sizeof(u32) - n - off) * BITS_PER_BYTE);
0407     return data;
0408 }
0409 #else
0410 static inline u32 get_upper_bits(u32 data, u32 shift)
0411 {
0412     return data << shift;
0413 }
0414 
0415 static inline u32 set_upper_bits(u32 data, u32 shift)
0416 {
0417     return data >> shift;
0418 }
0419 
0420 static inline u32 clear_upper_bytes(u32 data, u32 n, u32 off)
0421 {
0422     data >>= ((sizeof(u32) - n) * BITS_PER_BYTE);
0423     data <<= ((sizeof(u32) - n - off) * BITS_PER_BYTE);
0424     return data;
0425 }
0426 #endif
0427 
0428 static void qib_copy_io(u32 __iomem *piobuf, struct rvt_sge_state *ss,
0429             u32 length, unsigned flush_wc)
0430 {
0431     u32 extra = 0;
0432     u32 data = 0;
0433     u32 last;
0434 
0435     while (1) {
0436         u32 len = rvt_get_sge_length(&ss->sge, length);
0437         u32 off;
0438 
0439         /* If the source address is not aligned, try to align it. */
0440         off = (unsigned long)ss->sge.vaddr & (sizeof(u32) - 1);
0441         if (off) {
0442             u32 *addr = (u32 *)((unsigned long)ss->sge.vaddr &
0443                         ~(sizeof(u32) - 1));
0444             u32 v = get_upper_bits(*addr, off * BITS_PER_BYTE);
0445             u32 y;
0446 
0447             y = sizeof(u32) - off;
0448             if (len > y)
0449                 len = y;
0450             if (len + extra >= sizeof(u32)) {
0451                 data |= set_upper_bits(v, extra *
0452                                BITS_PER_BYTE);
0453                 len = sizeof(u32) - extra;
0454                 if (len == length) {
0455                     last = data;
0456                     break;
0457                 }
0458                 __raw_writel(data, piobuf);
0459                 piobuf++;
0460                 extra = 0;
0461                 data = 0;
0462             } else {
0463                 /* Clear unused upper bytes */
0464                 data |= clear_upper_bytes(v, len, extra);
0465                 if (len == length) {
0466                     last = data;
0467                     break;
0468                 }
0469                 extra += len;
0470             }
0471         } else if (extra) {
0472             /* Source address is aligned. */
0473             u32 *addr = (u32 *) ss->sge.vaddr;
0474             int shift = extra * BITS_PER_BYTE;
0475             int ushift = 32 - shift;
0476             u32 l = len;
0477 
0478             while (l >= sizeof(u32)) {
0479                 u32 v = *addr;
0480 
0481                 data |= set_upper_bits(v, shift);
0482                 __raw_writel(data, piobuf);
0483                 data = get_upper_bits(v, ushift);
0484                 piobuf++;
0485                 addr++;
0486                 l -= sizeof(u32);
0487             }
0488             /*
0489              * We still have 'extra' number of bytes leftover.
0490              */
0491             if (l) {
0492                 u32 v = *addr;
0493 
0494                 if (l + extra >= sizeof(u32)) {
0495                     data |= set_upper_bits(v, shift);
0496                     len -= l + extra - sizeof(u32);
0497                     if (len == length) {
0498                         last = data;
0499                         break;
0500                     }
0501                     __raw_writel(data, piobuf);
0502                     piobuf++;
0503                     extra = 0;
0504                     data = 0;
0505                 } else {
0506                     /* Clear unused upper bytes */
0507                     data |= clear_upper_bytes(v, l, extra);
0508                     if (len == length) {
0509                         last = data;
0510                         break;
0511                     }
0512                     extra += l;
0513                 }
0514             } else if (len == length) {
0515                 last = data;
0516                 break;
0517             }
0518         } else if (len == length) {
0519             u32 w;
0520 
0521             /*
0522              * Need to round up for the last dword in the
0523              * packet.
0524              */
0525             w = (len + 3) >> 2;
0526             qib_pio_copy(piobuf, ss->sge.vaddr, w - 1);
0527             piobuf += w - 1;
0528             last = ((u32 *) ss->sge.vaddr)[w - 1];
0529             break;
0530         } else {
0531             u32 w = len >> 2;
0532 
0533             qib_pio_copy(piobuf, ss->sge.vaddr, w);
0534             piobuf += w;
0535 
0536             extra = len & (sizeof(u32) - 1);
0537             if (extra) {
0538                 u32 v = ((u32 *) ss->sge.vaddr)[w];
0539 
0540                 /* Clear unused upper bytes */
0541                 data = clear_upper_bytes(v, extra, 0);
0542             }
0543         }
0544         rvt_update_sge(ss, len, false);
0545         length -= len;
0546     }
0547     /* Update address before sending packet. */
0548     rvt_update_sge(ss, length, false);
0549     if (flush_wc) {
0550         /* must flush early everything before trigger word */
0551         qib_flush_wc();
0552         __raw_writel(last, piobuf);
0553         /* be sure trigger word is written */
0554         qib_flush_wc();
0555     } else
0556         __raw_writel(last, piobuf);
0557 }
0558 
0559 static noinline struct qib_verbs_txreq *__get_txreq(struct qib_ibdev *dev,
0560                        struct rvt_qp *qp)
0561 {
0562     struct qib_qp_priv *priv = qp->priv;
0563     struct qib_verbs_txreq *tx;
0564     unsigned long flags;
0565 
0566     spin_lock_irqsave(&qp->s_lock, flags);
0567     spin_lock(&dev->rdi.pending_lock);
0568 
0569     if (!list_empty(&dev->txreq_free)) {
0570         struct list_head *l = dev->txreq_free.next;
0571 
0572         list_del(l);
0573         spin_unlock(&dev->rdi.pending_lock);
0574         spin_unlock_irqrestore(&qp->s_lock, flags);
0575         tx = list_entry(l, struct qib_verbs_txreq, txreq.list);
0576     } else {
0577         if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK &&
0578             list_empty(&priv->iowait)) {
0579             dev->n_txwait++;
0580             qp->s_flags |= RVT_S_WAIT_TX;
0581             list_add_tail(&priv->iowait, &dev->txwait);
0582         }
0583         qp->s_flags &= ~RVT_S_BUSY;
0584         spin_unlock(&dev->rdi.pending_lock);
0585         spin_unlock_irqrestore(&qp->s_lock, flags);
0586         tx = ERR_PTR(-EBUSY);
0587     }
0588     return tx;
0589 }
0590 
0591 static inline struct qib_verbs_txreq *get_txreq(struct qib_ibdev *dev,
0592                      struct rvt_qp *qp)
0593 {
0594     struct qib_verbs_txreq *tx;
0595     unsigned long flags;
0596 
0597     spin_lock_irqsave(&dev->rdi.pending_lock, flags);
0598     /* assume the list non empty */
0599     if (likely(!list_empty(&dev->txreq_free))) {
0600         struct list_head *l = dev->txreq_free.next;
0601 
0602         list_del(l);
0603         spin_unlock_irqrestore(&dev->rdi.pending_lock, flags);
0604         tx = list_entry(l, struct qib_verbs_txreq, txreq.list);
0605     } else {
0606         /* call slow path to get the extra lock */
0607         spin_unlock_irqrestore(&dev->rdi.pending_lock, flags);
0608         tx =  __get_txreq(dev, qp);
0609     }
0610     return tx;
0611 }
0612 
0613 void qib_put_txreq(struct qib_verbs_txreq *tx)
0614 {
0615     struct qib_ibdev *dev;
0616     struct rvt_qp *qp;
0617     struct qib_qp_priv *priv;
0618     unsigned long flags;
0619 
0620     qp = tx->qp;
0621     dev = to_idev(qp->ibqp.device);
0622 
0623     if (tx->mr) {
0624         rvt_put_mr(tx->mr);
0625         tx->mr = NULL;
0626     }
0627     if (tx->txreq.flags & QIB_SDMA_TXREQ_F_FREEBUF) {
0628         tx->txreq.flags &= ~QIB_SDMA_TXREQ_F_FREEBUF;
0629         dma_unmap_single(&dd_from_dev(dev)->pcidev->dev,
0630                  tx->txreq.addr, tx->hdr_dwords << 2,
0631                  DMA_TO_DEVICE);
0632         kfree(tx->align_buf);
0633     }
0634 
0635     spin_lock_irqsave(&dev->rdi.pending_lock, flags);
0636 
0637     /* Put struct back on free list */
0638     list_add(&tx->txreq.list, &dev->txreq_free);
0639 
0640     if (!list_empty(&dev->txwait)) {
0641         /* Wake up first QP wanting a free struct */
0642         priv = list_entry(dev->txwait.next, struct qib_qp_priv,
0643                   iowait);
0644         qp = priv->owner;
0645         list_del_init(&priv->iowait);
0646         rvt_get_qp(qp);
0647         spin_unlock_irqrestore(&dev->rdi.pending_lock, flags);
0648 
0649         spin_lock_irqsave(&qp->s_lock, flags);
0650         if (qp->s_flags & RVT_S_WAIT_TX) {
0651             qp->s_flags &= ~RVT_S_WAIT_TX;
0652             qib_schedule_send(qp);
0653         }
0654         spin_unlock_irqrestore(&qp->s_lock, flags);
0655 
0656         rvt_put_qp(qp);
0657     } else
0658         spin_unlock_irqrestore(&dev->rdi.pending_lock, flags);
0659 }
0660 
0661 /*
0662  * This is called when there are send DMA descriptors that might be
0663  * available.
0664  *
0665  * This is called with ppd->sdma_lock held.
0666  */
0667 void qib_verbs_sdma_desc_avail(struct qib_pportdata *ppd, unsigned avail)
0668 {
0669     struct rvt_qp *qp;
0670     struct qib_qp_priv *qpp, *nqpp;
0671     struct rvt_qp *qps[20];
0672     struct qib_ibdev *dev;
0673     unsigned i, n;
0674 
0675     n = 0;
0676     dev = &ppd->dd->verbs_dev;
0677     spin_lock(&dev->rdi.pending_lock);
0678 
0679     /* Search wait list for first QP wanting DMA descriptors. */
0680     list_for_each_entry_safe(qpp, nqpp, &dev->dmawait, iowait) {
0681         qp = qpp->owner;
0682         if (qp->port_num != ppd->port)
0683             continue;
0684         if (n == ARRAY_SIZE(qps))
0685             break;
0686         if (qpp->s_tx->txreq.sg_count > avail)
0687             break;
0688         avail -= qpp->s_tx->txreq.sg_count;
0689         list_del_init(&qpp->iowait);
0690         rvt_get_qp(qp);
0691         qps[n++] = qp;
0692     }
0693 
0694     spin_unlock(&dev->rdi.pending_lock);
0695 
0696     for (i = 0; i < n; i++) {
0697         qp = qps[i];
0698         spin_lock(&qp->s_lock);
0699         if (qp->s_flags & RVT_S_WAIT_DMA_DESC) {
0700             qp->s_flags &= ~RVT_S_WAIT_DMA_DESC;
0701             qib_schedule_send(qp);
0702         }
0703         spin_unlock(&qp->s_lock);
0704         rvt_put_qp(qp);
0705     }
0706 }
0707 
0708 /*
0709  * This is called with ppd->sdma_lock held.
0710  */
0711 static void sdma_complete(struct qib_sdma_txreq *cookie, int status)
0712 {
0713     struct qib_verbs_txreq *tx =
0714         container_of(cookie, struct qib_verbs_txreq, txreq);
0715     struct rvt_qp *qp = tx->qp;
0716     struct qib_qp_priv *priv = qp->priv;
0717 
0718     spin_lock(&qp->s_lock);
0719     if (tx->wqe)
0720         rvt_send_complete(qp, tx->wqe, IB_WC_SUCCESS);
0721     else if (qp->ibqp.qp_type == IB_QPT_RC) {
0722         struct ib_header *hdr;
0723 
0724         if (tx->txreq.flags & QIB_SDMA_TXREQ_F_FREEBUF)
0725             hdr = &tx->align_buf->hdr;
0726         else {
0727             struct qib_ibdev *dev = to_idev(qp->ibqp.device);
0728 
0729             hdr = &dev->pio_hdrs[tx->hdr_inx].hdr;
0730         }
0731         qib_rc_send_complete(qp, hdr);
0732     }
0733     if (atomic_dec_and_test(&priv->s_dma_busy)) {
0734         if (qp->state == IB_QPS_RESET)
0735             wake_up(&priv->wait_dma);
0736         else if (qp->s_flags & RVT_S_WAIT_DMA) {
0737             qp->s_flags &= ~RVT_S_WAIT_DMA;
0738             qib_schedule_send(qp);
0739         }
0740     }
0741     spin_unlock(&qp->s_lock);
0742 
0743     qib_put_txreq(tx);
0744 }
0745 
0746 static int wait_kmem(struct qib_ibdev *dev, struct rvt_qp *qp)
0747 {
0748     struct qib_qp_priv *priv = qp->priv;
0749     unsigned long flags;
0750     int ret = 0;
0751 
0752     spin_lock_irqsave(&qp->s_lock, flags);
0753     if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) {
0754         spin_lock(&dev->rdi.pending_lock);
0755         if (list_empty(&priv->iowait)) {
0756             if (list_empty(&dev->memwait))
0757                 mod_timer(&dev->mem_timer, jiffies + 1);
0758             qp->s_flags |= RVT_S_WAIT_KMEM;
0759             list_add_tail(&priv->iowait, &dev->memwait);
0760         }
0761         spin_unlock(&dev->rdi.pending_lock);
0762         qp->s_flags &= ~RVT_S_BUSY;
0763         ret = -EBUSY;
0764     }
0765     spin_unlock_irqrestore(&qp->s_lock, flags);
0766 
0767     return ret;
0768 }
0769 
0770 static int qib_verbs_send_dma(struct rvt_qp *qp, struct ib_header *hdr,
0771                   u32 hdrwords, struct rvt_sge_state *ss, u32 len,
0772                   u32 plen, u32 dwords)
0773 {
0774     struct qib_qp_priv *priv = qp->priv;
0775     struct qib_ibdev *dev = to_idev(qp->ibqp.device);
0776     struct qib_devdata *dd = dd_from_dev(dev);
0777     struct qib_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
0778     struct qib_pportdata *ppd = ppd_from_ibp(ibp);
0779     struct qib_verbs_txreq *tx;
0780     struct qib_pio_header *phdr;
0781     u32 control;
0782     u32 ndesc;
0783     int ret;
0784 
0785     tx = priv->s_tx;
0786     if (tx) {
0787         priv->s_tx = NULL;
0788         /* resend previously constructed packet */
0789         ret = qib_sdma_verbs_send(ppd, tx->ss, tx->dwords, tx);
0790         goto bail;
0791     }
0792 
0793     tx = get_txreq(dev, qp);
0794     if (IS_ERR(tx))
0795         goto bail_tx;
0796 
0797     control = dd->f_setpbc_control(ppd, plen, qp->s_srate,
0798                        be16_to_cpu(hdr->lrh[0]) >> 12);
0799     tx->qp = qp;
0800     tx->wqe = qp->s_wqe;
0801     tx->mr = qp->s_rdma_mr;
0802     if (qp->s_rdma_mr)
0803         qp->s_rdma_mr = NULL;
0804     tx->txreq.callback = sdma_complete;
0805     if (dd->flags & QIB_HAS_SDMA_TIMEOUT)
0806         tx->txreq.flags = QIB_SDMA_TXREQ_F_HEADTOHOST;
0807     else
0808         tx->txreq.flags = QIB_SDMA_TXREQ_F_INTREQ;
0809     if (plen + 1 > dd->piosize2kmax_dwords)
0810         tx->txreq.flags |= QIB_SDMA_TXREQ_F_USELARGEBUF;
0811 
0812     if (len) {
0813         /*
0814          * Don't try to DMA if it takes more descriptors than
0815          * the queue holds.
0816          */
0817         ndesc = qib_count_sge(ss, len);
0818         if (ndesc >= ppd->sdma_descq_cnt)
0819             ndesc = 0;
0820     } else
0821         ndesc = 1;
0822     if (ndesc) {
0823         phdr = &dev->pio_hdrs[tx->hdr_inx];
0824         phdr->pbc[0] = cpu_to_le32(plen);
0825         phdr->pbc[1] = cpu_to_le32(control);
0826         memcpy(&phdr->hdr, hdr, hdrwords << 2);
0827         tx->txreq.flags |= QIB_SDMA_TXREQ_F_FREEDESC;
0828         tx->txreq.sg_count = ndesc;
0829         tx->txreq.addr = dev->pio_hdrs_phys +
0830             tx->hdr_inx * sizeof(struct qib_pio_header);
0831         tx->hdr_dwords = hdrwords + 2; /* add PBC length */
0832         ret = qib_sdma_verbs_send(ppd, ss, dwords, tx);
0833         goto bail;
0834     }
0835 
0836     /* Allocate a buffer and copy the header and payload to it. */
0837     tx->hdr_dwords = plen + 1;
0838     phdr = kmalloc(tx->hdr_dwords << 2, GFP_ATOMIC);
0839     if (!phdr)
0840         goto err_tx;
0841     phdr->pbc[0] = cpu_to_le32(plen);
0842     phdr->pbc[1] = cpu_to_le32(control);
0843     memcpy(&phdr->hdr, hdr, hdrwords << 2);
0844     qib_copy_from_sge((u32 *) &phdr->hdr + hdrwords, ss, len);
0845 
0846     tx->txreq.addr = dma_map_single(&dd->pcidev->dev, phdr,
0847                     tx->hdr_dwords << 2, DMA_TO_DEVICE);
0848     if (dma_mapping_error(&dd->pcidev->dev, tx->txreq.addr))
0849         goto map_err;
0850     tx->align_buf = phdr;
0851     tx->txreq.flags |= QIB_SDMA_TXREQ_F_FREEBUF;
0852     tx->txreq.sg_count = 1;
0853     ret = qib_sdma_verbs_send(ppd, NULL, 0, tx);
0854     goto unaligned;
0855 
0856 map_err:
0857     kfree(phdr);
0858 err_tx:
0859     qib_put_txreq(tx);
0860     ret = wait_kmem(dev, qp);
0861 unaligned:
0862     ibp->rvp.n_unaligned++;
0863 bail:
0864     return ret;
0865 bail_tx:
0866     ret = PTR_ERR(tx);
0867     goto bail;
0868 }
0869 
0870 /*
0871  * If we are now in the error state, return zero to flush the
0872  * send work request.
0873  */
0874 static int no_bufs_available(struct rvt_qp *qp)
0875 {
0876     struct qib_qp_priv *priv = qp->priv;
0877     struct qib_ibdev *dev = to_idev(qp->ibqp.device);
0878     struct qib_devdata *dd;
0879     unsigned long flags;
0880     int ret = 0;
0881 
0882     /*
0883      * Note that as soon as want_buffer() is called and
0884      * possibly before it returns, qib_ib_piobufavail()
0885      * could be called. Therefore, put QP on the I/O wait list before
0886      * enabling the PIO avail interrupt.
0887      */
0888     spin_lock_irqsave(&qp->s_lock, flags);
0889     if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) {
0890         spin_lock(&dev->rdi.pending_lock);
0891         if (list_empty(&priv->iowait)) {
0892             dev->n_piowait++;
0893             qp->s_flags |= RVT_S_WAIT_PIO;
0894             list_add_tail(&priv->iowait, &dev->piowait);
0895             dd = dd_from_dev(dev);
0896             dd->f_wantpiobuf_intr(dd, 1);
0897         }
0898         spin_unlock(&dev->rdi.pending_lock);
0899         qp->s_flags &= ~RVT_S_BUSY;
0900         ret = -EBUSY;
0901     }
0902     spin_unlock_irqrestore(&qp->s_lock, flags);
0903     return ret;
0904 }
0905 
0906 static int qib_verbs_send_pio(struct rvt_qp *qp, struct ib_header *ibhdr,
0907                   u32 hdrwords, struct rvt_sge_state *ss, u32 len,
0908                   u32 plen, u32 dwords)
0909 {
0910     struct qib_devdata *dd = dd_from_ibdev(qp->ibqp.device);
0911     struct qib_pportdata *ppd = dd->pport + qp->port_num - 1;
0912     u32 *hdr = (u32 *) ibhdr;
0913     u32 __iomem *piobuf_orig;
0914     u32 __iomem *piobuf;
0915     u64 pbc;
0916     unsigned long flags;
0917     unsigned flush_wc;
0918     u32 control;
0919     u32 pbufn;
0920 
0921     control = dd->f_setpbc_control(ppd, plen, qp->s_srate,
0922         be16_to_cpu(ibhdr->lrh[0]) >> 12);
0923     pbc = ((u64) control << 32) | plen;
0924     piobuf = dd->f_getsendbuf(ppd, pbc, &pbufn);
0925     if (unlikely(piobuf == NULL))
0926         return no_bufs_available(qp);
0927 
0928     /*
0929      * Write the pbc.
0930      * We have to flush after the PBC for correctness on some cpus
0931      * or WC buffer can be written out of order.
0932      */
0933     writeq(pbc, piobuf);
0934     piobuf_orig = piobuf;
0935     piobuf += 2;
0936 
0937     flush_wc = dd->flags & QIB_PIO_FLUSH_WC;
0938     if (len == 0) {
0939         /*
0940          * If there is just the header portion, must flush before
0941          * writing last word of header for correctness, and after
0942          * the last header word (trigger word).
0943          */
0944         if (flush_wc) {
0945             qib_flush_wc();
0946             qib_pio_copy(piobuf, hdr, hdrwords - 1);
0947             qib_flush_wc();
0948             __raw_writel(hdr[hdrwords - 1], piobuf + hdrwords - 1);
0949             qib_flush_wc();
0950         } else
0951             qib_pio_copy(piobuf, hdr, hdrwords);
0952         goto done;
0953     }
0954 
0955     if (flush_wc)
0956         qib_flush_wc();
0957     qib_pio_copy(piobuf, hdr, hdrwords);
0958     piobuf += hdrwords;
0959 
0960     /* The common case is aligned and contained in one segment. */
0961     if (likely(ss->num_sge == 1 && len <= ss->sge.length &&
0962            !((unsigned long)ss->sge.vaddr & (sizeof(u32) - 1)))) {
0963         u32 *addr = (u32 *) ss->sge.vaddr;
0964 
0965         /* Update address before sending packet. */
0966         rvt_update_sge(ss, len, false);
0967         if (flush_wc) {
0968             qib_pio_copy(piobuf, addr, dwords - 1);
0969             /* must flush early everything before trigger word */
0970             qib_flush_wc();
0971             __raw_writel(addr[dwords - 1], piobuf + dwords - 1);
0972             /* be sure trigger word is written */
0973             qib_flush_wc();
0974         } else
0975             qib_pio_copy(piobuf, addr, dwords);
0976         goto done;
0977     }
0978     qib_copy_io(piobuf, ss, len, flush_wc);
0979 done:
0980     if (dd->flags & QIB_USE_SPCL_TRIG) {
0981         u32 spcl_off = (pbufn >= dd->piobcnt2k) ? 2047 : 1023;
0982 
0983         qib_flush_wc();
0984         __raw_writel(0xaebecede, piobuf_orig + spcl_off);
0985     }
0986     qib_sendbuf_done(dd, pbufn);
0987     if (qp->s_rdma_mr) {
0988         rvt_put_mr(qp->s_rdma_mr);
0989         qp->s_rdma_mr = NULL;
0990     }
0991     if (qp->s_wqe) {
0992         spin_lock_irqsave(&qp->s_lock, flags);
0993         rvt_send_complete(qp, qp->s_wqe, IB_WC_SUCCESS);
0994         spin_unlock_irqrestore(&qp->s_lock, flags);
0995     } else if (qp->ibqp.qp_type == IB_QPT_RC) {
0996         spin_lock_irqsave(&qp->s_lock, flags);
0997         qib_rc_send_complete(qp, ibhdr);
0998         spin_unlock_irqrestore(&qp->s_lock, flags);
0999     }
1000     return 0;
1001 }
1002 
1003 /**
1004  * qib_verbs_send - send a packet
1005  * @qp: the QP to send on
1006  * @hdr: the packet header
1007  * @hdrwords: the number of 32-bit words in the header
1008  * @ss: the SGE to send
1009  * @len: the length of the packet in bytes
1010  *
1011  * Return zero if packet is sent or queued OK.
1012  * Return non-zero and clear qp->s_flags RVT_S_BUSY otherwise.
1013  */
1014 int qib_verbs_send(struct rvt_qp *qp, struct ib_header *hdr,
1015            u32 hdrwords, struct rvt_sge_state *ss, u32 len)
1016 {
1017     struct qib_devdata *dd = dd_from_ibdev(qp->ibqp.device);
1018     u32 plen;
1019     int ret;
1020     u32 dwords = (len + 3) >> 2;
1021 
1022     /*
1023      * Calculate the send buffer trigger address.
1024      * The +1 counts for the pbc control dword following the pbc length.
1025      */
1026     plen = hdrwords + dwords + 1;
1027 
1028     /*
1029      * VL15 packets (IB_QPT_SMI) will always use PIO, so we
1030      * can defer SDMA restart until link goes ACTIVE without
1031      * worrying about just how we got there.
1032      */
1033     if (qp->ibqp.qp_type == IB_QPT_SMI ||
1034         !(dd->flags & QIB_HAS_SEND_DMA))
1035         ret = qib_verbs_send_pio(qp, hdr, hdrwords, ss, len,
1036                      plen, dwords);
1037     else
1038         ret = qib_verbs_send_dma(qp, hdr, hdrwords, ss, len,
1039                      plen, dwords);
1040 
1041     return ret;
1042 }
1043 
1044 int qib_snapshot_counters(struct qib_pportdata *ppd, u64 *swords,
1045               u64 *rwords, u64 *spkts, u64 *rpkts,
1046               u64 *xmit_wait)
1047 {
1048     int ret;
1049     struct qib_devdata *dd = ppd->dd;
1050 
1051     if (!(dd->flags & QIB_PRESENT)) {
1052         /* no hardware, freeze, etc. */
1053         ret = -EINVAL;
1054         goto bail;
1055     }
1056     *swords = dd->f_portcntr(ppd, QIBPORTCNTR_WORDSEND);
1057     *rwords = dd->f_portcntr(ppd, QIBPORTCNTR_WORDRCV);
1058     *spkts = dd->f_portcntr(ppd, QIBPORTCNTR_PKTSEND);
1059     *rpkts = dd->f_portcntr(ppd, QIBPORTCNTR_PKTRCV);
1060     *xmit_wait = dd->f_portcntr(ppd, QIBPORTCNTR_SENDSTALL);
1061 
1062     ret = 0;
1063 
1064 bail:
1065     return ret;
1066 }
1067 
1068 /**
1069  * qib_get_counters - get various chip counters
1070  * @ppd: the qlogic_ib device
1071  * @cntrs: counters are placed here
1072  *
1073  * Return the counters needed by recv_pma_get_portcounters().
1074  */
1075 int qib_get_counters(struct qib_pportdata *ppd,
1076              struct qib_verbs_counters *cntrs)
1077 {
1078     int ret;
1079 
1080     if (!(ppd->dd->flags & QIB_PRESENT)) {
1081         /* no hardware, freeze, etc. */
1082         ret = -EINVAL;
1083         goto bail;
1084     }
1085     cntrs->symbol_error_counter =
1086         ppd->dd->f_portcntr(ppd, QIBPORTCNTR_IBSYMBOLERR);
1087     cntrs->link_error_recovery_counter =
1088         ppd->dd->f_portcntr(ppd, QIBPORTCNTR_IBLINKERRRECOV);
1089     /*
1090      * The link downed counter counts when the other side downs the
1091      * connection.  We add in the number of times we downed the link
1092      * due to local link integrity errors to compensate.
1093      */
1094     cntrs->link_downed_counter =
1095         ppd->dd->f_portcntr(ppd, QIBPORTCNTR_IBLINKDOWN);
1096     cntrs->port_rcv_errors =
1097         ppd->dd->f_portcntr(ppd, QIBPORTCNTR_RXDROPPKT) +
1098         ppd->dd->f_portcntr(ppd, QIBPORTCNTR_RCVOVFL) +
1099         ppd->dd->f_portcntr(ppd, QIBPORTCNTR_ERR_RLEN) +
1100         ppd->dd->f_portcntr(ppd, QIBPORTCNTR_INVALIDRLEN) +
1101         ppd->dd->f_portcntr(ppd, QIBPORTCNTR_ERRLINK) +
1102         ppd->dd->f_portcntr(ppd, QIBPORTCNTR_ERRICRC) +
1103         ppd->dd->f_portcntr(ppd, QIBPORTCNTR_ERRVCRC) +
1104         ppd->dd->f_portcntr(ppd, QIBPORTCNTR_ERRLPCRC) +
1105         ppd->dd->f_portcntr(ppd, QIBPORTCNTR_BADFORMAT);
1106     cntrs->port_rcv_errors +=
1107         ppd->dd->f_portcntr(ppd, QIBPORTCNTR_RXLOCALPHYERR);
1108     cntrs->port_rcv_errors +=
1109         ppd->dd->f_portcntr(ppd, QIBPORTCNTR_RXVLERR);
1110     cntrs->port_rcv_remphys_errors =
1111         ppd->dd->f_portcntr(ppd, QIBPORTCNTR_RCVEBP);
1112     cntrs->port_xmit_discards =
1113         ppd->dd->f_portcntr(ppd, QIBPORTCNTR_UNSUPVL);
1114     cntrs->port_xmit_data = ppd->dd->f_portcntr(ppd,
1115             QIBPORTCNTR_WORDSEND);
1116     cntrs->port_rcv_data = ppd->dd->f_portcntr(ppd,
1117             QIBPORTCNTR_WORDRCV);
1118     cntrs->port_xmit_packets = ppd->dd->f_portcntr(ppd,
1119             QIBPORTCNTR_PKTSEND);
1120     cntrs->port_rcv_packets = ppd->dd->f_portcntr(ppd,
1121             QIBPORTCNTR_PKTRCV);
1122     cntrs->local_link_integrity_errors =
1123         ppd->dd->f_portcntr(ppd, QIBPORTCNTR_LLI);
1124     cntrs->excessive_buffer_overrun_errors =
1125         ppd->dd->f_portcntr(ppd, QIBPORTCNTR_EXCESSBUFOVFL);
1126     cntrs->vl15_dropped =
1127         ppd->dd->f_portcntr(ppd, QIBPORTCNTR_VL15PKTDROP);
1128 
1129     ret = 0;
1130 
1131 bail:
1132     return ret;
1133 }
1134 
1135 /**
1136  * qib_ib_piobufavail - callback when a PIO buffer is available
1137  * @dd: the device pointer
1138  *
1139  * This is called from qib_intr() at interrupt level when a PIO buffer is
1140  * available after qib_verbs_send() returned an error that no buffers were
1141  * available. Disable the interrupt if there are no more QPs waiting.
1142  */
1143 void qib_ib_piobufavail(struct qib_devdata *dd)
1144 {
1145     struct qib_ibdev *dev = &dd->verbs_dev;
1146     struct list_head *list;
1147     struct rvt_qp *qps[5];
1148     struct rvt_qp *qp;
1149     unsigned long flags;
1150     unsigned i, n;
1151     struct qib_qp_priv *priv;
1152 
1153     list = &dev->piowait;
1154     n = 0;
1155 
1156     /*
1157      * Note: checking that the piowait list is empty and clearing
1158      * the buffer available interrupt needs to be atomic or we
1159      * could end up with QPs on the wait list with the interrupt
1160      * disabled.
1161      */
1162     spin_lock_irqsave(&dev->rdi.pending_lock, flags);
1163     while (!list_empty(list)) {
1164         if (n == ARRAY_SIZE(qps))
1165             goto full;
1166         priv = list_entry(list->next, struct qib_qp_priv, iowait);
1167         qp = priv->owner;
1168         list_del_init(&priv->iowait);
1169         rvt_get_qp(qp);
1170         qps[n++] = qp;
1171     }
1172     dd->f_wantpiobuf_intr(dd, 0);
1173 full:
1174     spin_unlock_irqrestore(&dev->rdi.pending_lock, flags);
1175 
1176     for (i = 0; i < n; i++) {
1177         qp = qps[i];
1178 
1179         spin_lock_irqsave(&qp->s_lock, flags);
1180         if (qp->s_flags & RVT_S_WAIT_PIO) {
1181             qp->s_flags &= ~RVT_S_WAIT_PIO;
1182             qib_schedule_send(qp);
1183         }
1184         spin_unlock_irqrestore(&qp->s_lock, flags);
1185 
1186         /* Notify qib_destroy_qp() if it is waiting. */
1187         rvt_put_qp(qp);
1188     }
1189 }
1190 
1191 static int qib_query_port(struct rvt_dev_info *rdi, u32 port_num,
1192               struct ib_port_attr *props)
1193 {
1194     struct qib_ibdev *ibdev = container_of(rdi, struct qib_ibdev, rdi);
1195     struct qib_devdata *dd = dd_from_dev(ibdev);
1196     struct qib_pportdata *ppd = &dd->pport[port_num - 1];
1197     enum ib_mtu mtu;
1198     u16 lid = ppd->lid;
1199 
1200     /* props being zeroed by the caller, avoid zeroing it here */
1201     props->lid = lid ? lid : be16_to_cpu(IB_LID_PERMISSIVE);
1202     props->lmc = ppd->lmc;
1203     props->state = dd->f_iblink_state(ppd->lastibcstat);
1204     props->phys_state = dd->f_ibphys_portstate(ppd->lastibcstat);
1205     props->gid_tbl_len = QIB_GUIDS_PER_PORT;
1206     props->active_width = ppd->link_width_active;
1207     /* See rate_show() */
1208     props->active_speed = ppd->link_speed_active;
1209     props->max_vl_num = qib_num_vls(ppd->vls_supported);
1210 
1211     props->max_mtu = qib_ibmtu ? qib_ibmtu : IB_MTU_4096;
1212     switch (ppd->ibmtu) {
1213     case 4096:
1214         mtu = IB_MTU_4096;
1215         break;
1216     case 2048:
1217         mtu = IB_MTU_2048;
1218         break;
1219     case 1024:
1220         mtu = IB_MTU_1024;
1221         break;
1222     case 512:
1223         mtu = IB_MTU_512;
1224         break;
1225     case 256:
1226         mtu = IB_MTU_256;
1227         break;
1228     default:
1229         mtu = IB_MTU_2048;
1230     }
1231     props->active_mtu = mtu;
1232 
1233     return 0;
1234 }
1235 
1236 static int qib_modify_device(struct ib_device *device,
1237                  int device_modify_mask,
1238                  struct ib_device_modify *device_modify)
1239 {
1240     struct qib_devdata *dd = dd_from_ibdev(device);
1241     unsigned i;
1242     int ret;
1243 
1244     if (device_modify_mask & ~(IB_DEVICE_MODIFY_SYS_IMAGE_GUID |
1245                    IB_DEVICE_MODIFY_NODE_DESC)) {
1246         ret = -EOPNOTSUPP;
1247         goto bail;
1248     }
1249 
1250     if (device_modify_mask & IB_DEVICE_MODIFY_NODE_DESC) {
1251         memcpy(device->node_desc, device_modify->node_desc,
1252                IB_DEVICE_NODE_DESC_MAX);
1253         for (i = 0; i < dd->num_pports; i++) {
1254             struct qib_ibport *ibp = &dd->pport[i].ibport_data;
1255 
1256             qib_node_desc_chg(ibp);
1257         }
1258     }
1259 
1260     if (device_modify_mask & IB_DEVICE_MODIFY_SYS_IMAGE_GUID) {
1261         ib_qib_sys_image_guid =
1262             cpu_to_be64(device_modify->sys_image_guid);
1263         for (i = 0; i < dd->num_pports; i++) {
1264             struct qib_ibport *ibp = &dd->pport[i].ibport_data;
1265 
1266             qib_sys_guid_chg(ibp);
1267         }
1268     }
1269 
1270     ret = 0;
1271 
1272 bail:
1273     return ret;
1274 }
1275 
1276 static int qib_shut_down_port(struct rvt_dev_info *rdi, u32 port_num)
1277 {
1278     struct qib_ibdev *ibdev = container_of(rdi, struct qib_ibdev, rdi);
1279     struct qib_devdata *dd = dd_from_dev(ibdev);
1280     struct qib_pportdata *ppd = &dd->pport[port_num - 1];
1281 
1282     qib_set_linkstate(ppd, QIB_IB_LINKDOWN);
1283 
1284     return 0;
1285 }
1286 
1287 static int qib_get_guid_be(struct rvt_dev_info *rdi, struct rvt_ibport *rvp,
1288                int guid_index, __be64 *guid)
1289 {
1290     struct qib_ibport *ibp = container_of(rvp, struct qib_ibport, rvp);
1291     struct qib_pportdata *ppd = ppd_from_ibp(ibp);
1292 
1293     if (guid_index == 0)
1294         *guid = ppd->guid;
1295     else if (guid_index < QIB_GUIDS_PER_PORT)
1296         *guid = ibp->guids[guid_index - 1];
1297     else
1298         return -EINVAL;
1299 
1300     return 0;
1301 }
1302 
1303 int qib_check_ah(struct ib_device *ibdev, struct rdma_ah_attr *ah_attr)
1304 {
1305     if (rdma_ah_get_sl(ah_attr) > 15)
1306         return -EINVAL;
1307 
1308     if (rdma_ah_get_dlid(ah_attr) == 0)
1309         return -EINVAL;
1310     if (rdma_ah_get_dlid(ah_attr) >=
1311         be16_to_cpu(IB_MULTICAST_LID_BASE) &&
1312         rdma_ah_get_dlid(ah_attr) !=
1313         be16_to_cpu(IB_LID_PERMISSIVE) &&
1314         !(rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH))
1315         return -EINVAL;
1316 
1317     return 0;
1318 }
1319 
1320 static void qib_notify_new_ah(struct ib_device *ibdev,
1321                   struct rdma_ah_attr *ah_attr,
1322                   struct rvt_ah *ah)
1323 {
1324     struct qib_ibport *ibp;
1325     struct qib_pportdata *ppd;
1326 
1327     /*
1328      * Do not trust reading anything from rvt_ah at this point as it is not
1329      * done being setup. We can however modify things which we need to set.
1330      */
1331 
1332     ibp = to_iport(ibdev, rdma_ah_get_port_num(ah_attr));
1333     ppd = ppd_from_ibp(ibp);
1334     ah->vl = ibp->sl_to_vl[rdma_ah_get_sl(&ah->attr)];
1335     ah->log_pmtu = ilog2(ppd->ibmtu);
1336 }
1337 
1338 struct ib_ah *qib_create_qp0_ah(struct qib_ibport *ibp, u16 dlid)
1339 {
1340     struct rdma_ah_attr attr;
1341     struct ib_ah *ah = ERR_PTR(-EINVAL);
1342     struct rvt_qp *qp0;
1343     struct qib_pportdata *ppd = ppd_from_ibp(ibp);
1344     struct qib_devdata *dd = dd_from_ppd(ppd);
1345     u32 port_num = ppd->port;
1346 
1347     memset(&attr, 0, sizeof(attr));
1348     attr.type = rdma_ah_find_type(&dd->verbs_dev.rdi.ibdev, port_num);
1349     rdma_ah_set_dlid(&attr, dlid);
1350     rdma_ah_set_port_num(&attr, port_num);
1351     rcu_read_lock();
1352     qp0 = rcu_dereference(ibp->rvp.qp[0]);
1353     if (qp0)
1354         ah = rdma_create_ah(qp0->ibqp.pd, &attr, 0);
1355     rcu_read_unlock();
1356     return ah;
1357 }
1358 
1359 /**
1360  * qib_get_npkeys - return the size of the PKEY table for context 0
1361  * @dd: the qlogic_ib device
1362  */
1363 unsigned qib_get_npkeys(struct qib_devdata *dd)
1364 {
1365     return ARRAY_SIZE(dd->rcd[0]->pkeys);
1366 }
1367 
1368 /*
1369  * Return the indexed PKEY from the port PKEY table.
1370  * No need to validate rcd[ctxt]; the port is setup if we are here.
1371  */
1372 unsigned qib_get_pkey(struct qib_ibport *ibp, unsigned index)
1373 {
1374     struct qib_pportdata *ppd = ppd_from_ibp(ibp);
1375     struct qib_devdata *dd = ppd->dd;
1376     unsigned ctxt = ppd->hw_pidx;
1377     unsigned ret;
1378 
1379     /* dd->rcd null if mini_init or some init failures */
1380     if (!dd->rcd || index >= ARRAY_SIZE(dd->rcd[ctxt]->pkeys))
1381         ret = 0;
1382     else
1383         ret = dd->rcd[ctxt]->pkeys[index];
1384 
1385     return ret;
1386 }
1387 
1388 static void init_ibport(struct qib_pportdata *ppd)
1389 {
1390     struct qib_verbs_counters cntrs;
1391     struct qib_ibport *ibp = &ppd->ibport_data;
1392 
1393     spin_lock_init(&ibp->rvp.lock);
1394     /* Set the prefix to the default value (see ch. 4.1.1) */
1395     ibp->rvp.gid_prefix = IB_DEFAULT_GID_PREFIX;
1396     ibp->rvp.sm_lid = be16_to_cpu(IB_LID_PERMISSIVE);
1397     ibp->rvp.port_cap_flags = IB_PORT_SYS_IMAGE_GUID_SUP |
1398         IB_PORT_CLIENT_REG_SUP | IB_PORT_SL_MAP_SUP |
1399         IB_PORT_TRAP_SUP | IB_PORT_AUTO_MIGR_SUP |
1400         IB_PORT_DR_NOTICE_SUP | IB_PORT_CAP_MASK_NOTICE_SUP |
1401         IB_PORT_OTHER_LOCAL_CHANGES_SUP;
1402     if (ppd->dd->flags & QIB_HAS_LINK_LATENCY)
1403         ibp->rvp.port_cap_flags |= IB_PORT_LINK_LATENCY_SUP;
1404     ibp->rvp.pma_counter_select[0] = IB_PMA_PORT_XMIT_DATA;
1405     ibp->rvp.pma_counter_select[1] = IB_PMA_PORT_RCV_DATA;
1406     ibp->rvp.pma_counter_select[2] = IB_PMA_PORT_XMIT_PKTS;
1407     ibp->rvp.pma_counter_select[3] = IB_PMA_PORT_RCV_PKTS;
1408     ibp->rvp.pma_counter_select[4] = IB_PMA_PORT_XMIT_WAIT;
1409 
1410     /* Snapshot current HW counters to "clear" them. */
1411     qib_get_counters(ppd, &cntrs);
1412     ibp->z_symbol_error_counter = cntrs.symbol_error_counter;
1413     ibp->z_link_error_recovery_counter =
1414         cntrs.link_error_recovery_counter;
1415     ibp->z_link_downed_counter = cntrs.link_downed_counter;
1416     ibp->z_port_rcv_errors = cntrs.port_rcv_errors;
1417     ibp->z_port_rcv_remphys_errors = cntrs.port_rcv_remphys_errors;
1418     ibp->z_port_xmit_discards = cntrs.port_xmit_discards;
1419     ibp->z_port_xmit_data = cntrs.port_xmit_data;
1420     ibp->z_port_rcv_data = cntrs.port_rcv_data;
1421     ibp->z_port_xmit_packets = cntrs.port_xmit_packets;
1422     ibp->z_port_rcv_packets = cntrs.port_rcv_packets;
1423     ibp->z_local_link_integrity_errors =
1424         cntrs.local_link_integrity_errors;
1425     ibp->z_excessive_buffer_overrun_errors =
1426         cntrs.excessive_buffer_overrun_errors;
1427     ibp->z_vl15_dropped = cntrs.vl15_dropped;
1428     RCU_INIT_POINTER(ibp->rvp.qp[0], NULL);
1429     RCU_INIT_POINTER(ibp->rvp.qp[1], NULL);
1430 }
1431 
1432 /**
1433  * qib_fill_device_attr - Fill in rvt dev info device attributes.
1434  * @dd: the device data structure
1435  */
1436 static void qib_fill_device_attr(struct qib_devdata *dd)
1437 {
1438     struct rvt_dev_info *rdi = &dd->verbs_dev.rdi;
1439 
1440     memset(&rdi->dparms.props, 0, sizeof(rdi->dparms.props));
1441 
1442     rdi->dparms.props.max_pd = ib_qib_max_pds;
1443     rdi->dparms.props.max_ah = ib_qib_max_ahs;
1444     rdi->dparms.props.device_cap_flags = IB_DEVICE_BAD_PKEY_CNTR |
1445         IB_DEVICE_BAD_QKEY_CNTR | IB_DEVICE_SHUTDOWN_PORT |
1446         IB_DEVICE_SYS_IMAGE_GUID | IB_DEVICE_RC_RNR_NAK_GEN |
1447         IB_DEVICE_PORT_ACTIVE_EVENT | IB_DEVICE_SRQ_RESIZE;
1448     rdi->dparms.props.page_size_cap = PAGE_SIZE;
1449     rdi->dparms.props.vendor_id =
1450         QIB_SRC_OUI_1 << 16 | QIB_SRC_OUI_2 << 8 | QIB_SRC_OUI_3;
1451     rdi->dparms.props.vendor_part_id = dd->deviceid;
1452     rdi->dparms.props.hw_ver = dd->minrev;
1453     rdi->dparms.props.sys_image_guid = ib_qib_sys_image_guid;
1454     rdi->dparms.props.max_mr_size = ~0ULL;
1455     rdi->dparms.props.max_qp = ib_qib_max_qps;
1456     rdi->dparms.props.max_qp_wr = ib_qib_max_qp_wrs;
1457     rdi->dparms.props.max_send_sge = ib_qib_max_sges;
1458     rdi->dparms.props.max_recv_sge = ib_qib_max_sges;
1459     rdi->dparms.props.max_sge_rd = ib_qib_max_sges;
1460     rdi->dparms.props.max_cq = ib_qib_max_cqs;
1461     rdi->dparms.props.max_cqe = ib_qib_max_cqes;
1462     rdi->dparms.props.max_ah = ib_qib_max_ahs;
1463     rdi->dparms.props.max_qp_rd_atom = QIB_MAX_RDMA_ATOMIC;
1464     rdi->dparms.props.max_qp_init_rd_atom = 255;
1465     rdi->dparms.props.max_srq = ib_qib_max_srqs;
1466     rdi->dparms.props.max_srq_wr = ib_qib_max_srq_wrs;
1467     rdi->dparms.props.max_srq_sge = ib_qib_max_srq_sges;
1468     rdi->dparms.props.atomic_cap = IB_ATOMIC_GLOB;
1469     rdi->dparms.props.max_pkeys = qib_get_npkeys(dd);
1470     rdi->dparms.props.max_mcast_grp = ib_qib_max_mcast_grps;
1471     rdi->dparms.props.max_mcast_qp_attach = ib_qib_max_mcast_qp_attached;
1472     rdi->dparms.props.max_total_mcast_qp_attach =
1473                     rdi->dparms.props.max_mcast_qp_attach *
1474                     rdi->dparms.props.max_mcast_grp;
1475     /* post send table */
1476     dd->verbs_dev.rdi.post_parms = qib_post_parms;
1477 
1478     /* opcode translation table */
1479     dd->verbs_dev.rdi.wc_opcode = ib_qib_wc_opcode;
1480 }
1481 
1482 static const struct ib_device_ops qib_dev_ops = {
1483     .owner = THIS_MODULE,
1484     .driver_id = RDMA_DRIVER_QIB,
1485 
1486     .port_groups = qib_attr_port_groups,
1487     .device_group = &qib_attr_group,
1488     .modify_device = qib_modify_device,
1489     .process_mad = qib_process_mad,
1490 };
1491 
1492 /**
1493  * qib_register_ib_device - register our device with the infiniband core
1494  * @dd: the device data structure
1495  * Return the allocated qib_ibdev pointer or NULL on error.
1496  */
1497 int qib_register_ib_device(struct qib_devdata *dd)
1498 {
1499     struct qib_ibdev *dev = &dd->verbs_dev;
1500     struct ib_device *ibdev = &dev->rdi.ibdev;
1501     struct qib_pportdata *ppd = dd->pport;
1502     unsigned i, ctxt;
1503     int ret;
1504 
1505     for (i = 0; i < dd->num_pports; i++)
1506         init_ibport(ppd + i);
1507 
1508     /* Only need to initialize non-zero fields. */
1509     timer_setup(&dev->mem_timer, mem_timer, 0);
1510 
1511     INIT_LIST_HEAD(&dev->piowait);
1512     INIT_LIST_HEAD(&dev->dmawait);
1513     INIT_LIST_HEAD(&dev->txwait);
1514     INIT_LIST_HEAD(&dev->memwait);
1515     INIT_LIST_HEAD(&dev->txreq_free);
1516 
1517     if (ppd->sdma_descq_cnt) {
1518         dev->pio_hdrs = dma_alloc_coherent(&dd->pcidev->dev,
1519                         ppd->sdma_descq_cnt *
1520                         sizeof(struct qib_pio_header),
1521                         &dev->pio_hdrs_phys,
1522                         GFP_KERNEL);
1523         if (!dev->pio_hdrs) {
1524             ret = -ENOMEM;
1525             goto err_hdrs;
1526         }
1527     }
1528 
1529     for (i = 0; i < ppd->sdma_descq_cnt; i++) {
1530         struct qib_verbs_txreq *tx;
1531 
1532         tx = kzalloc(sizeof(*tx), GFP_KERNEL);
1533         if (!tx) {
1534             ret = -ENOMEM;
1535             goto err_tx;
1536         }
1537         tx->hdr_inx = i;
1538         list_add(&tx->txreq.list, &dev->txreq_free);
1539     }
1540 
1541     /*
1542      * The system image GUID is supposed to be the same for all
1543      * IB HCAs in a single system but since there can be other
1544      * device types in the system, we can't be sure this is unique.
1545      */
1546     if (!ib_qib_sys_image_guid)
1547         ib_qib_sys_image_guid = ppd->guid;
1548 
1549     ibdev->node_guid = ppd->guid;
1550     ibdev->phys_port_cnt = dd->num_pports;
1551     ibdev->dev.parent = &dd->pcidev->dev;
1552 
1553     snprintf(ibdev->node_desc, sizeof(ibdev->node_desc),
1554          "Intel Infiniband HCA %s", init_utsname()->nodename);
1555 
1556     /*
1557      * Fill in rvt info object.
1558      */
1559     dd->verbs_dev.rdi.driver_f.get_pci_dev = qib_get_pci_dev;
1560     dd->verbs_dev.rdi.driver_f.check_ah = qib_check_ah;
1561     dd->verbs_dev.rdi.driver_f.setup_wqe = qib_check_send_wqe;
1562     dd->verbs_dev.rdi.driver_f.notify_new_ah = qib_notify_new_ah;
1563     dd->verbs_dev.rdi.driver_f.alloc_qpn = qib_alloc_qpn;
1564     dd->verbs_dev.rdi.driver_f.qp_priv_alloc = qib_qp_priv_alloc;
1565     dd->verbs_dev.rdi.driver_f.qp_priv_free = qib_qp_priv_free;
1566     dd->verbs_dev.rdi.driver_f.free_all_qps = qib_free_all_qps;
1567     dd->verbs_dev.rdi.driver_f.notify_qp_reset = qib_notify_qp_reset;
1568     dd->verbs_dev.rdi.driver_f.do_send = qib_do_send;
1569     dd->verbs_dev.rdi.driver_f.schedule_send = qib_schedule_send;
1570     dd->verbs_dev.rdi.driver_f.quiesce_qp = qib_quiesce_qp;
1571     dd->verbs_dev.rdi.driver_f.stop_send_queue = qib_stop_send_queue;
1572     dd->verbs_dev.rdi.driver_f.flush_qp_waiters = qib_flush_qp_waiters;
1573     dd->verbs_dev.rdi.driver_f.notify_error_qp = qib_notify_error_qp;
1574     dd->verbs_dev.rdi.driver_f.notify_restart_rc = qib_restart_rc;
1575     dd->verbs_dev.rdi.driver_f.mtu_to_path_mtu = qib_mtu_to_path_mtu;
1576     dd->verbs_dev.rdi.driver_f.mtu_from_qp = qib_mtu_from_qp;
1577     dd->verbs_dev.rdi.driver_f.get_pmtu_from_attr = qib_get_pmtu_from_attr;
1578     dd->verbs_dev.rdi.driver_f.schedule_send_no_lock = _qib_schedule_send;
1579     dd->verbs_dev.rdi.driver_f.query_port_state = qib_query_port;
1580     dd->verbs_dev.rdi.driver_f.shut_down_port = qib_shut_down_port;
1581     dd->verbs_dev.rdi.driver_f.cap_mask_chg = qib_cap_mask_chg;
1582     dd->verbs_dev.rdi.driver_f.notify_create_mad_agent =
1583                         qib_notify_create_mad_agent;
1584     dd->verbs_dev.rdi.driver_f.notify_free_mad_agent =
1585                         qib_notify_free_mad_agent;
1586 
1587     dd->verbs_dev.rdi.dparms.max_rdma_atomic = QIB_MAX_RDMA_ATOMIC;
1588     dd->verbs_dev.rdi.driver_f.get_guid_be = qib_get_guid_be;
1589     dd->verbs_dev.rdi.dparms.lkey_table_size = qib_lkey_table_size;
1590     dd->verbs_dev.rdi.dparms.qp_table_size = ib_qib_qp_table_size;
1591     dd->verbs_dev.rdi.dparms.qpn_start = 1;
1592     dd->verbs_dev.rdi.dparms.qpn_res_start = QIB_KD_QP;
1593     dd->verbs_dev.rdi.dparms.qpn_res_end = QIB_KD_QP; /* Reserve one QP */
1594     dd->verbs_dev.rdi.dparms.qpn_inc = 1;
1595     dd->verbs_dev.rdi.dparms.qos_shift = 1;
1596     dd->verbs_dev.rdi.dparms.psn_mask = QIB_PSN_MASK;
1597     dd->verbs_dev.rdi.dparms.psn_shift = QIB_PSN_SHIFT;
1598     dd->verbs_dev.rdi.dparms.psn_modify_mask = QIB_PSN_MASK;
1599     dd->verbs_dev.rdi.dparms.nports = dd->num_pports;
1600     dd->verbs_dev.rdi.dparms.npkeys = qib_get_npkeys(dd);
1601     dd->verbs_dev.rdi.dparms.node = dd->assigned_node_id;
1602     dd->verbs_dev.rdi.dparms.core_cap_flags = RDMA_CORE_PORT_IBA_IB;
1603     dd->verbs_dev.rdi.dparms.max_mad_size = IB_MGMT_MAD_SIZE;
1604     dd->verbs_dev.rdi.dparms.sge_copy_mode = RVT_SGE_COPY_MEMCPY;
1605 
1606     qib_fill_device_attr(dd);
1607 
1608     ppd = dd->pport;
1609     for (i = 0; i < dd->num_pports; i++, ppd++) {
1610         ctxt = ppd->hw_pidx;
1611         rvt_init_port(&dd->verbs_dev.rdi,
1612                   &ppd->ibport_data.rvp,
1613                   i,
1614                   dd->rcd[ctxt]->pkeys);
1615     }
1616 
1617     ib_set_device_ops(ibdev, &qib_dev_ops);
1618     ret = rvt_register_device(&dd->verbs_dev.rdi);
1619     if (ret)
1620         goto err_tx;
1621 
1622     return ret;
1623 
1624 err_tx:
1625     while (!list_empty(&dev->txreq_free)) {
1626         struct list_head *l = dev->txreq_free.next;
1627         struct qib_verbs_txreq *tx;
1628 
1629         list_del(l);
1630         tx = list_entry(l, struct qib_verbs_txreq, txreq.list);
1631         kfree(tx);
1632     }
1633     if (ppd->sdma_descq_cnt)
1634         dma_free_coherent(&dd->pcidev->dev,
1635                   ppd->sdma_descq_cnt *
1636                     sizeof(struct qib_pio_header),
1637                   dev->pio_hdrs, dev->pio_hdrs_phys);
1638 err_hdrs:
1639     qib_dev_err(dd, "cannot register verbs: %d!\n", -ret);
1640     return ret;
1641 }
1642 
1643 void qib_unregister_ib_device(struct qib_devdata *dd)
1644 {
1645     struct qib_ibdev *dev = &dd->verbs_dev;
1646 
1647     rvt_unregister_device(&dd->verbs_dev.rdi);
1648 
1649     if (!list_empty(&dev->piowait))
1650         qib_dev_err(dd, "piowait list not empty!\n");
1651     if (!list_empty(&dev->dmawait))
1652         qib_dev_err(dd, "dmawait list not empty!\n");
1653     if (!list_empty(&dev->txwait))
1654         qib_dev_err(dd, "txwait list not empty!\n");
1655     if (!list_empty(&dev->memwait))
1656         qib_dev_err(dd, "memwait list not empty!\n");
1657 
1658     del_timer_sync(&dev->mem_timer);
1659     while (!list_empty(&dev->txreq_free)) {
1660         struct list_head *l = dev->txreq_free.next;
1661         struct qib_verbs_txreq *tx;
1662 
1663         list_del(l);
1664         tx = list_entry(l, struct qib_verbs_txreq, txreq.list);
1665         kfree(tx);
1666     }
1667     if (dd->pport->sdma_descq_cnt)
1668         dma_free_coherent(&dd->pcidev->dev,
1669                   dd->pport->sdma_descq_cnt *
1670                     sizeof(struct qib_pio_header),
1671                   dev->pio_hdrs, dev->pio_hdrs_phys);
1672 }
1673 
1674 /**
1675  * _qib_schedule_send - schedule progress
1676  * @qp: the qp
1677  *
1678  * This schedules progress w/o regard to the s_flags.
1679  *
1680  * It is only used in post send, which doesn't hold
1681  * the s_lock.
1682  */
1683 bool _qib_schedule_send(struct rvt_qp *qp)
1684 {
1685     struct qib_ibport *ibp =
1686         to_iport(qp->ibqp.device, qp->port_num);
1687     struct qib_pportdata *ppd = ppd_from_ibp(ibp);
1688     struct qib_qp_priv *priv = qp->priv;
1689 
1690     return queue_work(ppd->qib_wq, &priv->s_work);
1691 }
1692 
1693 /**
1694  * qib_schedule_send - schedule progress
1695  * @qp: the qp
1696  *
1697  * This schedules qp progress.  The s_lock
1698  * should be held.
1699  */
1700 bool qib_schedule_send(struct rvt_qp *qp)
1701 {
1702     if (qib_send_ok(qp))
1703         return _qib_schedule_send(qp);
1704     return false;
1705 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!