Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause
0002 /*
0003  * Copyright(c) 2015-2020 Intel Corporation.
0004  * Copyright(c) 2021 Cornelis Networks.
0005  */
0006 
0007 #include <linux/spinlock.h>
0008 #include <linux/pci.h>
0009 #include <linux/io.h>
0010 #include <linux/delay.h>
0011 #include <linux/netdevice.h>
0012 #include <linux/vmalloc.h>
0013 #include <linux/module.h>
0014 #include <linux/prefetch.h>
0015 #include <rdma/ib_verbs.h>
0016 #include <linux/etherdevice.h>
0017 
0018 #include "hfi.h"
0019 #include "trace.h"
0020 #include "qp.h"
0021 #include "sdma.h"
0022 #include "debugfs.h"
0023 #include "vnic.h"
0024 #include "fault.h"
0025 
0026 #include "ipoib.h"
0027 #include "netdev.h"
0028 
0029 #undef pr_fmt
0030 #define pr_fmt(fmt) DRIVER_NAME ": " fmt
0031 
0032 DEFINE_MUTEX(hfi1_mutex);   /* general driver use */
0033 
0034 unsigned int hfi1_max_mtu = HFI1_DEFAULT_MAX_MTU;
0035 module_param_named(max_mtu, hfi1_max_mtu, uint, S_IRUGO);
0036 MODULE_PARM_DESC(max_mtu, "Set max MTU bytes, default is " __stringify(
0037          HFI1_DEFAULT_MAX_MTU));
0038 
0039 unsigned int hfi1_cu = 1;
0040 module_param_named(cu, hfi1_cu, uint, S_IRUGO);
0041 MODULE_PARM_DESC(cu, "Credit return units");
0042 
0043 unsigned long hfi1_cap_mask = HFI1_CAP_MASK_DEFAULT;
0044 static int hfi1_caps_set(const char *val, const struct kernel_param *kp);
0045 static int hfi1_caps_get(char *buffer, const struct kernel_param *kp);
0046 static const struct kernel_param_ops cap_ops = {
0047     .set = hfi1_caps_set,
0048     .get = hfi1_caps_get
0049 };
0050 module_param_cb(cap_mask, &cap_ops, &hfi1_cap_mask, S_IWUSR | S_IRUGO);
0051 MODULE_PARM_DESC(cap_mask, "Bit mask of enabled/disabled HW features");
0052 
0053 MODULE_LICENSE("Dual BSD/GPL");
0054 MODULE_DESCRIPTION("Cornelis Omni-Path Express driver");
0055 
0056 /*
0057  * MAX_PKT_RCV is the max # if packets processed per receive interrupt.
0058  */
0059 #define MAX_PKT_RECV 64
0060 /*
0061  * MAX_PKT_THREAD_RCV is the max # of packets processed before
0062  * the qp_wait_list queue is flushed.
0063  */
0064 #define MAX_PKT_RECV_THREAD (MAX_PKT_RECV * 4)
0065 #define EGR_HEAD_UPDATE_THRESHOLD 16
0066 
0067 struct hfi1_ib_stats hfi1_stats;
0068 
0069 static int hfi1_caps_set(const char *val, const struct kernel_param *kp)
0070 {
0071     int ret = 0;
0072     unsigned long *cap_mask_ptr = (unsigned long *)kp->arg,
0073         cap_mask = *cap_mask_ptr, value, diff,
0074         write_mask = ((HFI1_CAP_WRITABLE_MASK << HFI1_CAP_USER_SHIFT) |
0075                   HFI1_CAP_WRITABLE_MASK);
0076 
0077     ret = kstrtoul(val, 0, &value);
0078     if (ret) {
0079         pr_warn("Invalid module parameter value for 'cap_mask'\n");
0080         goto done;
0081     }
0082     /* Get the changed bits (except the locked bit) */
0083     diff = value ^ (cap_mask & ~HFI1_CAP_LOCKED_SMASK);
0084 
0085     /* Remove any bits that are not allowed to change after driver load */
0086     if (HFI1_CAP_LOCKED() && (diff & ~write_mask)) {
0087         pr_warn("Ignoring non-writable capability bits %#lx\n",
0088             diff & ~write_mask);
0089         diff &= write_mask;
0090     }
0091 
0092     /* Mask off any reserved bits */
0093     diff &= ~HFI1_CAP_RESERVED_MASK;
0094     /* Clear any previously set and changing bits */
0095     cap_mask &= ~diff;
0096     /* Update the bits with the new capability */
0097     cap_mask |= (value & diff);
0098     /* Check for any kernel/user restrictions */
0099     diff = (cap_mask & (HFI1_CAP_MUST_HAVE_KERN << HFI1_CAP_USER_SHIFT)) ^
0100         ((cap_mask & HFI1_CAP_MUST_HAVE_KERN) << HFI1_CAP_USER_SHIFT);
0101     cap_mask &= ~diff;
0102     /* Set the bitmask to the final set */
0103     *cap_mask_ptr = cap_mask;
0104 done:
0105     return ret;
0106 }
0107 
0108 static int hfi1_caps_get(char *buffer, const struct kernel_param *kp)
0109 {
0110     unsigned long cap_mask = *(unsigned long *)kp->arg;
0111 
0112     cap_mask &= ~HFI1_CAP_LOCKED_SMASK;
0113     cap_mask |= ((cap_mask & HFI1_CAP_K2U) << HFI1_CAP_USER_SHIFT);
0114 
0115     return scnprintf(buffer, PAGE_SIZE, "0x%lx", cap_mask);
0116 }
0117 
0118 struct pci_dev *get_pci_dev(struct rvt_dev_info *rdi)
0119 {
0120     struct hfi1_ibdev *ibdev = container_of(rdi, struct hfi1_ibdev, rdi);
0121     struct hfi1_devdata *dd = container_of(ibdev,
0122                            struct hfi1_devdata, verbs_dev);
0123     return dd->pcidev;
0124 }
0125 
0126 /*
0127  * Return count of units with at least one port ACTIVE.
0128  */
0129 int hfi1_count_active_units(void)
0130 {
0131     struct hfi1_devdata *dd;
0132     struct hfi1_pportdata *ppd;
0133     unsigned long index, flags;
0134     int pidx, nunits_active = 0;
0135 
0136     xa_lock_irqsave(&hfi1_dev_table, flags);
0137     xa_for_each(&hfi1_dev_table, index, dd) {
0138         if (!(dd->flags & HFI1_PRESENT) || !dd->kregbase1)
0139             continue;
0140         for (pidx = 0; pidx < dd->num_pports; ++pidx) {
0141             ppd = dd->pport + pidx;
0142             if (ppd->lid && ppd->linkup) {
0143                 nunits_active++;
0144                 break;
0145             }
0146         }
0147     }
0148     xa_unlock_irqrestore(&hfi1_dev_table, flags);
0149     return nunits_active;
0150 }
0151 
0152 /*
0153  * Get address of eager buffer from it's index (allocated in chunks, not
0154  * contiguous).
0155  */
0156 static inline void *get_egrbuf(const struct hfi1_ctxtdata *rcd, u64 rhf,
0157                    u8 *update)
0158 {
0159     u32 idx = rhf_egr_index(rhf), offset = rhf_egr_buf_offset(rhf);
0160 
0161     *update |= !(idx & (rcd->egrbufs.threshold - 1)) && !offset;
0162     return (void *)(((u64)(rcd->egrbufs.rcvtids[idx].addr)) +
0163             (offset * RCV_BUF_BLOCK_SIZE));
0164 }
0165 
0166 static inline void *hfi1_get_header(struct hfi1_ctxtdata *rcd,
0167                     __le32 *rhf_addr)
0168 {
0169     u32 offset = rhf_hdrq_offset(rhf_to_cpu(rhf_addr));
0170 
0171     return (void *)(rhf_addr - rcd->rhf_offset + offset);
0172 }
0173 
0174 static inline struct ib_header *hfi1_get_msgheader(struct hfi1_ctxtdata *rcd,
0175                            __le32 *rhf_addr)
0176 {
0177     return (struct ib_header *)hfi1_get_header(rcd, rhf_addr);
0178 }
0179 
0180 static inline struct hfi1_16b_header
0181         *hfi1_get_16B_header(struct hfi1_ctxtdata *rcd,
0182                      __le32 *rhf_addr)
0183 {
0184     return (struct hfi1_16b_header *)hfi1_get_header(rcd, rhf_addr);
0185 }
0186 
0187 /*
0188  * Validate and encode the a given RcvArray Buffer size.
0189  * The function will check whether the given size falls within
0190  * allowed size ranges for the respective type and, optionally,
0191  * return the proper encoding.
0192  */
0193 int hfi1_rcvbuf_validate(u32 size, u8 type, u16 *encoded)
0194 {
0195     if (unlikely(!PAGE_ALIGNED(size)))
0196         return 0;
0197     if (unlikely(size < MIN_EAGER_BUFFER))
0198         return 0;
0199     if (size >
0200         (type == PT_EAGER ? MAX_EAGER_BUFFER : MAX_EXPECTED_BUFFER))
0201         return 0;
0202     if (encoded)
0203         *encoded = ilog2(size / PAGE_SIZE) + 1;
0204     return 1;
0205 }
0206 
0207 static void rcv_hdrerr(struct hfi1_ctxtdata *rcd, struct hfi1_pportdata *ppd,
0208                struct hfi1_packet *packet)
0209 {
0210     struct ib_header *rhdr = packet->hdr;
0211     u32 rte = rhf_rcv_type_err(packet->rhf);
0212     u32 mlid_base;
0213     struct hfi1_ibport *ibp = rcd_to_iport(rcd);
0214     struct hfi1_devdata *dd = ppd->dd;
0215     struct hfi1_ibdev *verbs_dev = &dd->verbs_dev;
0216     struct rvt_dev_info *rdi = &verbs_dev->rdi;
0217 
0218     if ((packet->rhf & RHF_DC_ERR) &&
0219         hfi1_dbg_fault_suppress_err(verbs_dev))
0220         return;
0221 
0222     if (packet->rhf & RHF_ICRC_ERR)
0223         return;
0224 
0225     if (packet->etype == RHF_RCV_TYPE_BYPASS) {
0226         goto drop;
0227     } else {
0228         u8 lnh = ib_get_lnh(rhdr);
0229 
0230         mlid_base = be16_to_cpu(IB_MULTICAST_LID_BASE);
0231         if (lnh == HFI1_LRH_BTH) {
0232             packet->ohdr = &rhdr->u.oth;
0233         } else if (lnh == HFI1_LRH_GRH) {
0234             packet->ohdr = &rhdr->u.l.oth;
0235             packet->grh = &rhdr->u.l.grh;
0236         } else {
0237             goto drop;
0238         }
0239     }
0240 
0241     if (packet->rhf & RHF_TID_ERR) {
0242         /* For TIDERR and RC QPs preemptively schedule a NAK */
0243         u32 tlen = rhf_pkt_len(packet->rhf); /* in bytes */
0244         u32 dlid = ib_get_dlid(rhdr);
0245         u32 qp_num;
0246 
0247         /* Sanity check packet */
0248         if (tlen < 24)
0249             goto drop;
0250 
0251         /* Check for GRH */
0252         if (packet->grh) {
0253             u32 vtf;
0254             struct ib_grh *grh = packet->grh;
0255 
0256             if (grh->next_hdr != IB_GRH_NEXT_HDR)
0257                 goto drop;
0258             vtf = be32_to_cpu(grh->version_tclass_flow);
0259             if ((vtf >> IB_GRH_VERSION_SHIFT) != IB_GRH_VERSION)
0260                 goto drop;
0261         }
0262 
0263         /* Get the destination QP number. */
0264         qp_num = ib_bth_get_qpn(packet->ohdr);
0265         if (dlid < mlid_base) {
0266             struct rvt_qp *qp;
0267             unsigned long flags;
0268 
0269             rcu_read_lock();
0270             qp = rvt_lookup_qpn(rdi, &ibp->rvp, qp_num);
0271             if (!qp) {
0272                 rcu_read_unlock();
0273                 goto drop;
0274             }
0275 
0276             /*
0277              * Handle only RC QPs - for other QP types drop error
0278              * packet.
0279              */
0280             spin_lock_irqsave(&qp->r_lock, flags);
0281 
0282             /* Check for valid receive state. */
0283             if (!(ib_rvt_state_ops[qp->state] &
0284                   RVT_PROCESS_RECV_OK)) {
0285                 ibp->rvp.n_pkt_drops++;
0286             }
0287 
0288             switch (qp->ibqp.qp_type) {
0289             case IB_QPT_RC:
0290                 hfi1_rc_hdrerr(rcd, packet, qp);
0291                 break;
0292             default:
0293                 /* For now don't handle any other QP types */
0294                 break;
0295             }
0296 
0297             spin_unlock_irqrestore(&qp->r_lock, flags);
0298             rcu_read_unlock();
0299         } /* Unicast QP */
0300     } /* Valid packet with TIDErr */
0301 
0302     /* handle "RcvTypeErr" flags */
0303     switch (rte) {
0304     case RHF_RTE_ERROR_OP_CODE_ERR:
0305     {
0306         void *ebuf = NULL;
0307         u8 opcode;
0308 
0309         if (rhf_use_egr_bfr(packet->rhf))
0310             ebuf = packet->ebuf;
0311 
0312         if (!ebuf)
0313             goto drop; /* this should never happen */
0314 
0315         opcode = ib_bth_get_opcode(packet->ohdr);
0316         if (opcode == IB_OPCODE_CNP) {
0317             /*
0318              * Only in pre-B0 h/w is the CNP_OPCODE handled
0319              * via this code path.
0320              */
0321             struct rvt_qp *qp = NULL;
0322             u32 lqpn, rqpn;
0323             u16 rlid;
0324             u8 svc_type, sl, sc5;
0325 
0326             sc5 = hfi1_9B_get_sc5(rhdr, packet->rhf);
0327             sl = ibp->sc_to_sl[sc5];
0328 
0329             lqpn = ib_bth_get_qpn(packet->ohdr);
0330             rcu_read_lock();
0331             qp = rvt_lookup_qpn(rdi, &ibp->rvp, lqpn);
0332             if (!qp) {
0333                 rcu_read_unlock();
0334                 goto drop;
0335             }
0336 
0337             switch (qp->ibqp.qp_type) {
0338             case IB_QPT_UD:
0339                 rlid = 0;
0340                 rqpn = 0;
0341                 svc_type = IB_CC_SVCTYPE_UD;
0342                 break;
0343             case IB_QPT_UC:
0344                 rlid = ib_get_slid(rhdr);
0345                 rqpn = qp->remote_qpn;
0346                 svc_type = IB_CC_SVCTYPE_UC;
0347                 break;
0348             default:
0349                 rcu_read_unlock();
0350                 goto drop;
0351             }
0352 
0353             process_becn(ppd, sl, rlid, lqpn, rqpn, svc_type);
0354             rcu_read_unlock();
0355         }
0356 
0357         packet->rhf &= ~RHF_RCV_TYPE_ERR_SMASK;
0358         break;
0359     }
0360     default:
0361         break;
0362     }
0363 
0364 drop:
0365     return;
0366 }
0367 
0368 static inline void init_packet(struct hfi1_ctxtdata *rcd,
0369                    struct hfi1_packet *packet)
0370 {
0371     packet->rsize = get_hdrqentsize(rcd); /* words */
0372     packet->maxcnt = get_hdrq_cnt(rcd) * packet->rsize; /* words */
0373     packet->rcd = rcd;
0374     packet->updegr = 0;
0375     packet->etail = -1;
0376     packet->rhf_addr = get_rhf_addr(rcd);
0377     packet->rhf = rhf_to_cpu(packet->rhf_addr);
0378     packet->rhqoff = hfi1_rcd_head(rcd);
0379     packet->numpkt = 0;
0380 }
0381 
0382 /* We support only two types - 9B and 16B for now */
0383 static const hfi1_handle_cnp hfi1_handle_cnp_tbl[2] = {
0384     [HFI1_PKT_TYPE_9B] = &return_cnp,
0385     [HFI1_PKT_TYPE_16B] = &return_cnp_16B
0386 };
0387 
0388 /**
0389  * hfi1_process_ecn_slowpath - Process FECN or BECN bits
0390  * @qp: The packet's destination QP
0391  * @pkt: The packet itself.
0392  * @prescan: Is the caller the RXQ prescan
0393  *
0394  * Process the packet's FECN or BECN bits. By now, the packet
0395  * has already been evaluated whether processing of those bit should
0396  * be done.
0397  * The significance of the @prescan argument is that if the caller
0398  * is the RXQ prescan, a CNP will be send out instead of waiting for the
0399  * normal packet processing to send an ACK with BECN set (or a CNP).
0400  */
0401 bool hfi1_process_ecn_slowpath(struct rvt_qp *qp, struct hfi1_packet *pkt,
0402                    bool prescan)
0403 {
0404     struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
0405     struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
0406     struct ib_other_headers *ohdr = pkt->ohdr;
0407     struct ib_grh *grh = pkt->grh;
0408     u32 rqpn = 0;
0409     u16 pkey;
0410     u32 rlid, slid, dlid = 0;
0411     u8 hdr_type, sc, svc_type, opcode;
0412     bool is_mcast = false, ignore_fecn = false, do_cnp = false,
0413         fecn, becn;
0414 
0415     /* can be called from prescan */
0416     if (pkt->etype == RHF_RCV_TYPE_BYPASS) {
0417         pkey = hfi1_16B_get_pkey(pkt->hdr);
0418         sc = hfi1_16B_get_sc(pkt->hdr);
0419         dlid = hfi1_16B_get_dlid(pkt->hdr);
0420         slid = hfi1_16B_get_slid(pkt->hdr);
0421         is_mcast = hfi1_is_16B_mcast(dlid);
0422         opcode = ib_bth_get_opcode(ohdr);
0423         hdr_type = HFI1_PKT_TYPE_16B;
0424         fecn = hfi1_16B_get_fecn(pkt->hdr);
0425         becn = hfi1_16B_get_becn(pkt->hdr);
0426     } else {
0427         pkey = ib_bth_get_pkey(ohdr);
0428         sc = hfi1_9B_get_sc5(pkt->hdr, pkt->rhf);
0429         dlid = qp->ibqp.qp_type != IB_QPT_UD ? ib_get_dlid(pkt->hdr) :
0430             ppd->lid;
0431         slid = ib_get_slid(pkt->hdr);
0432         is_mcast = (dlid > be16_to_cpu(IB_MULTICAST_LID_BASE)) &&
0433                (dlid != be16_to_cpu(IB_LID_PERMISSIVE));
0434         opcode = ib_bth_get_opcode(ohdr);
0435         hdr_type = HFI1_PKT_TYPE_9B;
0436         fecn = ib_bth_get_fecn(ohdr);
0437         becn = ib_bth_get_becn(ohdr);
0438     }
0439 
0440     switch (qp->ibqp.qp_type) {
0441     case IB_QPT_UD:
0442         rlid = slid;
0443         rqpn = ib_get_sqpn(pkt->ohdr);
0444         svc_type = IB_CC_SVCTYPE_UD;
0445         break;
0446     case IB_QPT_SMI:
0447     case IB_QPT_GSI:
0448         rlid = slid;
0449         rqpn = ib_get_sqpn(pkt->ohdr);
0450         svc_type = IB_CC_SVCTYPE_UD;
0451         break;
0452     case IB_QPT_UC:
0453         rlid = rdma_ah_get_dlid(&qp->remote_ah_attr);
0454         rqpn = qp->remote_qpn;
0455         svc_type = IB_CC_SVCTYPE_UC;
0456         break;
0457     case IB_QPT_RC:
0458         rlid = rdma_ah_get_dlid(&qp->remote_ah_attr);
0459         rqpn = qp->remote_qpn;
0460         svc_type = IB_CC_SVCTYPE_RC;
0461         break;
0462     default:
0463         return false;
0464     }
0465 
0466     ignore_fecn = is_mcast || (opcode == IB_OPCODE_CNP) ||
0467         (opcode == IB_OPCODE_RC_ACKNOWLEDGE);
0468     /*
0469      * ACKNOWLEDGE packets do not get a CNP but this will be
0470      * guarded by ignore_fecn above.
0471      */
0472     do_cnp = prescan ||
0473         (opcode >= IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST &&
0474          opcode <= IB_OPCODE_RC_ATOMIC_ACKNOWLEDGE) ||
0475         opcode == TID_OP(READ_RESP) ||
0476         opcode == TID_OP(ACK);
0477 
0478     /* Call appropriate CNP handler */
0479     if (!ignore_fecn && do_cnp && fecn)
0480         hfi1_handle_cnp_tbl[hdr_type](ibp, qp, rqpn, pkey,
0481                           dlid, rlid, sc, grh);
0482 
0483     if (becn) {
0484         u32 lqpn = be32_to_cpu(ohdr->bth[1]) & RVT_QPN_MASK;
0485         u8 sl = ibp->sc_to_sl[sc];
0486 
0487         process_becn(ppd, sl, rlid, lqpn, rqpn, svc_type);
0488     }
0489     return !ignore_fecn && fecn;
0490 }
0491 
0492 struct ps_mdata {
0493     struct hfi1_ctxtdata *rcd;
0494     u32 rsize;
0495     u32 maxcnt;
0496     u32 ps_head;
0497     u32 ps_tail;
0498     u32 ps_seq;
0499 };
0500 
0501 static inline void init_ps_mdata(struct ps_mdata *mdata,
0502                  struct hfi1_packet *packet)
0503 {
0504     struct hfi1_ctxtdata *rcd = packet->rcd;
0505 
0506     mdata->rcd = rcd;
0507     mdata->rsize = packet->rsize;
0508     mdata->maxcnt = packet->maxcnt;
0509     mdata->ps_head = packet->rhqoff;
0510 
0511     if (get_dma_rtail_setting(rcd)) {
0512         mdata->ps_tail = get_rcvhdrtail(rcd);
0513         if (rcd->ctxt == HFI1_CTRL_CTXT)
0514             mdata->ps_seq = hfi1_seq_cnt(rcd);
0515         else
0516             mdata->ps_seq = 0; /* not used with DMA_RTAIL */
0517     } else {
0518         mdata->ps_tail = 0; /* used only with DMA_RTAIL*/
0519         mdata->ps_seq = hfi1_seq_cnt(rcd);
0520     }
0521 }
0522 
0523 static inline int ps_done(struct ps_mdata *mdata, u64 rhf,
0524               struct hfi1_ctxtdata *rcd)
0525 {
0526     if (get_dma_rtail_setting(rcd))
0527         return mdata->ps_head == mdata->ps_tail;
0528     return mdata->ps_seq != rhf_rcv_seq(rhf);
0529 }
0530 
0531 static inline int ps_skip(struct ps_mdata *mdata, u64 rhf,
0532               struct hfi1_ctxtdata *rcd)
0533 {
0534     /*
0535      * Control context can potentially receive an invalid rhf.
0536      * Drop such packets.
0537      */
0538     if ((rcd->ctxt == HFI1_CTRL_CTXT) && (mdata->ps_head != mdata->ps_tail))
0539         return mdata->ps_seq != rhf_rcv_seq(rhf);
0540 
0541     return 0;
0542 }
0543 
0544 static inline void update_ps_mdata(struct ps_mdata *mdata,
0545                    struct hfi1_ctxtdata *rcd)
0546 {
0547     mdata->ps_head += mdata->rsize;
0548     if (mdata->ps_head >= mdata->maxcnt)
0549         mdata->ps_head = 0;
0550 
0551     /* Control context must do seq counting */
0552     if (!get_dma_rtail_setting(rcd) ||
0553         rcd->ctxt == HFI1_CTRL_CTXT)
0554         mdata->ps_seq = hfi1_seq_incr_wrap(mdata->ps_seq);
0555 }
0556 
0557 /*
0558  * prescan_rxq - search through the receive queue looking for packets
0559  * containing Excplicit Congestion Notifications (FECNs, or BECNs).
0560  * When an ECN is found, process the Congestion Notification, and toggle
0561  * it off.
0562  * This is declared as a macro to allow quick checking of the port to avoid
0563  * the overhead of a function call if not enabled.
0564  */
0565 #define prescan_rxq(rcd, packet) \
0566     do { \
0567         if (rcd->ppd->cc_prescan) \
0568             __prescan_rxq(packet); \
0569     } while (0)
0570 static void __prescan_rxq(struct hfi1_packet *packet)
0571 {
0572     struct hfi1_ctxtdata *rcd = packet->rcd;
0573     struct ps_mdata mdata;
0574 
0575     init_ps_mdata(&mdata, packet);
0576 
0577     while (1) {
0578         struct hfi1_ibport *ibp = rcd_to_iport(rcd);
0579         __le32 *rhf_addr = (__le32 *)rcd->rcvhdrq + mdata.ps_head +
0580                      packet->rcd->rhf_offset;
0581         struct rvt_qp *qp;
0582         struct ib_header *hdr;
0583         struct rvt_dev_info *rdi = &rcd->dd->verbs_dev.rdi;
0584         u64 rhf = rhf_to_cpu(rhf_addr);
0585         u32 etype = rhf_rcv_type(rhf), qpn, bth1;
0586         u8 lnh;
0587 
0588         if (ps_done(&mdata, rhf, rcd))
0589             break;
0590 
0591         if (ps_skip(&mdata, rhf, rcd))
0592             goto next;
0593 
0594         if (etype != RHF_RCV_TYPE_IB)
0595             goto next;
0596 
0597         packet->hdr = hfi1_get_msgheader(packet->rcd, rhf_addr);
0598         hdr = packet->hdr;
0599         lnh = ib_get_lnh(hdr);
0600 
0601         if (lnh == HFI1_LRH_BTH) {
0602             packet->ohdr = &hdr->u.oth;
0603             packet->grh = NULL;
0604         } else if (lnh == HFI1_LRH_GRH) {
0605             packet->ohdr = &hdr->u.l.oth;
0606             packet->grh = &hdr->u.l.grh;
0607         } else {
0608             goto next; /* just in case */
0609         }
0610 
0611         if (!hfi1_may_ecn(packet))
0612             goto next;
0613 
0614         bth1 = be32_to_cpu(packet->ohdr->bth[1]);
0615         qpn = bth1 & RVT_QPN_MASK;
0616         rcu_read_lock();
0617         qp = rvt_lookup_qpn(rdi, &ibp->rvp, qpn);
0618 
0619         if (!qp) {
0620             rcu_read_unlock();
0621             goto next;
0622         }
0623 
0624         hfi1_process_ecn_slowpath(qp, packet, true);
0625         rcu_read_unlock();
0626 
0627         /* turn off BECN, FECN */
0628         bth1 &= ~(IB_FECN_SMASK | IB_BECN_SMASK);
0629         packet->ohdr->bth[1] = cpu_to_be32(bth1);
0630 next:
0631         update_ps_mdata(&mdata, rcd);
0632     }
0633 }
0634 
0635 static void process_rcv_qp_work(struct hfi1_packet *packet)
0636 {
0637     struct rvt_qp *qp, *nqp;
0638     struct hfi1_ctxtdata *rcd = packet->rcd;
0639 
0640     /*
0641      * Iterate over all QPs waiting to respond.
0642      * The list won't change since the IRQ is only run on one CPU.
0643      */
0644     list_for_each_entry_safe(qp, nqp, &rcd->qp_wait_list, rspwait) {
0645         list_del_init(&qp->rspwait);
0646         if (qp->r_flags & RVT_R_RSP_NAK) {
0647             qp->r_flags &= ~RVT_R_RSP_NAK;
0648             packet->qp = qp;
0649             hfi1_send_rc_ack(packet, 0);
0650         }
0651         if (qp->r_flags & RVT_R_RSP_SEND) {
0652             unsigned long flags;
0653 
0654             qp->r_flags &= ~RVT_R_RSP_SEND;
0655             spin_lock_irqsave(&qp->s_lock, flags);
0656             if (ib_rvt_state_ops[qp->state] &
0657                     RVT_PROCESS_OR_FLUSH_SEND)
0658                 hfi1_schedule_send(qp);
0659             spin_unlock_irqrestore(&qp->s_lock, flags);
0660         }
0661         rvt_put_qp(qp);
0662     }
0663 }
0664 
0665 static noinline int max_packet_exceeded(struct hfi1_packet *packet, int thread)
0666 {
0667     if (thread) {
0668         if ((packet->numpkt & (MAX_PKT_RECV_THREAD - 1)) == 0)
0669             /* allow defered processing */
0670             process_rcv_qp_work(packet);
0671         cond_resched();
0672         return RCV_PKT_OK;
0673     } else {
0674         this_cpu_inc(*packet->rcd->dd->rcv_limit);
0675         return RCV_PKT_LIMIT;
0676     }
0677 }
0678 
0679 static inline int check_max_packet(struct hfi1_packet *packet, int thread)
0680 {
0681     int ret = RCV_PKT_OK;
0682 
0683     if (unlikely((packet->numpkt & (MAX_PKT_RECV - 1)) == 0))
0684         ret = max_packet_exceeded(packet, thread);
0685     return ret;
0686 }
0687 
0688 static noinline int skip_rcv_packet(struct hfi1_packet *packet, int thread)
0689 {
0690     int ret;
0691 
0692     packet->rcd->dd->ctx0_seq_drop++;
0693     /* Set up for the next packet */
0694     packet->rhqoff += packet->rsize;
0695     if (packet->rhqoff >= packet->maxcnt)
0696         packet->rhqoff = 0;
0697 
0698     packet->numpkt++;
0699     ret = check_max_packet(packet, thread);
0700 
0701     packet->rhf_addr = (__le32 *)packet->rcd->rcvhdrq + packet->rhqoff +
0702                      packet->rcd->rhf_offset;
0703     packet->rhf = rhf_to_cpu(packet->rhf_addr);
0704 
0705     return ret;
0706 }
0707 
0708 static void process_rcv_packet_napi(struct hfi1_packet *packet)
0709 {
0710     packet->etype = rhf_rcv_type(packet->rhf);
0711 
0712     /* total length */
0713     packet->tlen = rhf_pkt_len(packet->rhf); /* in bytes */
0714     /* retrieve eager buffer details */
0715     packet->etail = rhf_egr_index(packet->rhf);
0716     packet->ebuf = get_egrbuf(packet->rcd, packet->rhf,
0717                   &packet->updegr);
0718     /*
0719      * Prefetch the contents of the eager buffer.  It is
0720      * OK to send a negative length to prefetch_range().
0721      * The +2 is the size of the RHF.
0722      */
0723     prefetch_range(packet->ebuf,
0724                packet->tlen - ((packet->rcd->rcvhdrqentsize -
0725                        (rhf_hdrq_offset(packet->rhf)
0726                     + 2)) * 4));
0727 
0728     packet->rcd->rhf_rcv_function_map[packet->etype](packet);
0729     packet->numpkt++;
0730 
0731     /* Set up for the next packet */
0732     packet->rhqoff += packet->rsize;
0733     if (packet->rhqoff >= packet->maxcnt)
0734         packet->rhqoff = 0;
0735 
0736     packet->rhf_addr = (__le32 *)packet->rcd->rcvhdrq + packet->rhqoff +
0737                       packet->rcd->rhf_offset;
0738     packet->rhf = rhf_to_cpu(packet->rhf_addr);
0739 }
0740 
0741 static inline int process_rcv_packet(struct hfi1_packet *packet, int thread)
0742 {
0743     int ret;
0744 
0745     packet->etype = rhf_rcv_type(packet->rhf);
0746 
0747     /* total length */
0748     packet->tlen = rhf_pkt_len(packet->rhf); /* in bytes */
0749     /* retrieve eager buffer details */
0750     packet->ebuf = NULL;
0751     if (rhf_use_egr_bfr(packet->rhf)) {
0752         packet->etail = rhf_egr_index(packet->rhf);
0753         packet->ebuf = get_egrbuf(packet->rcd, packet->rhf,
0754                  &packet->updegr);
0755         /*
0756          * Prefetch the contents of the eager buffer.  It is
0757          * OK to send a negative length to prefetch_range().
0758          * The +2 is the size of the RHF.
0759          */
0760         prefetch_range(packet->ebuf,
0761                    packet->tlen - ((get_hdrqentsize(packet->rcd) -
0762                            (rhf_hdrq_offset(packet->rhf)
0763                         + 2)) * 4));
0764     }
0765 
0766     /*
0767      * Call a type specific handler for the packet. We
0768      * should be able to trust that etype won't be beyond
0769      * the range of valid indexes. If so something is really
0770      * wrong and we can probably just let things come
0771      * crashing down. There is no need to eat another
0772      * comparison in this performance critical code.
0773      */
0774     packet->rcd->rhf_rcv_function_map[packet->etype](packet);
0775     packet->numpkt++;
0776 
0777     /* Set up for the next packet */
0778     packet->rhqoff += packet->rsize;
0779     if (packet->rhqoff >= packet->maxcnt)
0780         packet->rhqoff = 0;
0781 
0782     ret = check_max_packet(packet, thread);
0783 
0784     packet->rhf_addr = (__le32 *)packet->rcd->rcvhdrq + packet->rhqoff +
0785                       packet->rcd->rhf_offset;
0786     packet->rhf = rhf_to_cpu(packet->rhf_addr);
0787 
0788     return ret;
0789 }
0790 
0791 static inline void process_rcv_update(int last, struct hfi1_packet *packet)
0792 {
0793     /*
0794      * Update head regs etc., every 16 packets, if not last pkt,
0795      * to help prevent rcvhdrq overflows, when many packets
0796      * are processed and queue is nearly full.
0797      * Don't request an interrupt for intermediate updates.
0798      */
0799     if (!last && !(packet->numpkt & 0xf)) {
0800         update_usrhead(packet->rcd, packet->rhqoff, packet->updegr,
0801                    packet->etail, 0, 0);
0802         packet->updegr = 0;
0803     }
0804     packet->grh = NULL;
0805 }
0806 
0807 static inline void finish_packet(struct hfi1_packet *packet)
0808 {
0809     /*
0810      * Nothing we need to free for the packet.
0811      *
0812      * The only thing we need to do is a final update and call for an
0813      * interrupt
0814      */
0815     update_usrhead(packet->rcd, hfi1_rcd_head(packet->rcd), packet->updegr,
0816                packet->etail, rcv_intr_dynamic, packet->numpkt);
0817 }
0818 
0819 /*
0820  * handle_receive_interrupt_napi_fp - receive a packet
0821  * @rcd: the context
0822  * @budget: polling budget
0823  *
0824  * Called from interrupt handler for receive interrupt.
0825  * This is the fast path interrupt handler
0826  * when executing napi soft irq environment.
0827  */
0828 int handle_receive_interrupt_napi_fp(struct hfi1_ctxtdata *rcd, int budget)
0829 {
0830     struct hfi1_packet packet;
0831 
0832     init_packet(rcd, &packet);
0833     if (last_rcv_seq(rcd, rhf_rcv_seq(packet.rhf)))
0834         goto bail;
0835 
0836     while (packet.numpkt < budget) {
0837         process_rcv_packet_napi(&packet);
0838         if (hfi1_seq_incr(rcd, rhf_rcv_seq(packet.rhf)))
0839             break;
0840 
0841         process_rcv_update(0, &packet);
0842     }
0843     hfi1_set_rcd_head(rcd, packet.rhqoff);
0844 bail:
0845     finish_packet(&packet);
0846     return packet.numpkt;
0847 }
0848 
0849 /*
0850  * Handle receive interrupts when using the no dma rtail option.
0851  */
0852 int handle_receive_interrupt_nodma_rtail(struct hfi1_ctxtdata *rcd, int thread)
0853 {
0854     int last = RCV_PKT_OK;
0855     struct hfi1_packet packet;
0856 
0857     init_packet(rcd, &packet);
0858     if (last_rcv_seq(rcd, rhf_rcv_seq(packet.rhf))) {
0859         last = RCV_PKT_DONE;
0860         goto bail;
0861     }
0862 
0863     prescan_rxq(rcd, &packet);
0864 
0865     while (last == RCV_PKT_OK) {
0866         last = process_rcv_packet(&packet, thread);
0867         if (hfi1_seq_incr(rcd, rhf_rcv_seq(packet.rhf)))
0868             last = RCV_PKT_DONE;
0869         process_rcv_update(last, &packet);
0870     }
0871     process_rcv_qp_work(&packet);
0872     hfi1_set_rcd_head(rcd, packet.rhqoff);
0873 bail:
0874     finish_packet(&packet);
0875     return last;
0876 }
0877 
0878 int handle_receive_interrupt_dma_rtail(struct hfi1_ctxtdata *rcd, int thread)
0879 {
0880     u32 hdrqtail;
0881     int last = RCV_PKT_OK;
0882     struct hfi1_packet packet;
0883 
0884     init_packet(rcd, &packet);
0885     hdrqtail = get_rcvhdrtail(rcd);
0886     if (packet.rhqoff == hdrqtail) {
0887         last = RCV_PKT_DONE;
0888         goto bail;
0889     }
0890     smp_rmb();  /* prevent speculative reads of dma'ed hdrq */
0891 
0892     prescan_rxq(rcd, &packet);
0893 
0894     while (last == RCV_PKT_OK) {
0895         last = process_rcv_packet(&packet, thread);
0896         if (packet.rhqoff == hdrqtail)
0897             last = RCV_PKT_DONE;
0898         process_rcv_update(last, &packet);
0899     }
0900     process_rcv_qp_work(&packet);
0901     hfi1_set_rcd_head(rcd, packet.rhqoff);
0902 bail:
0903     finish_packet(&packet);
0904     return last;
0905 }
0906 
0907 static void set_all_fastpath(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd)
0908 {
0909     u16 i;
0910 
0911     /*
0912      * For dynamically allocated kernel contexts (like vnic) switch
0913      * interrupt handler only for that context. Otherwise, switch
0914      * interrupt handler for all statically allocated kernel contexts.
0915      */
0916     if (rcd->ctxt >= dd->first_dyn_alloc_ctxt && !rcd->is_vnic) {
0917         hfi1_rcd_get(rcd);
0918         hfi1_set_fast(rcd);
0919         hfi1_rcd_put(rcd);
0920         return;
0921     }
0922 
0923     for (i = HFI1_CTRL_CTXT + 1; i < dd->num_rcv_contexts; i++) {
0924         rcd = hfi1_rcd_get_by_index(dd, i);
0925         if (rcd && (i < dd->first_dyn_alloc_ctxt || rcd->is_vnic))
0926             hfi1_set_fast(rcd);
0927         hfi1_rcd_put(rcd);
0928     }
0929 }
0930 
0931 void set_all_slowpath(struct hfi1_devdata *dd)
0932 {
0933     struct hfi1_ctxtdata *rcd;
0934     u16 i;
0935 
0936     /* HFI1_CTRL_CTXT must always use the slow path interrupt handler */
0937     for (i = HFI1_CTRL_CTXT + 1; i < dd->num_rcv_contexts; i++) {
0938         rcd = hfi1_rcd_get_by_index(dd, i);
0939         if (!rcd)
0940             continue;
0941         if (i < dd->first_dyn_alloc_ctxt || rcd->is_vnic)
0942             rcd->do_interrupt = rcd->slow_handler;
0943 
0944         hfi1_rcd_put(rcd);
0945     }
0946 }
0947 
0948 static bool __set_armed_to_active(struct hfi1_packet *packet)
0949 {
0950     u8 etype = rhf_rcv_type(packet->rhf);
0951     u8 sc = SC15_PACKET;
0952 
0953     if (etype == RHF_RCV_TYPE_IB) {
0954         struct ib_header *hdr = hfi1_get_msgheader(packet->rcd,
0955                                packet->rhf_addr);
0956         sc = hfi1_9B_get_sc5(hdr, packet->rhf);
0957     } else if (etype == RHF_RCV_TYPE_BYPASS) {
0958         struct hfi1_16b_header *hdr = hfi1_get_16B_header(
0959                         packet->rcd,
0960                         packet->rhf_addr);
0961         sc = hfi1_16B_get_sc(hdr);
0962     }
0963     if (sc != SC15_PACKET) {
0964         int hwstate = driver_lstate(packet->rcd->ppd);
0965         struct work_struct *lsaw =
0966                 &packet->rcd->ppd->linkstate_active_work;
0967 
0968         if (hwstate != IB_PORT_ACTIVE) {
0969             dd_dev_info(packet->rcd->dd,
0970                     "Unexpected link state %s\n",
0971                     opa_lstate_name(hwstate));
0972             return false;
0973         }
0974 
0975         queue_work(packet->rcd->ppd->link_wq, lsaw);
0976         return true;
0977     }
0978     return false;
0979 }
0980 
0981 /**
0982  * set_armed_to_active  - the fast path for armed to active
0983  * @packet: the packet structure
0984  *
0985  * Return true if packet processing needs to bail.
0986  */
0987 static bool set_armed_to_active(struct hfi1_packet *packet)
0988 {
0989     if (likely(packet->rcd->ppd->host_link_state != HLS_UP_ARMED))
0990         return false;
0991     return __set_armed_to_active(packet);
0992 }
0993 
0994 /*
0995  * handle_receive_interrupt - receive a packet
0996  * @rcd: the context
0997  *
0998  * Called from interrupt handler for errors or receive interrupt.
0999  * This is the slow path interrupt handler.
1000  */
1001 int handle_receive_interrupt(struct hfi1_ctxtdata *rcd, int thread)
1002 {
1003     struct hfi1_devdata *dd = rcd->dd;
1004     u32 hdrqtail;
1005     int needset, last = RCV_PKT_OK;
1006     struct hfi1_packet packet;
1007     int skip_pkt = 0;
1008 
1009     if (!rcd->rcvhdrq)
1010         return RCV_PKT_OK;
1011     /* Control context will always use the slow path interrupt handler */
1012     needset = (rcd->ctxt == HFI1_CTRL_CTXT) ? 0 : 1;
1013 
1014     init_packet(rcd, &packet);
1015 
1016     if (!get_dma_rtail_setting(rcd)) {
1017         if (last_rcv_seq(rcd, rhf_rcv_seq(packet.rhf))) {
1018             last = RCV_PKT_DONE;
1019             goto bail;
1020         }
1021         hdrqtail = 0;
1022     } else {
1023         hdrqtail = get_rcvhdrtail(rcd);
1024         if (packet.rhqoff == hdrqtail) {
1025             last = RCV_PKT_DONE;
1026             goto bail;
1027         }
1028         smp_rmb();  /* prevent speculative reads of dma'ed hdrq */
1029 
1030         /*
1031          * Control context can potentially receive an invalid
1032          * rhf. Drop such packets.
1033          */
1034         if (rcd->ctxt == HFI1_CTRL_CTXT)
1035             if (last_rcv_seq(rcd, rhf_rcv_seq(packet.rhf)))
1036                 skip_pkt = 1;
1037     }
1038 
1039     prescan_rxq(rcd, &packet);
1040 
1041     while (last == RCV_PKT_OK) {
1042         if (hfi1_need_drop(dd)) {
1043             /* On to the next packet */
1044             packet.rhqoff += packet.rsize;
1045             packet.rhf_addr = (__le32 *)rcd->rcvhdrq +
1046                       packet.rhqoff +
1047                       rcd->rhf_offset;
1048             packet.rhf = rhf_to_cpu(packet.rhf_addr);
1049 
1050         } else if (skip_pkt) {
1051             last = skip_rcv_packet(&packet, thread);
1052             skip_pkt = 0;
1053         } else {
1054             if (set_armed_to_active(&packet))
1055                 goto bail;
1056             last = process_rcv_packet(&packet, thread);
1057         }
1058 
1059         if (!get_dma_rtail_setting(rcd)) {
1060             if (hfi1_seq_incr(rcd, rhf_rcv_seq(packet.rhf)))
1061                 last = RCV_PKT_DONE;
1062         } else {
1063             if (packet.rhqoff == hdrqtail)
1064                 last = RCV_PKT_DONE;
1065             /*
1066              * Control context can potentially receive an invalid
1067              * rhf. Drop such packets.
1068              */
1069             if (rcd->ctxt == HFI1_CTRL_CTXT) {
1070                 bool lseq;
1071 
1072                 lseq = hfi1_seq_incr(rcd,
1073                              rhf_rcv_seq(packet.rhf));
1074                 if (!last && lseq)
1075                     skip_pkt = 1;
1076             }
1077         }
1078 
1079         if (needset) {
1080             needset = false;
1081             set_all_fastpath(dd, rcd);
1082         }
1083         process_rcv_update(last, &packet);
1084     }
1085 
1086     process_rcv_qp_work(&packet);
1087     hfi1_set_rcd_head(rcd, packet.rhqoff);
1088 
1089 bail:
1090     /*
1091      * Always write head at end, and setup rcv interrupt, even
1092      * if no packets were processed.
1093      */
1094     finish_packet(&packet);
1095     return last;
1096 }
1097 
1098 /*
1099  * handle_receive_interrupt_napi_sp - receive a packet
1100  * @rcd: the context
1101  * @budget: polling budget
1102  *
1103  * Called from interrupt handler for errors or receive interrupt.
1104  * This is the slow path interrupt handler
1105  * when executing napi soft irq environment.
1106  */
1107 int handle_receive_interrupt_napi_sp(struct hfi1_ctxtdata *rcd, int budget)
1108 {
1109     struct hfi1_devdata *dd = rcd->dd;
1110     int last = RCV_PKT_OK;
1111     bool needset = true;
1112     struct hfi1_packet packet;
1113 
1114     init_packet(rcd, &packet);
1115     if (last_rcv_seq(rcd, rhf_rcv_seq(packet.rhf)))
1116         goto bail;
1117 
1118     while (last != RCV_PKT_DONE && packet.numpkt < budget) {
1119         if (hfi1_need_drop(dd)) {
1120             /* On to the next packet */
1121             packet.rhqoff += packet.rsize;
1122             packet.rhf_addr = (__le32 *)rcd->rcvhdrq +
1123                       packet.rhqoff +
1124                       rcd->rhf_offset;
1125             packet.rhf = rhf_to_cpu(packet.rhf_addr);
1126 
1127         } else {
1128             if (set_armed_to_active(&packet))
1129                 goto bail;
1130             process_rcv_packet_napi(&packet);
1131         }
1132 
1133         if (hfi1_seq_incr(rcd, rhf_rcv_seq(packet.rhf)))
1134             last = RCV_PKT_DONE;
1135 
1136         if (needset) {
1137             needset = false;
1138             set_all_fastpath(dd, rcd);
1139         }
1140 
1141         process_rcv_update(last, &packet);
1142     }
1143 
1144     hfi1_set_rcd_head(rcd, packet.rhqoff);
1145 
1146 bail:
1147     /*
1148      * Always write head at end, and setup rcv interrupt, even
1149      * if no packets were processed.
1150      */
1151     finish_packet(&packet);
1152     return packet.numpkt;
1153 }
1154 
1155 /*
1156  * We may discover in the interrupt that the hardware link state has
1157  * changed from ARMED to ACTIVE (due to the arrival of a non-SC15 packet),
1158  * and we need to update the driver's notion of the link state.  We cannot
1159  * run set_link_state from interrupt context, so we queue this function on
1160  * a workqueue.
1161  *
1162  * We delay the regular interrupt processing until after the state changes
1163  * so that the link will be in the correct state by the time any application
1164  * we wake up attempts to send a reply to any message it received.
1165  * (Subsequent receive interrupts may possibly force the wakeup before we
1166  * update the link state.)
1167  *
1168  * The rcd is freed in hfi1_free_ctxtdata after hfi1_postinit_cleanup invokes
1169  * dd->f_cleanup(dd) to disable the interrupt handler and flush workqueues,
1170  * so we're safe from use-after-free of the rcd.
1171  */
1172 void receive_interrupt_work(struct work_struct *work)
1173 {
1174     struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
1175                           linkstate_active_work);
1176     struct hfi1_devdata *dd = ppd->dd;
1177     struct hfi1_ctxtdata *rcd;
1178     u16 i;
1179 
1180     /* Received non-SC15 packet implies neighbor_normal */
1181     ppd->neighbor_normal = 1;
1182     set_link_state(ppd, HLS_UP_ACTIVE);
1183 
1184     /*
1185      * Interrupt all statically allocated kernel contexts that could
1186      * have had an interrupt during auto activation.
1187      */
1188     for (i = HFI1_CTRL_CTXT; i < dd->first_dyn_alloc_ctxt; i++) {
1189         rcd = hfi1_rcd_get_by_index(dd, i);
1190         if (rcd)
1191             force_recv_intr(rcd);
1192         hfi1_rcd_put(rcd);
1193     }
1194 }
1195 
1196 /*
1197  * Convert a given MTU size to the on-wire MAD packet enumeration.
1198  * Return -1 if the size is invalid.
1199  */
1200 int mtu_to_enum(u32 mtu, int default_if_bad)
1201 {
1202     switch (mtu) {
1203     case     0: return OPA_MTU_0;
1204     case   256: return OPA_MTU_256;
1205     case   512: return OPA_MTU_512;
1206     case  1024: return OPA_MTU_1024;
1207     case  2048: return OPA_MTU_2048;
1208     case  4096: return OPA_MTU_4096;
1209     case  8192: return OPA_MTU_8192;
1210     case 10240: return OPA_MTU_10240;
1211     }
1212     return default_if_bad;
1213 }
1214 
1215 u16 enum_to_mtu(int mtu)
1216 {
1217     switch (mtu) {
1218     case OPA_MTU_0:     return 0;
1219     case OPA_MTU_256:   return 256;
1220     case OPA_MTU_512:   return 512;
1221     case OPA_MTU_1024:  return 1024;
1222     case OPA_MTU_2048:  return 2048;
1223     case OPA_MTU_4096:  return 4096;
1224     case OPA_MTU_8192:  return 8192;
1225     case OPA_MTU_10240: return 10240;
1226     default: return 0xffff;
1227     }
1228 }
1229 
1230 /*
1231  * set_mtu - set the MTU
1232  * @ppd: the per port data
1233  *
1234  * We can handle "any" incoming size, the issue here is whether we
1235  * need to restrict our outgoing size.  We do not deal with what happens
1236  * to programs that are already running when the size changes.
1237  */
1238 int set_mtu(struct hfi1_pportdata *ppd)
1239 {
1240     struct hfi1_devdata *dd = ppd->dd;
1241     int i, drain, ret = 0, is_up = 0;
1242 
1243     ppd->ibmtu = 0;
1244     for (i = 0; i < ppd->vls_supported; i++)
1245         if (ppd->ibmtu < dd->vld[i].mtu)
1246             ppd->ibmtu = dd->vld[i].mtu;
1247     ppd->ibmaxlen = ppd->ibmtu + lrh_max_header_bytes(ppd->dd);
1248 
1249     mutex_lock(&ppd->hls_lock);
1250     if (ppd->host_link_state == HLS_UP_INIT ||
1251         ppd->host_link_state == HLS_UP_ARMED ||
1252         ppd->host_link_state == HLS_UP_ACTIVE)
1253         is_up = 1;
1254 
1255     drain = !is_ax(dd) && is_up;
1256 
1257     if (drain)
1258         /*
1259          * MTU is specified per-VL. To ensure that no packet gets
1260          * stuck (due, e.g., to the MTU for the packet's VL being
1261          * reduced), empty the per-VL FIFOs before adjusting MTU.
1262          */
1263         ret = stop_drain_data_vls(dd);
1264 
1265     if (ret) {
1266         dd_dev_err(dd, "%s: cannot stop/drain VLs - refusing to change per-VL MTUs\n",
1267                __func__);
1268         goto err;
1269     }
1270 
1271     hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_MTU, 0);
1272 
1273     if (drain)
1274         open_fill_data_vls(dd); /* reopen all VLs */
1275 
1276 err:
1277     mutex_unlock(&ppd->hls_lock);
1278 
1279     return ret;
1280 }
1281 
1282 int hfi1_set_lid(struct hfi1_pportdata *ppd, u32 lid, u8 lmc)
1283 {
1284     struct hfi1_devdata *dd = ppd->dd;
1285 
1286     ppd->lid = lid;
1287     ppd->lmc = lmc;
1288     hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_LIDLMC, 0);
1289 
1290     dd_dev_info(dd, "port %u: got a lid: 0x%x\n", ppd->port, lid);
1291 
1292     return 0;
1293 }
1294 
1295 void shutdown_led_override(struct hfi1_pportdata *ppd)
1296 {
1297     struct hfi1_devdata *dd = ppd->dd;
1298 
1299     /*
1300      * This pairs with the memory barrier in hfi1_start_led_override to
1301      * ensure that we read the correct state of LED beaconing represented
1302      * by led_override_timer_active
1303      */
1304     smp_rmb();
1305     if (atomic_read(&ppd->led_override_timer_active)) {
1306         del_timer_sync(&ppd->led_override_timer);
1307         atomic_set(&ppd->led_override_timer_active, 0);
1308         /* Ensure the atomic_set is visible to all CPUs */
1309         smp_wmb();
1310     }
1311 
1312     /* Hand control of the LED to the DC for normal operation */
1313     write_csr(dd, DCC_CFG_LED_CNTRL, 0);
1314 }
1315 
1316 static void run_led_override(struct timer_list *t)
1317 {
1318     struct hfi1_pportdata *ppd = from_timer(ppd, t, led_override_timer);
1319     struct hfi1_devdata *dd = ppd->dd;
1320     unsigned long timeout;
1321     int phase_idx;
1322 
1323     if (!(dd->flags & HFI1_INITTED))
1324         return;
1325 
1326     phase_idx = ppd->led_override_phase & 1;
1327 
1328     setextled(dd, phase_idx);
1329 
1330     timeout = ppd->led_override_vals[phase_idx];
1331 
1332     /* Set up for next phase */
1333     ppd->led_override_phase = !ppd->led_override_phase;
1334 
1335     mod_timer(&ppd->led_override_timer, jiffies + timeout);
1336 }
1337 
1338 /*
1339  * To have the LED blink in a particular pattern, provide timeon and timeoff
1340  * in milliseconds.
1341  * To turn off custom blinking and return to normal operation, use
1342  * shutdown_led_override()
1343  */
1344 void hfi1_start_led_override(struct hfi1_pportdata *ppd, unsigned int timeon,
1345                  unsigned int timeoff)
1346 {
1347     if (!(ppd->dd->flags & HFI1_INITTED))
1348         return;
1349 
1350     /* Convert to jiffies for direct use in timer */
1351     ppd->led_override_vals[0] = msecs_to_jiffies(timeoff);
1352     ppd->led_override_vals[1] = msecs_to_jiffies(timeon);
1353 
1354     /* Arbitrarily start from LED on phase */
1355     ppd->led_override_phase = 1;
1356 
1357     /*
1358      * If the timer has not already been started, do so. Use a "quick"
1359      * timeout so the handler will be called soon to look at our request.
1360      */
1361     if (!timer_pending(&ppd->led_override_timer)) {
1362         timer_setup(&ppd->led_override_timer, run_led_override, 0);
1363         ppd->led_override_timer.expires = jiffies + 1;
1364         add_timer(&ppd->led_override_timer);
1365         atomic_set(&ppd->led_override_timer_active, 1);
1366         /* Ensure the atomic_set is visible to all CPUs */
1367         smp_wmb();
1368     }
1369 }
1370 
1371 /**
1372  * hfi1_reset_device - reset the chip if possible
1373  * @unit: the device to reset
1374  *
1375  * Whether or not reset is successful, we attempt to re-initialize the chip
1376  * (that is, much like a driver unload/reload).  We clear the INITTED flag
1377  * so that the various entry points will fail until we reinitialize.  For
1378  * now, we only allow this if no user contexts are open that use chip resources
1379  */
1380 int hfi1_reset_device(int unit)
1381 {
1382     int ret;
1383     struct hfi1_devdata *dd = hfi1_lookup(unit);
1384     struct hfi1_pportdata *ppd;
1385     int pidx;
1386 
1387     if (!dd) {
1388         ret = -ENODEV;
1389         goto bail;
1390     }
1391 
1392     dd_dev_info(dd, "Reset on unit %u requested\n", unit);
1393 
1394     if (!dd->kregbase1 || !(dd->flags & HFI1_PRESENT)) {
1395         dd_dev_info(dd,
1396                 "Invalid unit number %u or not initialized or not present\n",
1397                 unit);
1398         ret = -ENXIO;
1399         goto bail;
1400     }
1401 
1402     /* If there are any user/vnic contexts, we cannot reset */
1403     mutex_lock(&hfi1_mutex);
1404     if (dd->rcd)
1405         if (hfi1_stats.sps_ctxts) {
1406             mutex_unlock(&hfi1_mutex);
1407             ret = -EBUSY;
1408             goto bail;
1409         }
1410     mutex_unlock(&hfi1_mutex);
1411 
1412     for (pidx = 0; pidx < dd->num_pports; ++pidx) {
1413         ppd = dd->pport + pidx;
1414 
1415         shutdown_led_override(ppd);
1416     }
1417     if (dd->flags & HFI1_HAS_SEND_DMA)
1418         sdma_exit(dd);
1419 
1420     hfi1_reset_cpu_counters(dd);
1421 
1422     ret = hfi1_init(dd, 1);
1423 
1424     if (ret)
1425         dd_dev_err(dd,
1426                "Reinitialize unit %u after reset failed with %d\n",
1427                unit, ret);
1428     else
1429         dd_dev_info(dd, "Reinitialized unit %u after resetting\n",
1430                 unit);
1431 
1432 bail:
1433     return ret;
1434 }
1435 
1436 static inline void hfi1_setup_ib_header(struct hfi1_packet *packet)
1437 {
1438     packet->hdr = (struct hfi1_ib_message_header *)
1439             hfi1_get_msgheader(packet->rcd,
1440                        packet->rhf_addr);
1441     packet->hlen = (u8 *)packet->rhf_addr - (u8 *)packet->hdr;
1442 }
1443 
1444 static int hfi1_bypass_ingress_pkt_check(struct hfi1_packet *packet)
1445 {
1446     struct hfi1_pportdata *ppd = packet->rcd->ppd;
1447 
1448     /* slid and dlid cannot be 0 */
1449     if ((!packet->slid) || (!packet->dlid))
1450         return -EINVAL;
1451 
1452     /* Compare port lid with incoming packet dlid */
1453     if ((!(hfi1_is_16B_mcast(packet->dlid))) &&
1454         (packet->dlid !=
1455         opa_get_lid(be32_to_cpu(OPA_LID_PERMISSIVE), 16B))) {
1456         if ((packet->dlid & ~((1 << ppd->lmc) - 1)) != ppd->lid)
1457             return -EINVAL;
1458     }
1459 
1460     /* No multicast packets with SC15 */
1461     if ((hfi1_is_16B_mcast(packet->dlid)) && (packet->sc == 0xF))
1462         return -EINVAL;
1463 
1464     /* Packets with permissive DLID always on SC15 */
1465     if ((packet->dlid == opa_get_lid(be32_to_cpu(OPA_LID_PERMISSIVE),
1466                      16B)) &&
1467         (packet->sc != 0xF))
1468         return -EINVAL;
1469 
1470     return 0;
1471 }
1472 
1473 static int hfi1_setup_9B_packet(struct hfi1_packet *packet)
1474 {
1475     struct hfi1_ibport *ibp = rcd_to_iport(packet->rcd);
1476     struct ib_header *hdr;
1477     u8 lnh;
1478 
1479     hfi1_setup_ib_header(packet);
1480     hdr = packet->hdr;
1481 
1482     lnh = ib_get_lnh(hdr);
1483     if (lnh == HFI1_LRH_BTH) {
1484         packet->ohdr = &hdr->u.oth;
1485         packet->grh = NULL;
1486     } else if (lnh == HFI1_LRH_GRH) {
1487         u32 vtf;
1488 
1489         packet->ohdr = &hdr->u.l.oth;
1490         packet->grh = &hdr->u.l.grh;
1491         if (packet->grh->next_hdr != IB_GRH_NEXT_HDR)
1492             goto drop;
1493         vtf = be32_to_cpu(packet->grh->version_tclass_flow);
1494         if ((vtf >> IB_GRH_VERSION_SHIFT) != IB_GRH_VERSION)
1495             goto drop;
1496     } else {
1497         goto drop;
1498     }
1499 
1500     /* Query commonly used fields from packet header */
1501     packet->payload = packet->ebuf;
1502     packet->opcode = ib_bth_get_opcode(packet->ohdr);
1503     packet->slid = ib_get_slid(hdr);
1504     packet->dlid = ib_get_dlid(hdr);
1505     if (unlikely((packet->dlid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) &&
1506              (packet->dlid != be16_to_cpu(IB_LID_PERMISSIVE))))
1507         packet->dlid += opa_get_mcast_base(OPA_MCAST_NR) -
1508                 be16_to_cpu(IB_MULTICAST_LID_BASE);
1509     packet->sl = ib_get_sl(hdr);
1510     packet->sc = hfi1_9B_get_sc5(hdr, packet->rhf);
1511     packet->pad = ib_bth_get_pad(packet->ohdr);
1512     packet->extra_byte = 0;
1513     packet->pkey = ib_bth_get_pkey(packet->ohdr);
1514     packet->migrated = ib_bth_is_migration(packet->ohdr);
1515 
1516     return 0;
1517 drop:
1518     ibp->rvp.n_pkt_drops++;
1519     return -EINVAL;
1520 }
1521 
1522 static int hfi1_setup_bypass_packet(struct hfi1_packet *packet)
1523 {
1524     /*
1525      * Bypass packets have a different header/payload split
1526      * compared to an IB packet.
1527      * Current split is set such that 16 bytes of the actual
1528      * header is in the header buffer and the remining is in
1529      * the eager buffer. We chose 16 since hfi1 driver only
1530      * supports 16B bypass packets and we will be able to
1531      * receive the entire LRH with such a split.
1532      */
1533 
1534     struct hfi1_ctxtdata *rcd = packet->rcd;
1535     struct hfi1_pportdata *ppd = rcd->ppd;
1536     struct hfi1_ibport *ibp = &ppd->ibport_data;
1537     u8 l4;
1538 
1539     packet->hdr = (struct hfi1_16b_header *)
1540             hfi1_get_16B_header(packet->rcd,
1541                         packet->rhf_addr);
1542     l4 = hfi1_16B_get_l4(packet->hdr);
1543     if (l4 == OPA_16B_L4_IB_LOCAL) {
1544         packet->ohdr = packet->ebuf;
1545         packet->grh = NULL;
1546         packet->opcode = ib_bth_get_opcode(packet->ohdr);
1547         packet->pad = hfi1_16B_bth_get_pad(packet->ohdr);
1548         /* hdr_len_by_opcode already has an IB LRH factored in */
1549         packet->hlen = hdr_len_by_opcode[packet->opcode] +
1550             (LRH_16B_BYTES - LRH_9B_BYTES);
1551         packet->migrated = opa_bth_is_migration(packet->ohdr);
1552     } else if (l4 == OPA_16B_L4_IB_GLOBAL) {
1553         u32 vtf;
1554         u8 grh_len = sizeof(struct ib_grh);
1555 
1556         packet->ohdr = packet->ebuf + grh_len;
1557         packet->grh = packet->ebuf;
1558         packet->opcode = ib_bth_get_opcode(packet->ohdr);
1559         packet->pad = hfi1_16B_bth_get_pad(packet->ohdr);
1560         /* hdr_len_by_opcode already has an IB LRH factored in */
1561         packet->hlen = hdr_len_by_opcode[packet->opcode] +
1562             (LRH_16B_BYTES - LRH_9B_BYTES) + grh_len;
1563         packet->migrated = opa_bth_is_migration(packet->ohdr);
1564 
1565         if (packet->grh->next_hdr != IB_GRH_NEXT_HDR)
1566             goto drop;
1567         vtf = be32_to_cpu(packet->grh->version_tclass_flow);
1568         if ((vtf >> IB_GRH_VERSION_SHIFT) != IB_GRH_VERSION)
1569             goto drop;
1570     } else if (l4 == OPA_16B_L4_FM) {
1571         packet->mgmt = packet->ebuf;
1572         packet->ohdr = NULL;
1573         packet->grh = NULL;
1574         packet->opcode = IB_OPCODE_UD_SEND_ONLY;
1575         packet->pad = OPA_16B_L4_FM_PAD;
1576         packet->hlen = OPA_16B_L4_FM_HLEN;
1577         packet->migrated = false;
1578     } else {
1579         goto drop;
1580     }
1581 
1582     /* Query commonly used fields from packet header */
1583     packet->payload = packet->ebuf + packet->hlen - LRH_16B_BYTES;
1584     packet->slid = hfi1_16B_get_slid(packet->hdr);
1585     packet->dlid = hfi1_16B_get_dlid(packet->hdr);
1586     if (unlikely(hfi1_is_16B_mcast(packet->dlid)))
1587         packet->dlid += opa_get_mcast_base(OPA_MCAST_NR) -
1588                 opa_get_lid(opa_get_mcast_base(OPA_MCAST_NR),
1589                         16B);
1590     packet->sc = hfi1_16B_get_sc(packet->hdr);
1591     packet->sl = ibp->sc_to_sl[packet->sc];
1592     packet->extra_byte = SIZE_OF_LT;
1593     packet->pkey = hfi1_16B_get_pkey(packet->hdr);
1594 
1595     if (hfi1_bypass_ingress_pkt_check(packet))
1596         goto drop;
1597 
1598     return 0;
1599 drop:
1600     hfi1_cdbg(PKT, "%s: packet dropped\n", __func__);
1601     ibp->rvp.n_pkt_drops++;
1602     return -EINVAL;
1603 }
1604 
1605 static void show_eflags_errs(struct hfi1_packet *packet)
1606 {
1607     struct hfi1_ctxtdata *rcd = packet->rcd;
1608     u32 rte = rhf_rcv_type_err(packet->rhf);
1609 
1610     dd_dev_err(rcd->dd,
1611            "receive context %d: rhf 0x%016llx, errs [ %s%s%s%s%s%s%s] rte 0x%x\n",
1612            rcd->ctxt, packet->rhf,
1613            packet->rhf & RHF_K_HDR_LEN_ERR ? "k_hdr_len " : "",
1614            packet->rhf & RHF_DC_UNC_ERR ? "dc_unc " : "",
1615            packet->rhf & RHF_DC_ERR ? "dc " : "",
1616            packet->rhf & RHF_TID_ERR ? "tid " : "",
1617            packet->rhf & RHF_LEN_ERR ? "len " : "",
1618            packet->rhf & RHF_ECC_ERR ? "ecc " : "",
1619            packet->rhf & RHF_ICRC_ERR ? "icrc " : "",
1620            rte);
1621 }
1622 
1623 void handle_eflags(struct hfi1_packet *packet)
1624 {
1625     struct hfi1_ctxtdata *rcd = packet->rcd;
1626 
1627     rcv_hdrerr(rcd, rcd->ppd, packet);
1628     if (rhf_err_flags(packet->rhf))
1629         show_eflags_errs(packet);
1630 }
1631 
1632 static void hfi1_ipoib_ib_rcv(struct hfi1_packet *packet)
1633 {
1634     struct hfi1_ibport *ibp;
1635     struct net_device *netdev;
1636     struct hfi1_ctxtdata *rcd = packet->rcd;
1637     struct napi_struct *napi = rcd->napi;
1638     struct sk_buff *skb;
1639     struct hfi1_netdev_rxq *rxq = container_of(napi,
1640             struct hfi1_netdev_rxq, napi);
1641     u32 extra_bytes;
1642     u32 tlen, qpnum;
1643     bool do_work, do_cnp;
1644 
1645     trace_hfi1_rcvhdr(packet);
1646 
1647     hfi1_setup_ib_header(packet);
1648 
1649     packet->ohdr = &((struct ib_header *)packet->hdr)->u.oth;
1650     packet->grh = NULL;
1651 
1652     if (unlikely(rhf_err_flags(packet->rhf))) {
1653         handle_eflags(packet);
1654         return;
1655     }
1656 
1657     qpnum = ib_bth_get_qpn(packet->ohdr);
1658     netdev = hfi1_netdev_get_data(rcd->dd, qpnum);
1659     if (!netdev)
1660         goto drop_no_nd;
1661 
1662     trace_input_ibhdr(rcd->dd, packet, !!(rhf_dc_info(packet->rhf)));
1663     trace_ctxt_rsm_hist(rcd->ctxt);
1664 
1665     /* handle congestion notifications */
1666     do_work = hfi1_may_ecn(packet);
1667     if (unlikely(do_work)) {
1668         do_cnp = (packet->opcode != IB_OPCODE_CNP);
1669         (void)hfi1_process_ecn_slowpath(hfi1_ipoib_priv(netdev)->qp,
1670                          packet, do_cnp);
1671     }
1672 
1673     /*
1674      * We have split point after last byte of DETH
1675      * lets strip padding and CRC and ICRC.
1676      * tlen is whole packet len so we need to
1677      * subtract header size as well.
1678      */
1679     tlen = packet->tlen;
1680     extra_bytes = ib_bth_get_pad(packet->ohdr) + (SIZE_OF_CRC << 2) +
1681             packet->hlen;
1682     if (unlikely(tlen < extra_bytes))
1683         goto drop;
1684 
1685     tlen -= extra_bytes;
1686 
1687     skb = hfi1_ipoib_prepare_skb(rxq, tlen, packet->ebuf);
1688     if (unlikely(!skb))
1689         goto drop;
1690 
1691     dev_sw_netstats_rx_add(netdev, skb->len);
1692 
1693     skb->dev = netdev;
1694     skb->pkt_type = PACKET_HOST;
1695     netif_receive_skb(skb);
1696 
1697     return;
1698 
1699 drop:
1700     ++netdev->stats.rx_dropped;
1701 drop_no_nd:
1702     ibp = rcd_to_iport(packet->rcd);
1703     ++ibp->rvp.n_pkt_drops;
1704 }
1705 
1706 /*
1707  * The following functions are called by the interrupt handler. They are type
1708  * specific handlers for each packet type.
1709  */
1710 static void process_receive_ib(struct hfi1_packet *packet)
1711 {
1712     if (hfi1_setup_9B_packet(packet))
1713         return;
1714 
1715     if (unlikely(hfi1_dbg_should_fault_rx(packet)))
1716         return;
1717 
1718     trace_hfi1_rcvhdr(packet);
1719 
1720     if (unlikely(rhf_err_flags(packet->rhf))) {
1721         handle_eflags(packet);
1722         return;
1723     }
1724 
1725     hfi1_ib_rcv(packet);
1726 }
1727 
1728 static void process_receive_bypass(struct hfi1_packet *packet)
1729 {
1730     struct hfi1_devdata *dd = packet->rcd->dd;
1731 
1732     if (hfi1_setup_bypass_packet(packet))
1733         return;
1734 
1735     trace_hfi1_rcvhdr(packet);
1736 
1737     if (unlikely(rhf_err_flags(packet->rhf))) {
1738         handle_eflags(packet);
1739         return;
1740     }
1741 
1742     if (hfi1_16B_get_l2(packet->hdr) == 0x2) {
1743         hfi1_16B_rcv(packet);
1744     } else {
1745         dd_dev_err(dd,
1746                "Bypass packets other than 16B are not supported in normal operation. Dropping\n");
1747         incr_cntr64(&dd->sw_rcv_bypass_packet_errors);
1748         if (!(dd->err_info_rcvport.status_and_code &
1749               OPA_EI_STATUS_SMASK)) {
1750             u64 *flits = packet->ebuf;
1751 
1752             if (flits && !(packet->rhf & RHF_LEN_ERR)) {
1753                 dd->err_info_rcvport.packet_flit1 = flits[0];
1754                 dd->err_info_rcvport.packet_flit2 =
1755                     packet->tlen > sizeof(flits[0]) ?
1756                     flits[1] : 0;
1757             }
1758             dd->err_info_rcvport.status_and_code |=
1759                 (OPA_EI_STATUS_SMASK | BAD_L2_ERR);
1760         }
1761     }
1762 }
1763 
1764 static void process_receive_error(struct hfi1_packet *packet)
1765 {
1766     /* KHdrHCRCErr -- KDETH packet with a bad HCRC */
1767     if (unlikely(
1768          hfi1_dbg_fault_suppress_err(&packet->rcd->dd->verbs_dev) &&
1769          (rhf_rcv_type_err(packet->rhf) == RHF_RCV_TYPE_ERROR ||
1770           packet->rhf & RHF_DC_ERR)))
1771         return;
1772 
1773     hfi1_setup_ib_header(packet);
1774     handle_eflags(packet);
1775 
1776     if (unlikely(rhf_err_flags(packet->rhf)))
1777         dd_dev_err(packet->rcd->dd,
1778                "Unhandled error packet received. Dropping.\n");
1779 }
1780 
1781 static void kdeth_process_expected(struct hfi1_packet *packet)
1782 {
1783     hfi1_setup_9B_packet(packet);
1784     if (unlikely(hfi1_dbg_should_fault_rx(packet)))
1785         return;
1786 
1787     if (unlikely(rhf_err_flags(packet->rhf))) {
1788         struct hfi1_ctxtdata *rcd = packet->rcd;
1789 
1790         if (hfi1_handle_kdeth_eflags(rcd, rcd->ppd, packet))
1791             return;
1792     }
1793 
1794     hfi1_kdeth_expected_rcv(packet);
1795 }
1796 
1797 static void kdeth_process_eager(struct hfi1_packet *packet)
1798 {
1799     hfi1_setup_9B_packet(packet);
1800     if (unlikely(hfi1_dbg_should_fault_rx(packet)))
1801         return;
1802 
1803     trace_hfi1_rcvhdr(packet);
1804     if (unlikely(rhf_err_flags(packet->rhf))) {
1805         struct hfi1_ctxtdata *rcd = packet->rcd;
1806 
1807         show_eflags_errs(packet);
1808         if (hfi1_handle_kdeth_eflags(rcd, rcd->ppd, packet))
1809             return;
1810     }
1811 
1812     hfi1_kdeth_eager_rcv(packet);
1813 }
1814 
1815 static void process_receive_invalid(struct hfi1_packet *packet)
1816 {
1817     dd_dev_err(packet->rcd->dd, "Invalid packet type %d. Dropping\n",
1818            rhf_rcv_type(packet->rhf));
1819 }
1820 
1821 #define HFI1_RCVHDR_DUMP_MAX    5
1822 
1823 void seqfile_dump_rcd(struct seq_file *s, struct hfi1_ctxtdata *rcd)
1824 {
1825     struct hfi1_packet packet;
1826     struct ps_mdata mdata;
1827     int i;
1828 
1829     seq_printf(s, "Rcd %u: RcvHdr cnt %u entsize %u %s ctrl 0x%08llx status 0x%08llx, head %llu tail %llu  sw head %u\n",
1830            rcd->ctxt, get_hdrq_cnt(rcd), get_hdrqentsize(rcd),
1831            get_dma_rtail_setting(rcd) ?
1832            "dma_rtail" : "nodma_rtail",
1833            read_kctxt_csr(rcd->dd, rcd->ctxt, RCV_CTXT_CTRL),
1834            read_kctxt_csr(rcd->dd, rcd->ctxt, RCV_CTXT_STATUS),
1835            read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_HEAD) &
1836            RCV_HDR_HEAD_HEAD_MASK,
1837            read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL),
1838            rcd->head);
1839 
1840     init_packet(rcd, &packet);
1841     init_ps_mdata(&mdata, &packet);
1842 
1843     for (i = 0; i < HFI1_RCVHDR_DUMP_MAX; i++) {
1844         __le32 *rhf_addr = (__le32 *)rcd->rcvhdrq + mdata.ps_head +
1845                      rcd->rhf_offset;
1846         struct ib_header *hdr;
1847         u64 rhf = rhf_to_cpu(rhf_addr);
1848         u32 etype = rhf_rcv_type(rhf), qpn;
1849         u8 opcode;
1850         u32 psn;
1851         u8 lnh;
1852 
1853         if (ps_done(&mdata, rhf, rcd))
1854             break;
1855 
1856         if (ps_skip(&mdata, rhf, rcd))
1857             goto next;
1858 
1859         if (etype > RHF_RCV_TYPE_IB)
1860             goto next;
1861 
1862         packet.hdr = hfi1_get_msgheader(rcd, rhf_addr);
1863         hdr = packet.hdr;
1864 
1865         lnh = be16_to_cpu(hdr->lrh[0]) & 3;
1866 
1867         if (lnh == HFI1_LRH_BTH)
1868             packet.ohdr = &hdr->u.oth;
1869         else if (lnh == HFI1_LRH_GRH)
1870             packet.ohdr = &hdr->u.l.oth;
1871         else
1872             goto next; /* just in case */
1873 
1874         opcode = (be32_to_cpu(packet.ohdr->bth[0]) >> 24);
1875         qpn = be32_to_cpu(packet.ohdr->bth[1]) & RVT_QPN_MASK;
1876         psn = mask_psn(be32_to_cpu(packet.ohdr->bth[2]));
1877 
1878         seq_printf(s, "\tEnt %u: opcode 0x%x, qpn 0x%x, psn 0x%x\n",
1879                mdata.ps_head, opcode, qpn, psn);
1880 next:
1881         update_ps_mdata(&mdata, rcd);
1882     }
1883 }
1884 
1885 const rhf_rcv_function_ptr normal_rhf_rcv_functions[] = {
1886     [RHF_RCV_TYPE_EXPECTED] = kdeth_process_expected,
1887     [RHF_RCV_TYPE_EAGER] = kdeth_process_eager,
1888     [RHF_RCV_TYPE_IB] = process_receive_ib,
1889     [RHF_RCV_TYPE_ERROR] = process_receive_error,
1890     [RHF_RCV_TYPE_BYPASS] = process_receive_bypass,
1891     [RHF_RCV_TYPE_INVALID5] = process_receive_invalid,
1892     [RHF_RCV_TYPE_INVALID6] = process_receive_invalid,
1893     [RHF_RCV_TYPE_INVALID7] = process_receive_invalid,
1894 };
1895 
1896 const rhf_rcv_function_ptr netdev_rhf_rcv_functions[] = {
1897     [RHF_RCV_TYPE_EXPECTED] = process_receive_invalid,
1898     [RHF_RCV_TYPE_EAGER] = process_receive_invalid,
1899     [RHF_RCV_TYPE_IB] = hfi1_ipoib_ib_rcv,
1900     [RHF_RCV_TYPE_ERROR] = process_receive_error,
1901     [RHF_RCV_TYPE_BYPASS] = hfi1_vnic_bypass_rcv,
1902     [RHF_RCV_TYPE_INVALID5] = process_receive_invalid,
1903     [RHF_RCV_TYPE_INVALID6] = process_receive_invalid,
1904     [RHF_RCV_TYPE_INVALID7] = process_receive_invalid,
1905 };
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!