Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
0002 /*
0003  * Copyright(c) 2020 Intel Corporation.
0004  *
0005  */
0006 
0007 /*
0008  * This file contains HFI1 support for IPOIB SDMA functionality
0009  */
0010 
0011 #include <linux/log2.h>
0012 #include <linux/circ_buf.h>
0013 
0014 #include "sdma.h"
0015 #include "verbs.h"
0016 #include "trace_ibhdrs.h"
0017 #include "ipoib.h"
0018 #include "trace_tx.h"
0019 
0020 /* Add a convenience helper */
0021 #define CIRC_ADD(val, add, size) (((val) + (add)) & ((size) - 1))
0022 #define CIRC_NEXT(val, size) CIRC_ADD(val, 1, size)
0023 #define CIRC_PREV(val, size) CIRC_ADD(val, -1, size)
0024 
0025 struct ipoib_txparms {
0026     struct hfi1_devdata        *dd;
0027     struct rdma_ah_attr        *ah_attr;
0028     struct hfi1_ibport         *ibp;
0029     struct hfi1_ipoib_txq      *txq;
0030     union hfi1_ipoib_flow       flow;
0031     u32                         dqpn;
0032     u8                          hdr_dwords;
0033     u8                          entropy;
0034 };
0035 
0036 static struct ipoib_txreq *
0037 hfi1_txreq_from_idx(struct hfi1_ipoib_circ_buf *r, u32 idx)
0038 {
0039     return (struct ipoib_txreq *)(r->items + (idx << r->shift));
0040 }
0041 
0042 static u32 hfi1_ipoib_txreqs(const u64 sent, const u64 completed)
0043 {
0044     return sent - completed;
0045 }
0046 
0047 static u64 hfi1_ipoib_used(struct hfi1_ipoib_txq *txq)
0048 {
0049     return hfi1_ipoib_txreqs(txq->tx_ring.sent_txreqs,
0050                  txq->tx_ring.complete_txreqs);
0051 }
0052 
0053 static void hfi1_ipoib_stop_txq(struct hfi1_ipoib_txq *txq)
0054 {
0055     trace_hfi1_txq_stop(txq);
0056     if (atomic_inc_return(&txq->tx_ring.stops) == 1)
0057         netif_stop_subqueue(txq->priv->netdev, txq->q_idx);
0058 }
0059 
0060 static void hfi1_ipoib_wake_txq(struct hfi1_ipoib_txq *txq)
0061 {
0062     trace_hfi1_txq_wake(txq);
0063     if (atomic_dec_and_test(&txq->tx_ring.stops))
0064         netif_wake_subqueue(txq->priv->netdev, txq->q_idx);
0065 }
0066 
0067 static uint hfi1_ipoib_ring_hwat(struct hfi1_ipoib_txq *txq)
0068 {
0069     return min_t(uint, txq->priv->netdev->tx_queue_len,
0070              txq->tx_ring.max_items - 1);
0071 }
0072 
0073 static uint hfi1_ipoib_ring_lwat(struct hfi1_ipoib_txq *txq)
0074 {
0075     return min_t(uint, txq->priv->netdev->tx_queue_len,
0076              txq->tx_ring.max_items) >> 1;
0077 }
0078 
0079 static void hfi1_ipoib_check_queue_depth(struct hfi1_ipoib_txq *txq)
0080 {
0081     ++txq->tx_ring.sent_txreqs;
0082     if (hfi1_ipoib_used(txq) >= hfi1_ipoib_ring_hwat(txq) &&
0083         !atomic_xchg(&txq->tx_ring.ring_full, 1)) {
0084         trace_hfi1_txq_full(txq);
0085         hfi1_ipoib_stop_txq(txq);
0086     }
0087 }
0088 
0089 static void hfi1_ipoib_check_queue_stopped(struct hfi1_ipoib_txq *txq)
0090 {
0091     struct net_device *dev = txq->priv->netdev;
0092 
0093     /* If shutting down just return as queue state is irrelevant */
0094     if (unlikely(dev->reg_state != NETREG_REGISTERED))
0095         return;
0096 
0097     /*
0098      * When the queue has been drained to less than half full it will be
0099      * restarted.
0100      * The size of the txreq ring is fixed at initialization.
0101      * The tx queue len can be adjusted upward while the interface is
0102      * running.
0103      * The tx queue len can be large enough to overflow the txreq_ring.
0104      * Use the minimum of the current tx_queue_len or the rings max txreqs
0105      * to protect against ring overflow.
0106      */
0107     if (hfi1_ipoib_used(txq) < hfi1_ipoib_ring_lwat(txq) &&
0108         atomic_xchg(&txq->tx_ring.ring_full, 0)) {
0109         trace_hfi1_txq_xmit_unstopped(txq);
0110         hfi1_ipoib_wake_txq(txq);
0111     }
0112 }
0113 
0114 static void hfi1_ipoib_free_tx(struct ipoib_txreq *tx, int budget)
0115 {
0116     struct hfi1_ipoib_dev_priv *priv = tx->txq->priv;
0117 
0118     if (likely(!tx->sdma_status)) {
0119         dev_sw_netstats_tx_add(priv->netdev, 1, tx->skb->len);
0120     } else {
0121         ++priv->netdev->stats.tx_errors;
0122         dd_dev_warn(priv->dd,
0123                 "%s: Status = 0x%x pbc 0x%llx txq = %d sde = %d\n",
0124                 __func__, tx->sdma_status,
0125                 le64_to_cpu(tx->sdma_hdr->pbc), tx->txq->q_idx,
0126                 tx->txq->sde->this_idx);
0127     }
0128 
0129     napi_consume_skb(tx->skb, budget);
0130     tx->skb = NULL;
0131     sdma_txclean(priv->dd, &tx->txreq);
0132 }
0133 
0134 static void hfi1_ipoib_drain_tx_ring(struct hfi1_ipoib_txq *txq)
0135 {
0136     struct hfi1_ipoib_circ_buf *tx_ring = &txq->tx_ring;
0137     int i;
0138     struct ipoib_txreq *tx;
0139 
0140     for (i = 0; i < tx_ring->max_items; i++) {
0141         tx = hfi1_txreq_from_idx(tx_ring, i);
0142         tx->complete = 0;
0143         dev_kfree_skb_any(tx->skb);
0144         tx->skb = NULL;
0145         sdma_txclean(txq->priv->dd, &tx->txreq);
0146     }
0147     tx_ring->head = 0;
0148     tx_ring->tail = 0;
0149     tx_ring->complete_txreqs = 0;
0150     tx_ring->sent_txreqs = 0;
0151     tx_ring->avail = hfi1_ipoib_ring_hwat(txq);
0152 }
0153 
0154 static int hfi1_ipoib_poll_tx_ring(struct napi_struct *napi, int budget)
0155 {
0156     struct hfi1_ipoib_txq *txq =
0157         container_of(napi, struct hfi1_ipoib_txq, napi);
0158     struct hfi1_ipoib_circ_buf *tx_ring = &txq->tx_ring;
0159     u32 head = tx_ring->head;
0160     u32 max_tx = tx_ring->max_items;
0161     int work_done;
0162     struct ipoib_txreq *tx =  hfi1_txreq_from_idx(tx_ring, head);
0163 
0164     trace_hfi1_txq_poll(txq);
0165     for (work_done = 0; work_done < budget; work_done++) {
0166         /* See hfi1_ipoib_sdma_complete() */
0167         if (!smp_load_acquire(&tx->complete))
0168             break;
0169         tx->complete = 0;
0170         trace_hfi1_tx_produce(tx, head);
0171         hfi1_ipoib_free_tx(tx, budget);
0172         head = CIRC_NEXT(head, max_tx);
0173         tx =  hfi1_txreq_from_idx(tx_ring, head);
0174     }
0175     tx_ring->complete_txreqs += work_done;
0176 
0177     /* Finished freeing tx items so store the head value. */
0178     smp_store_release(&tx_ring->head, head);
0179 
0180     hfi1_ipoib_check_queue_stopped(txq);
0181 
0182     if (work_done < budget)
0183         napi_complete_done(napi, work_done);
0184 
0185     return work_done;
0186 }
0187 
0188 static void hfi1_ipoib_sdma_complete(struct sdma_txreq *txreq, int status)
0189 {
0190     struct ipoib_txreq *tx = container_of(txreq, struct ipoib_txreq, txreq);
0191 
0192     trace_hfi1_txq_complete(tx->txq);
0193     tx->sdma_status = status;
0194     /* see hfi1_ipoib_poll_tx_ring */
0195     smp_store_release(&tx->complete, 1);
0196     napi_schedule_irqoff(&tx->txq->napi);
0197 }
0198 
0199 static int hfi1_ipoib_build_ulp_payload(struct ipoib_txreq *tx,
0200                     struct ipoib_txparms *txp)
0201 {
0202     struct hfi1_devdata *dd = txp->dd;
0203     struct sdma_txreq *txreq = &tx->txreq;
0204     struct sk_buff *skb = tx->skb;
0205     int ret = 0;
0206     int i;
0207 
0208     if (skb_headlen(skb)) {
0209         ret = sdma_txadd_kvaddr(dd, txreq, skb->data, skb_headlen(skb));
0210         if (unlikely(ret))
0211             return ret;
0212     }
0213 
0214     for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
0215         const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
0216 
0217         ret = sdma_txadd_page(dd,
0218                       txreq,
0219                       skb_frag_page(frag),
0220                       frag->bv_offset,
0221                       skb_frag_size(frag));
0222         if (unlikely(ret))
0223             break;
0224     }
0225 
0226     return ret;
0227 }
0228 
0229 static int hfi1_ipoib_build_tx_desc(struct ipoib_txreq *tx,
0230                     struct ipoib_txparms *txp)
0231 {
0232     struct hfi1_devdata *dd = txp->dd;
0233     struct sdma_txreq *txreq = &tx->txreq;
0234     struct hfi1_sdma_header *sdma_hdr = tx->sdma_hdr;
0235     u16 pkt_bytes =
0236         sizeof(sdma_hdr->pbc) + (txp->hdr_dwords << 2) + tx->skb->len;
0237     int ret;
0238 
0239     ret = sdma_txinit(txreq, 0, pkt_bytes, hfi1_ipoib_sdma_complete);
0240     if (unlikely(ret))
0241         return ret;
0242 
0243     /* add pbc + headers */
0244     ret = sdma_txadd_kvaddr(dd,
0245                 txreq,
0246                 sdma_hdr,
0247                 sizeof(sdma_hdr->pbc) + (txp->hdr_dwords << 2));
0248     if (unlikely(ret))
0249         return ret;
0250 
0251     /* add the ulp payload */
0252     return hfi1_ipoib_build_ulp_payload(tx, txp);
0253 }
0254 
0255 static void hfi1_ipoib_build_ib_tx_headers(struct ipoib_txreq *tx,
0256                        struct ipoib_txparms *txp)
0257 {
0258     struct hfi1_ipoib_dev_priv *priv = tx->txq->priv;
0259     struct hfi1_sdma_header *sdma_hdr = tx->sdma_hdr;
0260     struct sk_buff *skb = tx->skb;
0261     struct hfi1_pportdata *ppd = ppd_from_ibp(txp->ibp);
0262     struct rdma_ah_attr *ah_attr = txp->ah_attr;
0263     struct ib_other_headers *ohdr;
0264     struct ib_grh *grh;
0265     u16 dwords;
0266     u16 slid;
0267     u16 dlid;
0268     u16 lrh0;
0269     u32 bth0;
0270     u32 sqpn = (u32)(priv->netdev->dev_addr[1] << 16 |
0271              priv->netdev->dev_addr[2] << 8 |
0272              priv->netdev->dev_addr[3]);
0273     u16 payload_dwords;
0274     u8 pad_cnt;
0275 
0276     pad_cnt = -skb->len & 3;
0277 
0278     /* Includes ICRC */
0279     payload_dwords = ((skb->len + pad_cnt) >> 2) + SIZE_OF_CRC;
0280 
0281     /* header size in dwords LRH+BTH+DETH = (8+12+8)/4. */
0282     txp->hdr_dwords = 7;
0283 
0284     if (rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH) {
0285         grh = &sdma_hdr->hdr.ibh.u.l.grh;
0286         txp->hdr_dwords +=
0287             hfi1_make_grh(txp->ibp,
0288                       grh,
0289                       rdma_ah_read_grh(ah_attr),
0290                       txp->hdr_dwords - LRH_9B_DWORDS,
0291                       payload_dwords);
0292         lrh0 = HFI1_LRH_GRH;
0293         ohdr = &sdma_hdr->hdr.ibh.u.l.oth;
0294     } else {
0295         lrh0 = HFI1_LRH_BTH;
0296         ohdr = &sdma_hdr->hdr.ibh.u.oth;
0297     }
0298 
0299     lrh0 |= (rdma_ah_get_sl(ah_attr) & 0xf) << 4;
0300     lrh0 |= (txp->flow.sc5 & 0xf) << 12;
0301 
0302     dlid = opa_get_lid(rdma_ah_get_dlid(ah_attr), 9B);
0303     if (dlid == be16_to_cpu(IB_LID_PERMISSIVE)) {
0304         slid = be16_to_cpu(IB_LID_PERMISSIVE);
0305     } else {
0306         u16 lid = (u16)ppd->lid;
0307 
0308         if (lid) {
0309             lid |= rdma_ah_get_path_bits(ah_attr) &
0310                 ((1 << ppd->lmc) - 1);
0311             slid = lid;
0312         } else {
0313             slid = be16_to_cpu(IB_LID_PERMISSIVE);
0314         }
0315     }
0316 
0317     /* Includes ICRC */
0318     dwords = txp->hdr_dwords + payload_dwords;
0319 
0320     /* Build the lrh */
0321     sdma_hdr->hdr.hdr_type = HFI1_PKT_TYPE_9B;
0322     hfi1_make_ib_hdr(&sdma_hdr->hdr.ibh, lrh0, dwords, dlid, slid);
0323 
0324     /* Build the bth */
0325     bth0 = (IB_OPCODE_UD_SEND_ONLY << 24) | (pad_cnt << 20) | priv->pkey;
0326 
0327     ohdr->bth[0] = cpu_to_be32(bth0);
0328     ohdr->bth[1] = cpu_to_be32(txp->dqpn);
0329     ohdr->bth[2] = cpu_to_be32(mask_psn((u32)txp->txq->tx_ring.sent_txreqs));
0330 
0331     /* Build the deth */
0332     ohdr->u.ud.deth[0] = cpu_to_be32(priv->qkey);
0333     ohdr->u.ud.deth[1] = cpu_to_be32((txp->entropy <<
0334                       HFI1_IPOIB_ENTROPY_SHIFT) | sqpn);
0335 
0336     /* Construct the pbc. */
0337     sdma_hdr->pbc =
0338         cpu_to_le64(create_pbc(ppd,
0339                        ib_is_sc5(txp->flow.sc5) <<
0340                                   PBC_DC_INFO_SHIFT,
0341                        0,
0342                        sc_to_vlt(priv->dd, txp->flow.sc5),
0343                        dwords - SIZE_OF_CRC +
0344                         (sizeof(sdma_hdr->pbc) >> 2)));
0345 }
0346 
0347 static struct ipoib_txreq *hfi1_ipoib_send_dma_common(struct net_device *dev,
0348                               struct sk_buff *skb,
0349                               struct ipoib_txparms *txp)
0350 {
0351     struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
0352     struct hfi1_ipoib_txq *txq = txp->txq;
0353     struct ipoib_txreq *tx;
0354     struct hfi1_ipoib_circ_buf *tx_ring = &txq->tx_ring;
0355     u32 tail = tx_ring->tail;
0356     int ret;
0357 
0358     if (unlikely(!tx_ring->avail)) {
0359         u32 head;
0360 
0361         if (hfi1_ipoib_used(txq) >= hfi1_ipoib_ring_hwat(txq))
0362             /* This shouldn't happen with a stopped queue */
0363             return ERR_PTR(-ENOMEM);
0364         /* See hfi1_ipoib_poll_tx_ring() */
0365         head = smp_load_acquire(&tx_ring->head);
0366         tx_ring->avail =
0367             min_t(u32, hfi1_ipoib_ring_hwat(txq),
0368                   CIRC_CNT(head, tail, tx_ring->max_items));
0369     } else {
0370         tx_ring->avail--;
0371     }
0372     tx = hfi1_txreq_from_idx(tx_ring, tail);
0373     trace_hfi1_txq_alloc_tx(txq);
0374 
0375     /* so that we can test if the sdma descriptors are there */
0376     tx->txreq.num_desc = 0;
0377     tx->txq = txq;
0378     tx->skb = skb;
0379     INIT_LIST_HEAD(&tx->txreq.list);
0380 
0381     hfi1_ipoib_build_ib_tx_headers(tx, txp);
0382 
0383     ret = hfi1_ipoib_build_tx_desc(tx, txp);
0384     if (likely(!ret)) {
0385         if (txq->flow.as_int != txp->flow.as_int) {
0386             txq->flow.tx_queue = txp->flow.tx_queue;
0387             txq->flow.sc5 = txp->flow.sc5;
0388             txq->sde =
0389                 sdma_select_engine_sc(priv->dd,
0390                               txp->flow.tx_queue,
0391                               txp->flow.sc5);
0392             trace_hfi1_flow_switch(txq);
0393         }
0394 
0395         return tx;
0396     }
0397 
0398     sdma_txclean(priv->dd, &tx->txreq);
0399 
0400     return ERR_PTR(ret);
0401 }
0402 
0403 static int hfi1_ipoib_submit_tx_list(struct net_device *dev,
0404                      struct hfi1_ipoib_txq *txq)
0405 {
0406     int ret;
0407     u16 count_out;
0408 
0409     ret = sdma_send_txlist(txq->sde,
0410                    iowait_get_ib_work(&txq->wait),
0411                    &txq->tx_list,
0412                    &count_out);
0413     if (likely(!ret) || ret == -EBUSY || ret == -ECOMM)
0414         return ret;
0415 
0416     dd_dev_warn(txq->priv->dd, "cannot send skb tx list, err %d.\n", ret);
0417 
0418     return ret;
0419 }
0420 
0421 static int hfi1_ipoib_flush_tx_list(struct net_device *dev,
0422                     struct hfi1_ipoib_txq *txq)
0423 {
0424     int ret = 0;
0425 
0426     if (!list_empty(&txq->tx_list)) {
0427         /* Flush the current list */
0428         ret = hfi1_ipoib_submit_tx_list(dev, txq);
0429 
0430         if (unlikely(ret))
0431             if (ret != -EBUSY)
0432                 ++dev->stats.tx_carrier_errors;
0433     }
0434 
0435     return ret;
0436 }
0437 
0438 static int hfi1_ipoib_submit_tx(struct hfi1_ipoib_txq *txq,
0439                 struct ipoib_txreq *tx)
0440 {
0441     int ret;
0442 
0443     ret = sdma_send_txreq(txq->sde,
0444                   iowait_get_ib_work(&txq->wait),
0445                   &tx->txreq,
0446                   txq->pkts_sent);
0447     if (likely(!ret)) {
0448         txq->pkts_sent = true;
0449         iowait_starve_clear(txq->pkts_sent, &txq->wait);
0450     }
0451 
0452     return ret;
0453 }
0454 
0455 static int hfi1_ipoib_send_dma_single(struct net_device *dev,
0456                       struct sk_buff *skb,
0457                       struct ipoib_txparms *txp)
0458 {
0459     struct hfi1_ipoib_txq *txq = txp->txq;
0460     struct hfi1_ipoib_circ_buf *tx_ring;
0461     struct ipoib_txreq *tx;
0462     int ret;
0463 
0464     tx = hfi1_ipoib_send_dma_common(dev, skb, txp);
0465     if (IS_ERR(tx)) {
0466         int ret = PTR_ERR(tx);
0467 
0468         dev_kfree_skb_any(skb);
0469 
0470         if (ret == -ENOMEM)
0471             ++dev->stats.tx_errors;
0472         else
0473             ++dev->stats.tx_carrier_errors;
0474 
0475         return NETDEV_TX_OK;
0476     }
0477 
0478     tx_ring = &txq->tx_ring;
0479     trace_hfi1_tx_consume(tx, tx_ring->tail);
0480     /* consume tx */
0481     smp_store_release(&tx_ring->tail, CIRC_NEXT(tx_ring->tail, tx_ring->max_items));
0482     ret = hfi1_ipoib_submit_tx(txq, tx);
0483     if (likely(!ret)) {
0484 tx_ok:
0485         trace_sdma_output_ibhdr(txq->priv->dd,
0486                     &tx->sdma_hdr->hdr,
0487                     ib_is_sc5(txp->flow.sc5));
0488         hfi1_ipoib_check_queue_depth(txq);
0489         return NETDEV_TX_OK;
0490     }
0491 
0492     txq->pkts_sent = false;
0493 
0494     if (ret == -EBUSY || ret == -ECOMM)
0495         goto tx_ok;
0496 
0497     /* mark complete and kick napi tx */
0498     smp_store_release(&tx->complete, 1);
0499     napi_schedule(&tx->txq->napi);
0500 
0501     ++dev->stats.tx_carrier_errors;
0502 
0503     return NETDEV_TX_OK;
0504 }
0505 
0506 static int hfi1_ipoib_send_dma_list(struct net_device *dev,
0507                     struct sk_buff *skb,
0508                     struct ipoib_txparms *txp)
0509 {
0510     struct hfi1_ipoib_txq *txq = txp->txq;
0511     struct hfi1_ipoib_circ_buf *tx_ring;
0512     struct ipoib_txreq *tx;
0513 
0514     /* Has the flow change ? */
0515     if (txq->flow.as_int != txp->flow.as_int) {
0516         int ret;
0517 
0518         trace_hfi1_flow_flush(txq);
0519         ret = hfi1_ipoib_flush_tx_list(dev, txq);
0520         if (unlikely(ret)) {
0521             if (ret == -EBUSY)
0522                 ++dev->stats.tx_dropped;
0523             dev_kfree_skb_any(skb);
0524             return NETDEV_TX_OK;
0525         }
0526     }
0527     tx = hfi1_ipoib_send_dma_common(dev, skb, txp);
0528     if (IS_ERR(tx)) {
0529         int ret = PTR_ERR(tx);
0530 
0531         dev_kfree_skb_any(skb);
0532 
0533         if (ret == -ENOMEM)
0534             ++dev->stats.tx_errors;
0535         else
0536             ++dev->stats.tx_carrier_errors;
0537 
0538         return NETDEV_TX_OK;
0539     }
0540 
0541     tx_ring = &txq->tx_ring;
0542     trace_hfi1_tx_consume(tx, tx_ring->tail);
0543     /* consume tx */
0544     smp_store_release(&tx_ring->tail, CIRC_NEXT(tx_ring->tail, tx_ring->max_items));
0545     list_add_tail(&tx->txreq.list, &txq->tx_list);
0546 
0547     hfi1_ipoib_check_queue_depth(txq);
0548 
0549     trace_sdma_output_ibhdr(txq->priv->dd,
0550                 &tx->sdma_hdr->hdr,
0551                 ib_is_sc5(txp->flow.sc5));
0552 
0553     if (!netdev_xmit_more())
0554         (void)hfi1_ipoib_flush_tx_list(dev, txq);
0555 
0556     return NETDEV_TX_OK;
0557 }
0558 
0559 static u8 hfi1_ipoib_calc_entropy(struct sk_buff *skb)
0560 {
0561     if (skb_transport_header_was_set(skb)) {
0562         u8 *hdr = (u8 *)skb_transport_header(skb);
0563 
0564         return (hdr[0] ^ hdr[1] ^ hdr[2] ^ hdr[3]);
0565     }
0566 
0567     return (u8)skb_get_queue_mapping(skb);
0568 }
0569 
0570 int hfi1_ipoib_send(struct net_device *dev,
0571             struct sk_buff *skb,
0572             struct ib_ah *address,
0573             u32 dqpn)
0574 {
0575     struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
0576     struct ipoib_txparms txp;
0577     struct rdma_netdev *rn = netdev_priv(dev);
0578 
0579     if (unlikely(skb->len > rn->mtu + HFI1_IPOIB_ENCAP_LEN)) {
0580         dd_dev_warn(priv->dd, "packet len %d (> %d) too long to send, dropping\n",
0581                 skb->len,
0582                 rn->mtu + HFI1_IPOIB_ENCAP_LEN);
0583         ++dev->stats.tx_dropped;
0584         ++dev->stats.tx_errors;
0585         dev_kfree_skb_any(skb);
0586         return NETDEV_TX_OK;
0587     }
0588 
0589     txp.dd = priv->dd;
0590     txp.ah_attr = &ibah_to_rvtah(address)->attr;
0591     txp.ibp = to_iport(priv->device, priv->port_num);
0592     txp.txq = &priv->txqs[skb_get_queue_mapping(skb)];
0593     txp.dqpn = dqpn;
0594     txp.flow.sc5 = txp.ibp->sl_to_sc[rdma_ah_get_sl(txp.ah_attr)];
0595     txp.flow.tx_queue = (u8)skb_get_queue_mapping(skb);
0596     txp.entropy = hfi1_ipoib_calc_entropy(skb);
0597 
0598     if (netdev_xmit_more() || !list_empty(&txp.txq->tx_list))
0599         return hfi1_ipoib_send_dma_list(dev, skb, &txp);
0600 
0601     return hfi1_ipoib_send_dma_single(dev, skb,  &txp);
0602 }
0603 
0604 /*
0605  * hfi1_ipoib_sdma_sleep - ipoib sdma sleep function
0606  *
0607  * This function gets called from sdma_send_txreq() when there are not enough
0608  * sdma descriptors available to send the packet. It adds Tx queue's wait
0609  * structure to sdma engine's dmawait list to be woken up when descriptors
0610  * become available.
0611  */
0612 static int hfi1_ipoib_sdma_sleep(struct sdma_engine *sde,
0613                  struct iowait_work *wait,
0614                  struct sdma_txreq *txreq,
0615                  uint seq,
0616                  bool pkts_sent)
0617 {
0618     struct hfi1_ipoib_txq *txq =
0619         container_of(wait->iow, struct hfi1_ipoib_txq, wait);
0620 
0621     write_seqlock(&sde->waitlock);
0622 
0623     if (likely(txq->priv->netdev->reg_state == NETREG_REGISTERED)) {
0624         if (sdma_progress(sde, seq, txreq)) {
0625             write_sequnlock(&sde->waitlock);
0626             return -EAGAIN;
0627         }
0628 
0629         if (list_empty(&txreq->list))
0630             /* came from non-list submit */
0631             list_add_tail(&txreq->list, &txq->tx_list);
0632         if (list_empty(&txq->wait.list)) {
0633             struct hfi1_ibport *ibp = &sde->ppd->ibport_data;
0634 
0635             if (!atomic_xchg(&txq->tx_ring.no_desc, 1)) {
0636                 trace_hfi1_txq_queued(txq);
0637                 hfi1_ipoib_stop_txq(txq);
0638             }
0639             ibp->rvp.n_dmawait++;
0640             iowait_queue(pkts_sent, wait->iow, &sde->dmawait);
0641         }
0642 
0643         write_sequnlock(&sde->waitlock);
0644         return -EBUSY;
0645     }
0646 
0647     write_sequnlock(&sde->waitlock);
0648     return -EINVAL;
0649 }
0650 
0651 /*
0652  * hfi1_ipoib_sdma_wakeup - ipoib sdma wakeup function
0653  *
0654  * This function gets called when SDMA descriptors becomes available and Tx
0655  * queue's wait structure was previously added to sdma engine's dmawait list.
0656  */
0657 static void hfi1_ipoib_sdma_wakeup(struct iowait *wait, int reason)
0658 {
0659     struct hfi1_ipoib_txq *txq =
0660         container_of(wait, struct hfi1_ipoib_txq, wait);
0661 
0662     trace_hfi1_txq_wakeup(txq);
0663     if (likely(txq->priv->netdev->reg_state == NETREG_REGISTERED))
0664         iowait_schedule(wait, system_highpri_wq, WORK_CPU_UNBOUND);
0665 }
0666 
0667 static void hfi1_ipoib_flush_txq(struct work_struct *work)
0668 {
0669     struct iowait_work *ioww =
0670         container_of(work, struct iowait_work, iowork);
0671     struct iowait *wait = iowait_ioww_to_iow(ioww);
0672     struct hfi1_ipoib_txq *txq =
0673         container_of(wait, struct hfi1_ipoib_txq, wait);
0674     struct net_device *dev = txq->priv->netdev;
0675 
0676     if (likely(dev->reg_state == NETREG_REGISTERED) &&
0677         likely(!hfi1_ipoib_flush_tx_list(dev, txq)))
0678         if (atomic_xchg(&txq->tx_ring.no_desc, 0))
0679             hfi1_ipoib_wake_txq(txq);
0680 }
0681 
0682 int hfi1_ipoib_txreq_init(struct hfi1_ipoib_dev_priv *priv)
0683 {
0684     struct net_device *dev = priv->netdev;
0685     u32 tx_ring_size, tx_item_size;
0686     struct hfi1_ipoib_circ_buf *tx_ring;
0687     int i, j;
0688 
0689     /*
0690      * Ring holds 1 less than tx_ring_size
0691      * Round up to next power of 2 in order to hold at least tx_queue_len
0692      */
0693     tx_ring_size = roundup_pow_of_two(dev->tx_queue_len + 1);
0694     tx_item_size = roundup_pow_of_two(sizeof(struct ipoib_txreq));
0695 
0696     priv->txqs = kcalloc_node(dev->num_tx_queues,
0697                   sizeof(struct hfi1_ipoib_txq),
0698                   GFP_KERNEL,
0699                   priv->dd->node);
0700     if (!priv->txqs)
0701         return -ENOMEM;
0702 
0703     for (i = 0; i < dev->num_tx_queues; i++) {
0704         struct hfi1_ipoib_txq *txq = &priv->txqs[i];
0705         struct ipoib_txreq *tx;
0706 
0707         tx_ring = &txq->tx_ring;
0708         iowait_init(&txq->wait,
0709                 0,
0710                 hfi1_ipoib_flush_txq,
0711                 NULL,
0712                 hfi1_ipoib_sdma_sleep,
0713                 hfi1_ipoib_sdma_wakeup,
0714                 NULL,
0715                 NULL);
0716         txq->priv = priv;
0717         txq->sde = NULL;
0718         INIT_LIST_HEAD(&txq->tx_list);
0719         atomic_set(&txq->tx_ring.stops, 0);
0720         atomic_set(&txq->tx_ring.ring_full, 0);
0721         atomic_set(&txq->tx_ring.no_desc, 0);
0722         txq->q_idx = i;
0723         txq->flow.tx_queue = 0xff;
0724         txq->flow.sc5 = 0xff;
0725         txq->pkts_sent = false;
0726 
0727         netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i),
0728                          priv->dd->node);
0729 
0730         txq->tx_ring.items =
0731             kvzalloc_node(array_size(tx_ring_size, tx_item_size),
0732                       GFP_KERNEL, priv->dd->node);
0733         if (!txq->tx_ring.items)
0734             goto free_txqs;
0735 
0736         txq->tx_ring.max_items = tx_ring_size;
0737         txq->tx_ring.shift = ilog2(tx_item_size);
0738         txq->tx_ring.avail = hfi1_ipoib_ring_hwat(txq);
0739         tx_ring = &txq->tx_ring;
0740         for (j = 0; j < tx_ring_size; j++)
0741             hfi1_txreq_from_idx(tx_ring, j)->sdma_hdr =
0742                 kzalloc_node(sizeof(*tx->sdma_hdr),
0743                          GFP_KERNEL, priv->dd->node);
0744 
0745         netif_napi_add_tx(dev, &txq->napi, hfi1_ipoib_poll_tx_ring);
0746     }
0747 
0748     return 0;
0749 
0750 free_txqs:
0751     for (i--; i >= 0; i--) {
0752         struct hfi1_ipoib_txq *txq = &priv->txqs[i];
0753 
0754         netif_napi_del(&txq->napi);
0755         tx_ring = &txq->tx_ring;
0756         for (j = 0; j < tx_ring_size; j++)
0757             kfree(hfi1_txreq_from_idx(tx_ring, j)->sdma_hdr);
0758         kvfree(tx_ring->items);
0759     }
0760 
0761     kfree(priv->txqs);
0762     priv->txqs = NULL;
0763     return -ENOMEM;
0764 }
0765 
0766 static void hfi1_ipoib_drain_tx_list(struct hfi1_ipoib_txq *txq)
0767 {
0768     struct sdma_txreq *txreq;
0769     struct sdma_txreq *txreq_tmp;
0770 
0771     list_for_each_entry_safe(txreq, txreq_tmp, &txq->tx_list, list) {
0772         struct ipoib_txreq *tx =
0773             container_of(txreq, struct ipoib_txreq, txreq);
0774 
0775         list_del(&txreq->list);
0776         sdma_txclean(txq->priv->dd, &tx->txreq);
0777         dev_kfree_skb_any(tx->skb);
0778         tx->skb = NULL;
0779         txq->tx_ring.complete_txreqs++;
0780     }
0781 
0782     if (hfi1_ipoib_used(txq))
0783         dd_dev_warn(txq->priv->dd,
0784                 "txq %d not empty found %u requests\n",
0785                 txq->q_idx,
0786                 hfi1_ipoib_txreqs(txq->tx_ring.sent_txreqs,
0787                           txq->tx_ring.complete_txreqs));
0788 }
0789 
0790 void hfi1_ipoib_txreq_deinit(struct hfi1_ipoib_dev_priv *priv)
0791 {
0792     int i, j;
0793 
0794     for (i = 0; i < priv->netdev->num_tx_queues; i++) {
0795         struct hfi1_ipoib_txq *txq = &priv->txqs[i];
0796         struct hfi1_ipoib_circ_buf *tx_ring = &txq->tx_ring;
0797 
0798         iowait_cancel_work(&txq->wait);
0799         iowait_sdma_drain(&txq->wait);
0800         hfi1_ipoib_drain_tx_list(txq);
0801         netif_napi_del(&txq->napi);
0802         hfi1_ipoib_drain_tx_ring(txq);
0803         for (j = 0; j < tx_ring->max_items; j++)
0804             kfree(hfi1_txreq_from_idx(tx_ring, j)->sdma_hdr);
0805         kvfree(tx_ring->items);
0806     }
0807 
0808     kfree(priv->txqs);
0809     priv->txqs = NULL;
0810 }
0811 
0812 void hfi1_ipoib_napi_tx_enable(struct net_device *dev)
0813 {
0814     struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
0815     int i;
0816 
0817     for (i = 0; i < dev->num_tx_queues; i++) {
0818         struct hfi1_ipoib_txq *txq = &priv->txqs[i];
0819 
0820         napi_enable(&txq->napi);
0821     }
0822 }
0823 
0824 void hfi1_ipoib_napi_tx_disable(struct net_device *dev)
0825 {
0826     struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
0827     int i;
0828 
0829     for (i = 0; i < dev->num_tx_queues; i++) {
0830         struct hfi1_ipoib_txq *txq = &priv->txqs[i];
0831 
0832         napi_disable(&txq->napi);
0833         hfi1_ipoib_drain_tx_ring(txq);
0834     }
0835 }
0836 
0837 void hfi1_ipoib_tx_timeout(struct net_device *dev, unsigned int q)
0838 {
0839     struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
0840     struct hfi1_ipoib_txq *txq = &priv->txqs[q];
0841 
0842     dd_dev_info(priv->dd, "timeout txq %p q %u stopped %u stops %d no_desc %d ring_full %d\n",
0843             txq, q,
0844             __netif_subqueue_stopped(dev, txq->q_idx),
0845             atomic_read(&txq->tx_ring.stops),
0846             atomic_read(&txq->tx_ring.no_desc),
0847             atomic_read(&txq->tx_ring.ring_full));
0848     dd_dev_info(priv->dd, "sde %p engine %u\n",
0849             txq->sde,
0850             txq->sde ? txq->sde->this_idx : 0);
0851     dd_dev_info(priv->dd, "flow %x\n", txq->flow.as_int);
0852     dd_dev_info(priv->dd, "sent %llu completed %llu used %llu\n",
0853             txq->tx_ring.sent_txreqs, txq->tx_ring.complete_txreqs,
0854             hfi1_ipoib_used(txq));
0855     dd_dev_info(priv->dd, "tx_queue_len %u max_items %u\n",
0856             dev->tx_queue_len, txq->tx_ring.max_items);
0857     dd_dev_info(priv->dd, "head %u tail %u\n",
0858             txq->tx_ring.head, txq->tx_ring.tail);
0859     dd_dev_info(priv->dd, "wait queued %u\n",
0860             !list_empty(&txq->wait.list));
0861     dd_dev_info(priv->dd, "tx_list empty %u\n",
0862             list_empty(&txq->tx_list));
0863 }
0864
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!