hw/hfi1/iowait.h

0001 /* SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause */
0002 /*
0003  * Copyright(c) 2015 - 2018 Intel Corporation.
0004  */
0005
0006 #ifndef _HFI1_IOWAIT_H
0007 #define _HFI1_IOWAIT_H
0008
0009 #include <linux/list.h>
0010 #include <linux/workqueue.h>
0011 #include <linux/wait.h>
0012 #include <linux/sched.h>
0013
0014 #include "sdma_txreq.h"
0015
0016 /*
0017  * typedef (*restart_t)() - restart callback
0018  * @work: pointer to work structure
0019  */
0020 typedef void (*restart_t)(struct work_struct *work);
0021
0022 #define IOWAIT_PENDING_IB  0x0
0023 #define IOWAIT_PENDING_TID 0x1
0024
0025 /*
0026  * A QP can have multiple Send Engines (SEs).
0027  *
0028  * The current use case is for supporting a TID RDMA
0029  * packet build/xmit mechanism independent from verbs.
0030  */
0031 #define IOWAIT_SES 2
0032 #define IOWAIT_IB_SE 0
0033 #define IOWAIT_TID_SE 1
0034
0035 struct sdma_txreq;
0036 struct sdma_engine;
0037 /**
0038  * @iowork: the work struct
0039  * @tx_head: list of prebuilt packets
0040  * @iow: the parent iowait structure
0041  *
0042  * This structure is the work item (process) specific
0043  * details associated with the each of the two SEs of the
0044  * QP.
0045  *
0046  * The workstruct and the queued TXs are unique to each
0047  * SE.
0048  */
0049 struct iowait;
0050 struct iowait_work {
0051     struct work_struct iowork;
0052     struct list_head tx_head;
0053     struct iowait *iow;
0054 };
0055
0056 /**
0057  * @list: used to add/insert into QP/PQ wait lists
0058  * @tx_head: overflow list of sdma_txreq's
0059  * @sleep: no space callback
0060  * @wakeup: space callback wakeup
0061  * @sdma_drained: sdma count drained
0062  * @init_priority: callback to manipulate priority
0063  * @lock: lock protected head of wait queue
0064  * @iowork: workqueue overhead
0065  * @wait_dma: wait for sdma_busy == 0
0066  * @wait_pio: wait for pio_busy == 0
0067  * @sdma_busy: # of packets in flight
0068  * @count: total number of descriptors in tx_head'ed list
0069  * @tx_limit: limit for overflow queuing
0070  * @tx_count: number of tx entry's in tx_head'ed list
0071  * @flags: wait flags (one per QP)
0072  * @wait: SE array for multiple legs
0073  *
0074  * This is to be embedded in user's state structure
0075  * (QP or PQ).
0076  *
0077  * The sleep and wakeup members are a
0078  * bit misnamed.   They do not strictly
0079  * speaking sleep or wake up, but they
0080  * are callbacks for the ULP to implement
0081  * what ever queuing/dequeuing of
0082  * the embedded iowait and its containing struct
0083  * when a resource shortage like SDMA ring space
0084  * or PIO credit space is seen.
0085  *
0086  * Both potentially have locks help
0087  * so sleeping is not allowed and it is not
0088  * supported to submit txreqs from the wakeup
0089  * call directly because of lock conflicts.
0090  *
0091  * The wait_dma member along with the iow
0092  *
0093  * The lock field is used by waiters to record
0094  * the seqlock_t that guards the list head.
0095  * Waiters explicity know that, but the destroy
0096  * code that unwaits QPs does not.
0097  */
0098 struct iowait {
0099     struct list_head list;
0100     int (*sleep)(
0101         struct sdma_engine *sde,
0102         struct iowait_work *wait,
0103         struct sdma_txreq *tx,
0104         uint seq,
0105         bool pkts_sent
0106         );
0107     void (*wakeup)(struct iowait *wait, int reason);
0108     void (*sdma_drained)(struct iowait *wait);
0109     void (*init_priority)(struct iowait *wait);
0110     seqlock_t *lock;
0111     wait_queue_head_t wait_dma;
0112     wait_queue_head_t wait_pio;
0113     atomic_t sdma_busy;
0114     atomic_t pio_busy;
0115     u32 count;
0116     u32 tx_limit;
0117     u32 tx_count;
0118     u8 starved_cnt;
0119     u8 priority;
0120     unsigned long flags;
0121     struct iowait_work wait[IOWAIT_SES];
0122 };
0123
0124 #define SDMA_AVAIL_REASON 0
0125
0126 void iowait_set_flag(struct iowait *wait, u32 flag);
0127 bool iowait_flag_set(struct iowait *wait, u32 flag);
0128 void iowait_clear_flag(struct iowait *wait, u32 flag);
0129
0130 void iowait_init(struct iowait *wait, u32 tx_limit,
0131          void (*func)(struct work_struct *work),
0132          void (*tidfunc)(struct work_struct *work),
0133          int (*sleep)(struct sdma_engine *sde,
0134                   struct iowait_work *wait,
0135                   struct sdma_txreq *tx,
0136                   uint seq,
0137                   bool pkts_sent),
0138          void (*wakeup)(struct iowait *wait, int reason),
0139          void (*sdma_drained)(struct iowait *wait),
0140          void (*init_priority)(struct iowait *wait));
0141
0142 /**
0143  * iowait_schedule() - schedule the default send engine work
0144  * @wait: wait struct to schedule
0145  * @wq: workqueue for schedule
0146  * @cpu: cpu
0147  */
0148 static inline bool iowait_schedule(struct iowait *wait,
0149                    struct workqueue_struct *wq, int cpu)
0150 {
0151     return !!queue_work_on(cpu, wq, &wait->wait[IOWAIT_IB_SE].iowork);
0152 }
0153
0154 /**
0155  * iowait_tid_schedule - schedule the tid SE
0156  * @wait: the iowait structure
0157  * @wq: the work queue
0158  * @cpu: the cpu
0159  */
0160 static inline bool iowait_tid_schedule(struct iowait *wait,
0161                        struct workqueue_struct *wq, int cpu)
0162 {
0163     return !!queue_work_on(cpu, wq, &wait->wait[IOWAIT_TID_SE].iowork);
0164 }
0165
0166 /**
0167  * iowait_sdma_drain() - wait for DMAs to drain
0168  *
0169  * @wait: iowait structure
0170  *
0171  * This will delay until the iowait sdmas have
0172  * completed.
0173  */
0174 static inline void iowait_sdma_drain(struct iowait *wait)
0175 {
0176     wait_event(wait->wait_dma, !atomic_read(&wait->sdma_busy));
0177 }
0178
0179 /**
0180  * iowait_sdma_pending() - return sdma pending count
0181  *
0182  * @wait: iowait structure
0183  *
0184  */
0185 static inline int iowait_sdma_pending(struct iowait *wait)
0186 {
0187     return atomic_read(&wait->sdma_busy);
0188 }
0189
0190 /**
0191  * iowait_sdma_inc - note sdma io pending
0192  * @wait: iowait structure
0193  */
0194 static inline void iowait_sdma_inc(struct iowait *wait)
0195 {
0196     atomic_inc(&wait->sdma_busy);
0197 }
0198
0199 /**
0200  * iowait_sdma_add - add count to pending
0201  * @wait: iowait structure
0202  */
0203 static inline void iowait_sdma_add(struct iowait *wait, int count)
0204 {
0205     atomic_add(count, &wait->sdma_busy);
0206 }
0207
0208 /**
0209  * iowait_sdma_dec - note sdma complete
0210  * @wait: iowait structure
0211  */
0212 static inline int iowait_sdma_dec(struct iowait *wait)
0213 {
0214     if (!wait)
0215         return 0;
0216     return atomic_dec_and_test(&wait->sdma_busy);
0217 }
0218
0219 /**
0220  * iowait_pio_drain() - wait for pios to drain
0221  *
0222  * @wait: iowait structure
0223  *
0224  * This will delay until the iowait pios have
0225  * completed.
0226  */
0227 static inline void iowait_pio_drain(struct iowait *wait)
0228 {
0229     wait_event_timeout(wait->wait_pio,
0230                !atomic_read(&wait->pio_busy),
0231                HZ);
0232 }
0233
0234 /**
0235  * iowait_pio_pending() - return pio pending count
0236  *
0237  * @wait: iowait structure
0238  *
0239  */
0240 static inline int iowait_pio_pending(struct iowait *wait)
0241 {
0242     return atomic_read(&wait->pio_busy);
0243 }
0244
0245 /**
0246  * iowait_pio_inc - note pio pending
0247  * @wait: iowait structure
0248  */
0249 static inline void iowait_pio_inc(struct iowait *wait)
0250 {
0251     atomic_inc(&wait->pio_busy);
0252 }
0253
0254 /**
0255  * iowait_pio_dec - note pio complete
0256  * @wait: iowait structure
0257  */
0258 static inline int iowait_pio_dec(struct iowait *wait)
0259 {
0260     if (!wait)
0261         return 0;
0262     return atomic_dec_and_test(&wait->pio_busy);
0263 }
0264
0265 /**
0266  * iowait_drain_wakeup() - trigger iowait_drain() waiter
0267  *
0268  * @wait: iowait structure
0269  *
0270  * This will trigger any waiters.
0271  */
0272 static inline void iowait_drain_wakeup(struct iowait *wait)
0273 {
0274     wake_up(&wait->wait_dma);
0275     wake_up(&wait->wait_pio);
0276     if (wait->sdma_drained)
0277         wait->sdma_drained(wait);
0278 }
0279
0280 /**
0281  * iowait_get_txhead() - get packet off of iowait list
0282  *
0283  * @wait: iowait_work structure
0284  */
0285 static inline struct sdma_txreq *iowait_get_txhead(struct iowait_work *wait)
0286 {
0287     struct sdma_txreq *tx = NULL;
0288
0289     if (!list_empty(&wait->tx_head)) {
0290         tx = list_first_entry(
0291             &wait->tx_head,
0292             struct sdma_txreq,
0293             list);
0294         list_del_init(&tx->list);
0295     }
0296     return tx;
0297 }
0298
0299 static inline u16 iowait_get_desc(struct iowait_work *w)
0300 {
0301     u16 num_desc = 0;
0302     struct sdma_txreq *tx = NULL;
0303
0304     if (!list_empty(&w->tx_head)) {
0305         tx = list_first_entry(&w->tx_head, struct sdma_txreq,
0306                       list);
0307         num_desc = tx->num_desc;
0308         if (tx->flags & SDMA_TXREQ_F_VIP)
0309             w->iow->priority++;
0310     }
0311     return num_desc;
0312 }
0313
0314 static inline u32 iowait_get_all_desc(struct iowait *w)
0315 {
0316     u32 num_desc = 0;
0317
0318     num_desc = iowait_get_desc(&w->wait[IOWAIT_IB_SE]);
0319     num_desc += iowait_get_desc(&w->wait[IOWAIT_TID_SE]);
0320     return num_desc;
0321 }
0322
0323 static inline void iowait_update_priority(struct iowait_work *w)
0324 {
0325     struct sdma_txreq *tx = NULL;
0326
0327     if (!list_empty(&w->tx_head)) {
0328         tx = list_first_entry(&w->tx_head, struct sdma_txreq,
0329                       list);
0330         if (tx->flags & SDMA_TXREQ_F_VIP)
0331             w->iow->priority++;
0332     }
0333 }
0334
0335 static inline void iowait_update_all_priority(struct iowait *w)
0336 {
0337     iowait_update_priority(&w->wait[IOWAIT_IB_SE]);
0338     iowait_update_priority(&w->wait[IOWAIT_TID_SE]);
0339 }
0340
0341 static inline void iowait_init_priority(struct iowait *w)
0342 {
0343     w->priority = 0;
0344     if (w->init_priority)
0345         w->init_priority(w);
0346 }
0347
0348 static inline void iowait_get_priority(struct iowait *w)
0349 {
0350     iowait_init_priority(w);
0351     iowait_update_all_priority(w);
0352 }
0353
0354 /**
0355  * iowait_queue - Put the iowait on a wait queue
0356  * @pkts_sent: have some packets been sent before queuing?
0357  * @w: the iowait struct
0358  * @wait_head: the wait queue
0359  *
0360  * This function is called to insert an iowait struct into a
0361  * wait queue after a resource (eg, sdma descriptor or pio
0362  * buffer) is run out.
0363  */
0364 static inline void iowait_queue(bool pkts_sent, struct iowait *w,
0365                 struct list_head *wait_head)
0366 {
0367     /*
0368      * To play fair, insert the iowait at the tail of the wait queue if it
0369      * has already sent some packets; Otherwise, put it at the head.
0370      * However, if it has priority packets to send, also put it at the
0371      * head.
0372      */
0373     if (pkts_sent)
0374         w->starved_cnt = 0;
0375     else
0376         w->starved_cnt++;
0377
0378     if (w->priority > 0 || !pkts_sent)
0379         list_add(&w->list, wait_head);
0380     else
0381         list_add_tail(&w->list, wait_head);
0382 }
0383
0384 /**
0385  * iowait_starve_clear - clear the wait queue's starve count
0386  * @pkts_sent: have some packets been sent?
0387  * @w: the iowait struct
0388  *
0389  * This function is called to clear the starve count. If no
0390  * packets have been sent, the starve count will not be cleared.
0391  */
0392 static inline void iowait_starve_clear(bool pkts_sent, struct iowait *w)
0393 {
0394     if (pkts_sent)
0395         w->starved_cnt = 0;
0396 }
0397
0398 /* Update the top priority index */
0399 uint iowait_priority_update_top(struct iowait *w,
0400                 struct iowait *top,
0401                 uint idx, uint top_idx);
0402
0403 /**
0404  * iowait_packet_queued() - determine if a packet is queued
0405  * @wait: the iowait_work structure
0406  */
0407 static inline bool iowait_packet_queued(struct iowait_work *wait)
0408 {
0409     return !list_empty(&wait->tx_head);
0410 }
0411
0412 /**
0413  * inc_wait_count - increment wait counts
0414  * @w: the log work struct
0415  * @n: the count
0416  */
0417 static inline void iowait_inc_wait_count(struct iowait_work *w, u16 n)
0418 {
0419     if (!w)
0420         return;
0421     w->iow->tx_count++;
0422     w->iow->count += n;
0423 }
0424
0425 /**
0426  * iowait_get_tid_work - return iowait_work for tid SE
0427  * @w: the iowait struct
0428  */
0429 static inline struct iowait_work *iowait_get_tid_work(struct iowait *w)
0430 {
0431     return &w->wait[IOWAIT_TID_SE];
0432 }
0433
0434 /**
0435  * iowait_get_ib_work - return iowait_work for ib SE
0436  * @w: the iowait struct
0437  */
0438 static inline struct iowait_work *iowait_get_ib_work(struct iowait *w)
0439 {
0440     return &w->wait[IOWAIT_IB_SE];
0441 }
0442
0443 /**
0444  * iowait_ioww_to_iow - return iowait given iowait_work
0445  * @w: the iowait_work struct
0446  */
0447 static inline struct iowait *iowait_ioww_to_iow(struct iowait_work *w)
0448 {
0449     if (likely(w))
0450         return w->iow;
0451     return NULL;
0452 }
0453
0454 void iowait_cancel_work(struct iowait *w);
0455 int iowait_set_work_flag(struct iowait_work *w);
0456
0457 #endif