0001
0002
0003
0004
0005
0006
0007
0008
0009
0010
0011
0012
0013
0014
0015
0016
0017
0018
0019
0020
0021
0022
0023
0024 #ifndef DRM_SCHEDULER_SPSC_QUEUE_H_
0025 #define DRM_SCHEDULER_SPSC_QUEUE_H_
0026
0027 #include <linux/atomic.h>
0028 #include <linux/preempt.h>
0029
0030
0031
0032 struct spsc_node {
0033
0034
0035 struct spsc_node *next;
0036 };
0037
0038 struct spsc_queue {
0039
0040 struct spsc_node *head;
0041
0042
0043 atomic_long_t tail;
0044
0045 atomic_t job_count;
0046 };
0047
0048 static inline void spsc_queue_init(struct spsc_queue *queue)
0049 {
0050 queue->head = NULL;
0051 atomic_long_set(&queue->tail, (long)&queue->head);
0052 atomic_set(&queue->job_count, 0);
0053 }
0054
0055 static inline struct spsc_node *spsc_queue_peek(struct spsc_queue *queue)
0056 {
0057 return queue->head;
0058 }
0059
0060 static inline int spsc_queue_count(struct spsc_queue *queue)
0061 {
0062 return atomic_read(&queue->job_count);
0063 }
0064
0065 static inline bool spsc_queue_push(struct spsc_queue *queue, struct spsc_node *node)
0066 {
0067 struct spsc_node **tail;
0068
0069 node->next = NULL;
0070
0071 preempt_disable();
0072
0073 tail = (struct spsc_node **)atomic_long_xchg(&queue->tail, (long)&node->next);
0074 WRITE_ONCE(*tail, node);
0075 atomic_inc(&queue->job_count);
0076
0077
0078
0079
0080
0081 smp_wmb();
0082
0083 preempt_enable();
0084
0085 return tail == &queue->head;
0086 }
0087
0088
0089 static inline struct spsc_node *spsc_queue_pop(struct spsc_queue *queue)
0090 {
0091 struct spsc_node *next, *node;
0092
0093
0094 smp_rmb();
0095
0096 node = READ_ONCE(queue->head);
0097
0098 if (!node)
0099 return NULL;
0100
0101 next = READ_ONCE(node->next);
0102 WRITE_ONCE(queue->head, next);
0103
0104 if (unlikely(!next)) {
0105
0106
0107 if (atomic_long_cmpxchg(&queue->tail,
0108 (long)&node->next, (long) &queue->head) != (long)&node->next) {
0109
0110 do {
0111 smp_rmb();
0112 } while (unlikely(!(queue->head = READ_ONCE(node->next))));
0113 }
0114 }
0115
0116 atomic_dec(&queue->job_count);
0117 return node;
0118 }
0119
0120
0121
0122 #endif