i915/gt/intel_ring.h

0001 /* SPDX-License-Identifier: MIT */
0002 /*
0003  * Copyright © 2019 Intel Corporation
0004  */
0005
0006 #ifndef INTEL_RING_H
0007 #define INTEL_RING_H
0008
0009 #include "i915_gem.h" /* GEM_BUG_ON */
0010 #include "i915_request.h"
0011 #include "intel_ring_types.h"
0012
0013 struct intel_engine_cs;
0014
0015 struct intel_ring *
0016 intel_engine_create_ring(struct intel_engine_cs *engine, int size);
0017
0018 u32 *intel_ring_begin(struct i915_request *rq, unsigned int num_dwords);
0019 int intel_ring_cacheline_align(struct i915_request *rq);
0020
0021 unsigned int intel_ring_update_space(struct intel_ring *ring);
0022
0023 void __intel_ring_pin(struct intel_ring *ring);
0024 int intel_ring_pin(struct intel_ring *ring, struct i915_gem_ww_ctx *ww);
0025 void intel_ring_unpin(struct intel_ring *ring);
0026 void intel_ring_reset(struct intel_ring *ring, u32 tail);
0027
0028 void intel_ring_free(struct kref *ref);
0029
0030 static inline struct intel_ring *intel_ring_get(struct intel_ring *ring)
0031 {
0032     kref_get(&ring->ref);
0033     return ring;
0034 }
0035
0036 static inline void intel_ring_put(struct intel_ring *ring)
0037 {
0038     kref_put(&ring->ref, intel_ring_free);
0039 }
0040
0041 static inline void intel_ring_advance(struct i915_request *rq, u32 *cs)
0042 {
0043     /* Dummy function.
0044      *
0045      * This serves as a placeholder in the code so that the reader
0046      * can compare against the preceding intel_ring_begin() and
0047      * check that the number of dwords emitted matches the space
0048      * reserved for the command packet (i.e. the value passed to
0049      * intel_ring_begin()).
0050      */
0051     GEM_BUG_ON((rq->ring->vaddr + rq->ring->emit) != cs);
0052     GEM_BUG_ON(!IS_ALIGNED(rq->ring->emit, 8)); /* RING_TAIL qword align */
0053 }
0054
0055 static inline u32 intel_ring_wrap(const struct intel_ring *ring, u32 pos)
0056 {
0057     return pos & (ring->size - 1);
0058 }
0059
0060 static inline int intel_ring_direction(const struct intel_ring *ring,
0061                        u32 next, u32 prev)
0062 {
0063     typecheck(typeof(ring->size), next);
0064     typecheck(typeof(ring->size), prev);
0065     return (next - prev) << ring->wrap;
0066 }
0067
0068 static inline bool
0069 intel_ring_offset_valid(const struct intel_ring *ring,
0070             unsigned int pos)
0071 {
0072     if (pos & -ring->size) /* must be strictly within the ring */
0073         return false;
0074
0075     if (!IS_ALIGNED(pos, 8)) /* must be qword aligned */
0076         return false;
0077
0078     return true;
0079 }
0080
0081 static inline u32 intel_ring_offset(const struct i915_request *rq, void *addr)
0082 {
0083     /* Don't write ring->size (equivalent to 0) as that hangs some GPUs. */
0084     u32 offset = addr - rq->ring->vaddr;
0085
0086     GEM_BUG_ON(offset > rq->ring->size);
0087     return intel_ring_wrap(rq->ring, offset);
0088 }
0089
0090 static inline void
0091 assert_ring_tail_valid(const struct intel_ring *ring, unsigned int tail)
0092 {
0093     unsigned int head = READ_ONCE(ring->head);
0094
0095     GEM_BUG_ON(!intel_ring_offset_valid(ring, tail));
0096
0097     /*
0098      * "Ring Buffer Use"
0099      *  Gen2 BSpec "1. Programming Environment" / 1.4.4.6
0100      *  Gen3 BSpec "1c Memory Interface Functions" / 2.3.4.5
0101      *  Gen4+ BSpec "1c Memory Interface and Command Stream" / 5.3.4.5
0102      * "If the Ring Buffer Head Pointer and the Tail Pointer are on the
0103      * same cacheline, the Head Pointer must not be greater than the Tail
0104      * Pointer."
0105      *
0106      * We use ring->head as the last known location of the actual RING_HEAD,
0107      * it may have advanced but in the worst case it is equally the same
0108      * as ring->head and so we should never program RING_TAIL to advance
0109      * into the same cacheline as ring->head.
0110      */
0111 #define cacheline(a) round_down(a, CACHELINE_BYTES)
0112     GEM_BUG_ON(cacheline(tail) == cacheline(head) && tail < head);
0113 #undef cacheline
0114 }
0115
0116 static inline unsigned int
0117 intel_ring_set_tail(struct intel_ring *ring, unsigned int tail)
0118 {
0119     /* Whilst writes to the tail are strictly order, there is no
0120      * serialisation between readers and the writers. The tail may be
0121      * read by i915_request_retire() just as it is being updated
0122      * by execlists, as although the breadcrumb is complete, the context
0123      * switch hasn't been seen.
0124      */
0125     assert_ring_tail_valid(ring, tail);
0126     ring->tail = tail;
0127     return tail;
0128 }
0129
0130 static inline unsigned int
0131 __intel_ring_space(unsigned int head, unsigned int tail, unsigned int size)
0132 {
0133     /*
0134      * "If the Ring Buffer Head Pointer and the Tail Pointer are on the
0135      * same cacheline, the Head Pointer must not be greater than the Tail
0136      * Pointer."
0137      */
0138     GEM_BUG_ON(!is_power_of_2(size));
0139     return (head - tail - CACHELINE_BYTES) & (size - 1);
0140 }
0141
0142 #endif /* INTEL_RING_H */