i915/gt/intel_engine.h

0001 /* SPDX-License-Identifier: MIT */
0002 #ifndef _INTEL_RINGBUFFER_H_
0003 #define _INTEL_RINGBUFFER_H_
0004
0005 #include <asm/cacheflush.h>
0006 #include <drm/drm_util.h>
0007 #include <drm/drm_cache.h>
0008
0009 #include <linux/hashtable.h>
0010 #include <linux/irq_work.h>
0011 #include <linux/random.h>
0012 #include <linux/seqlock.h>
0013
0014 #include "i915_pmu.h"
0015 #include "i915_request.h"
0016 #include "i915_selftest.h"
0017 #include "intel_engine_types.h"
0018 #include "intel_gt_types.h"
0019 #include "intel_timeline.h"
0020 #include "intel_workarounds.h"
0021
0022 struct drm_printer;
0023 struct intel_context;
0024 struct intel_gt;
0025 struct lock_class_key;
0026
0027 /* Early gen2 devices have a cacheline of just 32 bytes, using 64 is overkill,
0028  * but keeps the logic simple. Indeed, the whole purpose of this macro is just
0029  * to give some inclination as to some of the magic values used in the various
0030  * workarounds!
0031  */
0032 #define CACHELINE_BYTES 64
0033 #define CACHELINE_DWORDS (CACHELINE_BYTES / sizeof(u32))
0034
0035 #define ENGINE_TRACE(e, fmt, ...) do {                  \
0036     const struct intel_engine_cs *e__ __maybe_unused = (e);     \
0037     GEM_TRACE("%s %s: " fmt,                    \
0038           dev_name(e__->i915->drm.dev), e__->name,      \
0039           ##__VA_ARGS__);                   \
0040 } while (0)
0041
0042 /*
0043  * The register defines to be used with the following macros need to accept a
0044  * base param, e.g:
0045  *
0046  * REG_FOO(base) _MMIO((base) + <relative offset>)
0047  * ENGINE_READ(engine, REG_FOO);
0048  *
0049  * register arrays are to be defined and accessed as follows:
0050  *
0051  * REG_BAR(base, i) _MMIO((base) + <relative offset> + (i) * <shift>)
0052  * ENGINE_READ_IDX(engine, REG_BAR, i)
0053  */
0054
0055 #define __ENGINE_REG_OP(op__, engine__, ...) \
0056     intel_uncore_##op__((engine__)->uncore, __VA_ARGS__)
0057
0058 #define __ENGINE_READ_OP(op__, engine__, reg__) \
0059     __ENGINE_REG_OP(op__, (engine__), reg__((engine__)->mmio_base))
0060
0061 #define ENGINE_READ16(...)  __ENGINE_READ_OP(read16, __VA_ARGS__)
0062 #define ENGINE_READ(...)    __ENGINE_READ_OP(read, __VA_ARGS__)
0063 #define ENGINE_READ_FW(...) __ENGINE_READ_OP(read_fw, __VA_ARGS__)
0064 #define ENGINE_POSTING_READ(...) __ENGINE_READ_OP(posting_read_fw, __VA_ARGS__)
0065 #define ENGINE_POSTING_READ16(...) __ENGINE_READ_OP(posting_read16, __VA_ARGS__)
0066
0067 #define ENGINE_READ64(engine__, lower_reg__, upper_reg__) \
0068     __ENGINE_REG_OP(read64_2x32, (engine__), \
0069             lower_reg__((engine__)->mmio_base), \
0070             upper_reg__((engine__)->mmio_base))
0071
0072 #define ENGINE_READ_IDX(engine__, reg__, idx__) \
0073     __ENGINE_REG_OP(read, (engine__), reg__((engine__)->mmio_base, (idx__)))
0074
0075 #define __ENGINE_WRITE_OP(op__, engine__, reg__, val__) \
0076     __ENGINE_REG_OP(op__, (engine__), reg__((engine__)->mmio_base), (val__))
0077
0078 #define ENGINE_WRITE16(...) __ENGINE_WRITE_OP(write16, __VA_ARGS__)
0079 #define ENGINE_WRITE(...)   __ENGINE_WRITE_OP(write, __VA_ARGS__)
0080 #define ENGINE_WRITE_FW(...)    __ENGINE_WRITE_OP(write_fw, __VA_ARGS__)
0081
0082 #define GEN6_RING_FAULT_REG_READ(engine__) \
0083     intel_uncore_read((engine__)->uncore, RING_FAULT_REG(engine__))
0084
0085 #define GEN6_RING_FAULT_REG_POSTING_READ(engine__) \
0086     intel_uncore_posting_read((engine__)->uncore, RING_FAULT_REG(engine__))
0087
0088 #define GEN6_RING_FAULT_REG_RMW(engine__, clear__, set__) \
0089 ({ \
0090     u32 __val; \
0091 \
0092     __val = intel_uncore_read((engine__)->uncore, \
0093                   RING_FAULT_REG(engine__)); \
0094     __val &= ~(clear__); \
0095     __val |= (set__); \
0096     intel_uncore_write((engine__)->uncore, RING_FAULT_REG(engine__), \
0097                __val); \
0098 })
0099
0100 /* seqno size is actually only a uint32, but since we plan to use MI_FLUSH_DW to
0101  * do the writes, and that must have qw aligned offsets, simply pretend it's 8b.
0102  */
0103
0104 static inline unsigned int
0105 execlists_num_ports(const struct intel_engine_execlists * const execlists)
0106 {
0107     return execlists->port_mask + 1;
0108 }
0109
0110 static inline struct i915_request *
0111 execlists_active(const struct intel_engine_execlists *execlists)
0112 {
0113     struct i915_request * const *cur, * const *old, *active;
0114
0115     cur = READ_ONCE(execlists->active);
0116     smp_rmb(); /* pairs with overwrite protection in process_csb() */
0117     do {
0118         old = cur;
0119
0120         active = READ_ONCE(*cur);
0121         cur = READ_ONCE(execlists->active);
0122
0123         smp_rmb(); /* and complete the seqlock retry */
0124     } while (unlikely(cur != old));
0125
0126     return active;
0127 }
0128
0129 struct i915_request *
0130 execlists_unwind_incomplete_requests(struct intel_engine_execlists *execlists);
0131
0132 static inline u32
0133 intel_read_status_page(const struct intel_engine_cs *engine, int reg)
0134 {
0135     /* Ensure that the compiler doesn't optimize away the load. */
0136     return READ_ONCE(engine->status_page.addr[reg]);
0137 }
0138
0139 static inline void
0140 intel_write_status_page(struct intel_engine_cs *engine, int reg, u32 value)
0141 {
0142     /* Writing into the status page should be done sparingly. Since
0143      * we do when we are uncertain of the device state, we take a bit
0144      * of extra paranoia to try and ensure that the HWS takes the value
0145      * we give and that it doesn't end up trapped inside the CPU!
0146      */
0147     drm_clflush_virt_range(&engine->status_page.addr[reg], sizeof(value));
0148     WRITE_ONCE(engine->status_page.addr[reg], value);
0149     drm_clflush_virt_range(&engine->status_page.addr[reg], sizeof(value));
0150 }
0151
0152 /*
0153  * Reads a dword out of the status page, which is written to from the command
0154  * queue by automatic updates, MI_REPORT_HEAD, MI_STORE_DATA_INDEX, or
0155  * MI_STORE_DATA_IMM.
0156  *
0157  * The following dwords have a reserved meaning:
0158  * 0x00: ISR copy, updated when an ISR bit not set in the HWSTAM changes.
0159  * 0x04: ring 0 head pointer
0160  * 0x05: ring 1 head pointer (915-class)
0161  * 0x06: ring 2 head pointer (915-class)
0162  * 0x10-0x1b: Context status DWords (GM45)
0163  * 0x1f: Last written status offset. (GM45)
0164  * 0x20-0x2f: Reserved (Gen6+)
0165  *
0166  * The area from dword 0x30 to 0x3ff is available for driver usage.
0167  */
0168 #define I915_GEM_HWS_PREEMPT        0x32
0169 #define I915_GEM_HWS_PREEMPT_ADDR   (I915_GEM_HWS_PREEMPT * sizeof(u32))
0170 #define I915_GEM_HWS_SEQNO      0x40
0171 #define I915_GEM_HWS_SEQNO_ADDR     (I915_GEM_HWS_SEQNO * sizeof(u32))
0172 #define I915_GEM_HWS_MIGRATE        (0x42 * sizeof(u32))
0173 #define I915_GEM_HWS_PXP        0x60
0174 #define I915_GEM_HWS_PXP_ADDR       (I915_GEM_HWS_PXP * sizeof(u32))
0175 #define I915_GEM_HWS_SCRATCH        0x80
0176
0177 #define I915_HWS_CSB_BUF0_INDEX     0x10
0178 #define I915_HWS_CSB_WRITE_INDEX    0x1f
0179 #define ICL_HWS_CSB_WRITE_INDEX     0x2f
0180 #define INTEL_HWS_CSB_WRITE_INDEX(__i915) \
0181     (GRAPHICS_VER(__i915) >= 11 ? ICL_HWS_CSB_WRITE_INDEX : I915_HWS_CSB_WRITE_INDEX)
0182
0183 void intel_engine_stop(struct intel_engine_cs *engine);
0184 void intel_engine_cleanup(struct intel_engine_cs *engine);
0185
0186 int intel_engines_init_mmio(struct intel_gt *gt);
0187 int intel_engines_init(struct intel_gt *gt);
0188
0189 void intel_engine_free_request_pool(struct intel_engine_cs *engine);
0190
0191 void intel_engines_release(struct intel_gt *gt);
0192 void intel_engines_free(struct intel_gt *gt);
0193
0194 int intel_engine_init_common(struct intel_engine_cs *engine);
0195 void intel_engine_cleanup_common(struct intel_engine_cs *engine);
0196
0197 int intel_engine_resume(struct intel_engine_cs *engine);
0198
0199 int intel_ring_submission_setup(struct intel_engine_cs *engine);
0200
0201 int intel_engine_stop_cs(struct intel_engine_cs *engine);
0202 void intel_engine_cancel_stop_cs(struct intel_engine_cs *engine);
0203
0204 void intel_engine_wait_for_pending_mi_fw(struct intel_engine_cs *engine);
0205
0206 void intel_engine_set_hwsp_writemask(struct intel_engine_cs *engine, u32 mask);
0207
0208 u64 intel_engine_get_active_head(const struct intel_engine_cs *engine);
0209 u64 intel_engine_get_last_batch_head(const struct intel_engine_cs *engine);
0210
0211 void intel_engine_get_instdone(const struct intel_engine_cs *engine,
0212                    struct intel_instdone *instdone);
0213
0214 void intel_engine_init_execlists(struct intel_engine_cs *engine);
0215
0216 bool intel_engine_irq_enable(struct intel_engine_cs *engine);
0217 void intel_engine_irq_disable(struct intel_engine_cs *engine);
0218
0219 static inline void __intel_engine_reset(struct intel_engine_cs *engine,
0220                     bool stalled)
0221 {
0222     if (engine->reset.rewind)
0223         engine->reset.rewind(engine, stalled);
0224     engine->serial++; /* contexts lost */
0225 }
0226
0227 bool intel_engines_are_idle(struct intel_gt *gt);
0228 bool intel_engine_is_idle(struct intel_engine_cs *engine);
0229
0230 void __intel_engine_flush_submission(struct intel_engine_cs *engine, bool sync);
0231 static inline void intel_engine_flush_submission(struct intel_engine_cs *engine)
0232 {
0233     __intel_engine_flush_submission(engine, true);
0234 }
0235
0236 void intel_engines_reset_default_submission(struct intel_gt *gt);
0237
0238 bool intel_engine_can_store_dword(struct intel_engine_cs *engine);
0239
0240 __printf(3, 4)
0241 void intel_engine_dump(struct intel_engine_cs *engine,
0242                struct drm_printer *m,
0243                const char *header, ...);
0244 void intel_engine_dump_active_requests(struct list_head *requests,
0245                        struct i915_request *hung_rq,
0246                        struct drm_printer *m);
0247
0248 ktime_t intel_engine_get_busy_time(struct intel_engine_cs *engine,
0249                    ktime_t *now);
0250
0251 struct i915_request *
0252 intel_engine_execlist_find_hung_request(struct intel_engine_cs *engine);
0253
0254 u32 intel_engine_context_size(struct intel_gt *gt, u8 class);
0255 struct intel_context *
0256 intel_engine_create_pinned_context(struct intel_engine_cs *engine,
0257                    struct i915_address_space *vm,
0258                    unsigned int ring_size,
0259                    unsigned int hwsp,
0260                    struct lock_class_key *key,
0261                    const char *name);
0262
0263 void intel_engine_destroy_pinned_context(struct intel_context *ce);
0264
0265 void xehp_enable_ccs_engines(struct intel_engine_cs *engine);
0266
0267 #define ENGINE_PHYSICAL 0
0268 #define ENGINE_MOCK 1
0269 #define ENGINE_VIRTUAL  2
0270
0271 static inline bool intel_engine_uses_guc(const struct intel_engine_cs *engine)
0272 {
0273     return engine->gt->submission_method >= INTEL_SUBMISSION_GUC;
0274 }
0275
0276 static inline bool
0277 intel_engine_has_preempt_reset(const struct intel_engine_cs *engine)
0278 {
0279     if (!CONFIG_DRM_I915_PREEMPT_TIMEOUT)
0280         return false;
0281
0282     return intel_engine_has_preemption(engine);
0283 }
0284
0285 #define FORCE_VIRTUAL   BIT(0)
0286 struct intel_context *
0287 intel_engine_create_virtual(struct intel_engine_cs **siblings,
0288                 unsigned int count, unsigned long flags);
0289
0290 static inline struct intel_context *
0291 intel_engine_create_parallel(struct intel_engine_cs **engines,
0292                  unsigned int num_engines,
0293                  unsigned int width)
0294 {
0295     GEM_BUG_ON(!engines[0]->cops->create_parallel);
0296     return engines[0]->cops->create_parallel(engines, num_engines, width);
0297 }
0298
0299 static inline bool
0300 intel_virtual_engine_has_heartbeat(const struct intel_engine_cs *engine)
0301 {
0302     /*
0303      * For non-GuC submission we expect the back-end to look at the
0304      * heartbeat status of the actual physical engine that the work
0305      * has been (or is being) scheduled on, so we should only reach
0306      * here with GuC submission enabled.
0307      */
0308     GEM_BUG_ON(!intel_engine_uses_guc(engine));
0309
0310     return intel_guc_virtual_engine_has_heartbeat(engine);
0311 }
0312
0313 static inline bool
0314 intel_engine_has_heartbeat(const struct intel_engine_cs *engine)
0315 {
0316     if (!CONFIG_DRM_I915_HEARTBEAT_INTERVAL)
0317         return false;
0318
0319     if (intel_engine_is_virtual(engine))
0320         return intel_virtual_engine_has_heartbeat(engine);
0321     else
0322         return READ_ONCE(engine->props.heartbeat_interval_ms);
0323 }
0324
0325 static inline struct intel_engine_cs *
0326 intel_engine_get_sibling(struct intel_engine_cs *engine, unsigned int sibling)
0327 {
0328     GEM_BUG_ON(!intel_engine_is_virtual(engine));
0329     return engine->cops->get_sibling(engine, sibling);
0330 }
0331
0332 static inline void
0333 intel_engine_set_hung_context(struct intel_engine_cs *engine,
0334                   struct intel_context *ce)
0335 {
0336     engine->hung_ce = ce;
0337 }
0338
0339 static inline void
0340 intel_engine_clear_hung_context(struct intel_engine_cs *engine)
0341 {
0342     intel_engine_set_hung_context(engine, NULL);
0343 }
0344
0345 static inline struct intel_context *
0346 intel_engine_get_hung_context(struct intel_engine_cs *engine)
0347 {
0348     return engine->hung_ce;
0349 }
0350
0351 #endif /* _INTEL_RINGBUFFER_H_ */