i915/display/intel_frontbuffer.c

0001 /*
0002  * Copyright © 2014 Intel Corporation
0003  *
0004  * Permission is hereby granted, free of charge, to any person obtaining a
0005  * copy of this software and associated documentation files (the "Software"),
0006  * to deal in the Software without restriction, including without limitation
0007  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
0008  * and/or sell copies of the Software, and to permit persons to whom the
0009  * Software is furnished to do so, subject to the following conditions:
0010  *
0011  * The above copyright notice and this permission notice (including the next
0012  * paragraph) shall be included in all copies or substantial portions of the
0013  * Software.
0014  *
0015  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
0016  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
0017  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
0018  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
0019  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
0020  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
0021  * DEALINGS IN THE SOFTWARE.
0022  *
0023  * Authors:
0024  *  Daniel Vetter <daniel.vetter@ffwll.ch>
0025  */
0026
0027 /**
0028  * DOC: frontbuffer tracking
0029  *
0030  * Many features require us to track changes to the currently active
0031  * frontbuffer, especially rendering targeted at the frontbuffer.
0032  *
0033  * To be able to do so we track frontbuffers using a bitmask for all possible
0034  * frontbuffer slots through intel_frontbuffer_track(). The functions in this
0035  * file are then called when the contents of the frontbuffer are invalidated,
0036  * when frontbuffer rendering has stopped again to flush out all the changes
0037  * and when the frontbuffer is exchanged with a flip. Subsystems interested in
0038  * frontbuffer changes (e.g. PSR, FBC, DRRS) should directly put their callbacks
0039  * into the relevant places and filter for the frontbuffer slots that they are
0040  * interested int.
0041  *
0042  * On a high level there are two types of powersaving features. The first one
0043  * work like a special cache (FBC and PSR) and are interested when they should
0044  * stop caching and when to restart caching. This is done by placing callbacks
0045  * into the invalidate and the flush functions: At invalidate the caching must
0046  * be stopped and at flush time it can be restarted. And maybe they need to know
0047  * when the frontbuffer changes (e.g. when the hw doesn't initiate an invalidate
0048  * and flush on its own) which can be achieved with placing callbacks into the
0049  * flip functions.
0050  *
0051  * The other type of display power saving feature only cares about busyness
0052  * (e.g. DRRS). In that case all three (invalidate, flush and flip) indicate
0053  * busyness. There is no direct way to detect idleness. Instead an idle timer
0054  * work delayed work should be started from the flush and flip functions and
0055  * cancelled as soon as busyness is detected.
0056  */
0057
0058 #include "i915_drv.h"
0059 #include "intel_display_trace.h"
0060 #include "intel_display_types.h"
0061 #include "intel_dp.h"
0062 #include "intel_drrs.h"
0063 #include "intel_fbc.h"
0064 #include "intel_frontbuffer.h"
0065 #include "intel_psr.h"
0066
0067 /**
0068  * frontbuffer_flush - flush frontbuffer
0069  * @i915: i915 device
0070  * @frontbuffer_bits: frontbuffer plane tracking bits
0071  * @origin: which operation caused the flush
0072  *
0073  * This function gets called every time rendering on the given planes has
0074  * completed and frontbuffer caching can be started again. Flushes will get
0075  * delayed if they're blocked by some outstanding asynchronous rendering.
0076  *
0077  * Can be called without any locks held.
0078  */
0079 static void frontbuffer_flush(struct drm_i915_private *i915,
0080                   unsigned int frontbuffer_bits,
0081                   enum fb_op_origin origin)
0082 {
0083     /* Delay flushing when rings are still busy.*/
0084     spin_lock(&i915->fb_tracking.lock);
0085     frontbuffer_bits &= ~i915->fb_tracking.busy_bits;
0086     spin_unlock(&i915->fb_tracking.lock);
0087
0088     if (!frontbuffer_bits)
0089         return;
0090
0091     trace_intel_frontbuffer_flush(frontbuffer_bits, origin);
0092
0093     might_sleep();
0094     intel_drrs_flush(i915, frontbuffer_bits);
0095     intel_psr_flush(i915, frontbuffer_bits, origin);
0096     intel_fbc_flush(i915, frontbuffer_bits, origin);
0097 }
0098
0099 /**
0100  * intel_frontbuffer_flip_prepare - prepare asynchronous frontbuffer flip
0101  * @i915: i915 device
0102  * @frontbuffer_bits: frontbuffer plane tracking bits
0103  *
0104  * This function gets called after scheduling a flip on @obj. The actual
0105  * frontbuffer flushing will be delayed until completion is signalled with
0106  * intel_frontbuffer_flip_complete. If an invalidate happens in between this
0107  * flush will be cancelled.
0108  *
0109  * Can be called without any locks held.
0110  */
0111 void intel_frontbuffer_flip_prepare(struct drm_i915_private *i915,
0112                     unsigned frontbuffer_bits)
0113 {
0114     spin_lock(&i915->fb_tracking.lock);
0115     i915->fb_tracking.flip_bits |= frontbuffer_bits;
0116     /* Remove stale busy bits due to the old buffer. */
0117     i915->fb_tracking.busy_bits &= ~frontbuffer_bits;
0118     spin_unlock(&i915->fb_tracking.lock);
0119 }
0120
0121 /**
0122  * intel_frontbuffer_flip_complete - complete asynchronous frontbuffer flip
0123  * @i915: i915 device
0124  * @frontbuffer_bits: frontbuffer plane tracking bits
0125  *
0126  * This function gets called after the flip has been latched and will complete
0127  * on the next vblank. It will execute the flush if it hasn't been cancelled yet.
0128  *
0129  * Can be called without any locks held.
0130  */
0131 void intel_frontbuffer_flip_complete(struct drm_i915_private *i915,
0132                      unsigned frontbuffer_bits)
0133 {
0134     spin_lock(&i915->fb_tracking.lock);
0135     /* Mask any cancelled flips. */
0136     frontbuffer_bits &= i915->fb_tracking.flip_bits;
0137     i915->fb_tracking.flip_bits &= ~frontbuffer_bits;
0138     spin_unlock(&i915->fb_tracking.lock);
0139
0140     if (frontbuffer_bits)
0141         frontbuffer_flush(i915, frontbuffer_bits, ORIGIN_FLIP);
0142 }
0143
0144 /**
0145  * intel_frontbuffer_flip - synchronous frontbuffer flip
0146  * @i915: i915 device
0147  * @frontbuffer_bits: frontbuffer plane tracking bits
0148  *
0149  * This function gets called after scheduling a flip on @obj. This is for
0150  * synchronous plane updates which will happen on the next vblank and which will
0151  * not get delayed by pending gpu rendering.
0152  *
0153  * Can be called without any locks held.
0154  */
0155 void intel_frontbuffer_flip(struct drm_i915_private *i915,
0156                 unsigned frontbuffer_bits)
0157 {
0158     spin_lock(&i915->fb_tracking.lock);
0159     /* Remove stale busy bits due to the old buffer. */
0160     i915->fb_tracking.busy_bits &= ~frontbuffer_bits;
0161     spin_unlock(&i915->fb_tracking.lock);
0162
0163     frontbuffer_flush(i915, frontbuffer_bits, ORIGIN_FLIP);
0164 }
0165
0166 void __intel_fb_invalidate(struct intel_frontbuffer *front,
0167                enum fb_op_origin origin,
0168                unsigned int frontbuffer_bits)
0169 {
0170     struct drm_i915_private *i915 = to_i915(front->obj->base.dev);
0171
0172     if (origin == ORIGIN_CS) {
0173         spin_lock(&i915->fb_tracking.lock);
0174         i915->fb_tracking.busy_bits |= frontbuffer_bits;
0175         i915->fb_tracking.flip_bits &= ~frontbuffer_bits;
0176         spin_unlock(&i915->fb_tracking.lock);
0177     }
0178
0179     trace_intel_frontbuffer_invalidate(frontbuffer_bits, origin);
0180
0181     might_sleep();
0182     intel_psr_invalidate(i915, frontbuffer_bits, origin);
0183     intel_drrs_invalidate(i915, frontbuffer_bits);
0184     intel_fbc_invalidate(i915, frontbuffer_bits, origin);
0185 }
0186
0187 void __intel_fb_flush(struct intel_frontbuffer *front,
0188               enum fb_op_origin origin,
0189               unsigned int frontbuffer_bits)
0190 {
0191     struct drm_i915_private *i915 = to_i915(front->obj->base.dev);
0192
0193     if (origin == ORIGIN_CS) {
0194         spin_lock(&i915->fb_tracking.lock);
0195         /* Filter out new bits since rendering started. */
0196         frontbuffer_bits &= i915->fb_tracking.busy_bits;
0197         i915->fb_tracking.busy_bits &= ~frontbuffer_bits;
0198         spin_unlock(&i915->fb_tracking.lock);
0199     }
0200
0201     if (frontbuffer_bits)
0202         frontbuffer_flush(i915, frontbuffer_bits, origin);
0203 }
0204
0205 static int frontbuffer_active(struct i915_active *ref)
0206 {
0207     struct intel_frontbuffer *front =
0208         container_of(ref, typeof(*front), write);
0209
0210     kref_get(&front->ref);
0211     return 0;
0212 }
0213
0214 static void frontbuffer_retire(struct i915_active *ref)
0215 {
0216     struct intel_frontbuffer *front =
0217         container_of(ref, typeof(*front), write);
0218
0219     intel_frontbuffer_flush(front, ORIGIN_CS);
0220     intel_frontbuffer_put(front);
0221 }
0222
0223 static void frontbuffer_release(struct kref *ref)
0224     __releases(&to_i915(front->obj->base.dev)->fb_tracking.lock)
0225 {
0226     struct intel_frontbuffer *front =
0227         container_of(ref, typeof(*front), ref);
0228     struct drm_i915_gem_object *obj = front->obj;
0229     struct i915_vma *vma;
0230
0231     drm_WARN_ON(obj->base.dev, atomic_read(&front->bits));
0232
0233     spin_lock(&obj->vma.lock);
0234     for_each_ggtt_vma(vma, obj) {
0235         i915_vma_clear_scanout(vma);
0236         vma->display_alignment = I915_GTT_MIN_ALIGNMENT;
0237     }
0238     spin_unlock(&obj->vma.lock);
0239
0240     RCU_INIT_POINTER(obj->frontbuffer, NULL);
0241     spin_unlock(&to_i915(obj->base.dev)->fb_tracking.lock);
0242
0243     i915_active_fini(&front->write);
0244
0245     i915_gem_object_put(obj);
0246     kfree_rcu(front, rcu);
0247 }
0248
0249 struct intel_frontbuffer *
0250 intel_frontbuffer_get(struct drm_i915_gem_object *obj)
0251 {
0252     struct drm_i915_private *i915 = to_i915(obj->base.dev);
0253     struct intel_frontbuffer *front;
0254
0255     front = __intel_frontbuffer_get(obj);
0256     if (front)
0257         return front;
0258
0259     front = kmalloc(sizeof(*front), GFP_KERNEL);
0260     if (!front)
0261         return NULL;
0262
0263     front->obj = obj;
0264     kref_init(&front->ref);
0265     atomic_set(&front->bits, 0);
0266     i915_active_init(&front->write,
0267              frontbuffer_active,
0268              frontbuffer_retire,
0269              I915_ACTIVE_RETIRE_SLEEPS);
0270
0271     spin_lock(&i915->fb_tracking.lock);
0272     if (rcu_access_pointer(obj->frontbuffer)) {
0273         kfree(front);
0274         front = rcu_dereference_protected(obj->frontbuffer, true);
0275         kref_get(&front->ref);
0276     } else {
0277         i915_gem_object_get(obj);
0278         rcu_assign_pointer(obj->frontbuffer, front);
0279     }
0280     spin_unlock(&i915->fb_tracking.lock);
0281
0282     return front;
0283 }
0284
0285 void intel_frontbuffer_put(struct intel_frontbuffer *front)
0286 {
0287     kref_put_lock(&front->ref,
0288               frontbuffer_release,
0289               &to_i915(front->obj->base.dev)->fb_tracking.lock);
0290 }
0291
0292 /**
0293  * intel_frontbuffer_track - update frontbuffer tracking
0294  * @old: current buffer for the frontbuffer slots
0295  * @new: new buffer for the frontbuffer slots
0296  * @frontbuffer_bits: bitmask of frontbuffer slots
0297  *
0298  * This updates the frontbuffer tracking bits @frontbuffer_bits by clearing them
0299  * from @old and setting them in @new. Both @old and @new can be NULL.
0300  */
0301 void intel_frontbuffer_track(struct intel_frontbuffer *old,
0302                  struct intel_frontbuffer *new,
0303                  unsigned int frontbuffer_bits)
0304 {
0305     /*
0306      * Control of individual bits within the mask are guarded by
0307      * the owning plane->mutex, i.e. we can never see concurrent
0308      * manipulation of individual bits. But since the bitfield as a whole
0309      * is updated using RMW, we need to use atomics in order to update
0310      * the bits.
0311      */
0312     BUILD_BUG_ON(INTEL_FRONTBUFFER_BITS_PER_PIPE * I915_MAX_PIPES >
0313              BITS_PER_TYPE(atomic_t));
0314
0315     if (old) {
0316         drm_WARN_ON(old->obj->base.dev,
0317                 !(atomic_read(&old->bits) & frontbuffer_bits));
0318         atomic_andnot(frontbuffer_bits, &old->bits);
0319     }
0320
0321     if (new) {
0322         drm_WARN_ON(new->obj->base.dev,
0323                 atomic_read(&new->bits) & frontbuffer_bits);
0324         atomic_or(frontbuffer_bits, &new->bits);
0325     }
0326 }