i915/display/intel_crtc.c

0001 // SPDX-License-Identifier: MIT
0002 /*
0003  * Copyright © 2020 Intel Corporation
0004  */
0005 #include <linux/kernel.h>
0006 #include <linux/pm_qos.h>
0007 #include <linux/slab.h>
0008
0009 #include <drm/drm_atomic_helper.h>
0010 #include <drm/drm_fourcc.h>
0011 #include <drm/drm_plane.h>
0012 #include <drm/drm_plane_helper.h>
0013 #include <drm/drm_vblank_work.h>
0014
0015 #include "i915_irq.h"
0016 #include "i915_vgpu.h"
0017 #include "i9xx_plane.h"
0018 #include "icl_dsi.h"
0019 #include "intel_atomic.h"
0020 #include "intel_atomic_plane.h"
0021 #include "intel_color.h"
0022 #include "intel_crtc.h"
0023 #include "intel_cursor.h"
0024 #include "intel_display_debugfs.h"
0025 #include "intel_display_trace.h"
0026 #include "intel_display_types.h"
0027 #include "intel_drrs.h"
0028 #include "intel_dsi.h"
0029 #include "intel_pipe_crc.h"
0030 #include "intel_psr.h"
0031 #include "intel_sprite.h"
0032 #include "intel_vrr.h"
0033 #include "skl_universal_plane.h"
0034
0035 static void assert_vblank_disabled(struct drm_crtc *crtc)
0036 {
0037     if (I915_STATE_WARN_ON(drm_crtc_vblank_get(crtc) == 0))
0038         drm_crtc_vblank_put(crtc);
0039 }
0040
0041 struct intel_crtc *intel_first_crtc(struct drm_i915_private *i915)
0042 {
0043     return to_intel_crtc(drm_crtc_from_index(&i915->drm, 0));
0044 }
0045
0046 struct intel_crtc *intel_crtc_for_pipe(struct drm_i915_private *i915,
0047                        enum pipe pipe)
0048 {
0049     struct intel_crtc *crtc;
0050
0051     for_each_intel_crtc(&i915->drm, crtc) {
0052         if (crtc->pipe == pipe)
0053             return crtc;
0054     }
0055
0056     return NULL;
0057 }
0058
0059 void intel_crtc_wait_for_next_vblank(struct intel_crtc *crtc)
0060 {
0061     drm_crtc_wait_one_vblank(&crtc->base);
0062 }
0063
0064 void intel_wait_for_vblank_if_active(struct drm_i915_private *i915,
0065                      enum pipe pipe)
0066 {
0067     struct intel_crtc *crtc = intel_crtc_for_pipe(i915, pipe);
0068
0069     if (crtc->active)
0070         intel_crtc_wait_for_next_vblank(crtc);
0071 }
0072
0073 u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc)
0074 {
0075     struct drm_device *dev = crtc->base.dev;
0076     struct drm_vblank_crtc *vblank = &dev->vblank[drm_crtc_index(&crtc->base)];
0077
0078     if (!crtc->active)
0079         return 0;
0080
0081     if (!vblank->max_vblank_count)
0082         return (u32)drm_crtc_accurate_vblank_count(&crtc->base);
0083
0084     return crtc->base.funcs->get_vblank_counter(&crtc->base);
0085 }
0086
0087 u32 intel_crtc_max_vblank_count(const struct intel_crtc_state *crtc_state)
0088 {
0089     struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
0090
0091     /*
0092      * From Gen 11, In case of dsi cmd mode, frame counter wouldnt
0093      * have updated at the beginning of TE, if we want to use
0094      * the hw counter, then we would find it updated in only
0095      * the next TE, hence switching to sw counter.
0096      */
0097     if (crtc_state->mode_flags & (I915_MODE_FLAG_DSI_USE_TE0 |
0098                       I915_MODE_FLAG_DSI_USE_TE1))
0099         return 0;
0100
0101     /*
0102      * On i965gm the hardware frame counter reads
0103      * zero when the TV encoder is enabled :(
0104      */
0105     if (IS_I965GM(dev_priv) &&
0106         (crtc_state->output_types & BIT(INTEL_OUTPUT_TVOUT)))
0107         return 0;
0108
0109     if (DISPLAY_VER(dev_priv) >= 5 || IS_G4X(dev_priv))
0110         return 0xffffffff; /* full 32 bit counter */
0111     else if (DISPLAY_VER(dev_priv) >= 3)
0112         return 0xffffff; /* only 24 bits of frame count */
0113     else
0114         return 0; /* Gen2 doesn't have a hardware frame counter */
0115 }
0116
0117 void intel_crtc_vblank_on(const struct intel_crtc_state *crtc_state)
0118 {
0119     struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
0120
0121     assert_vblank_disabled(&crtc->base);
0122     drm_crtc_set_max_vblank_count(&crtc->base,
0123                       intel_crtc_max_vblank_count(crtc_state));
0124     drm_crtc_vblank_on(&crtc->base);
0125
0126     /*
0127      * Should really happen exactly when we enable the pipe
0128      * but we want the frame counters in the trace, and that
0129      * requires vblank support on some platforms/outputs.
0130      */
0131     trace_intel_pipe_enable(crtc);
0132 }
0133
0134 void intel_crtc_vblank_off(const struct intel_crtc_state *crtc_state)
0135 {
0136     struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
0137
0138     /*
0139      * Should really happen exactly when we disable the pipe
0140      * but we want the frame counters in the trace, and that
0141      * requires vblank support on some platforms/outputs.
0142      */
0143     trace_intel_pipe_disable(crtc);
0144
0145     drm_crtc_vblank_off(&crtc->base);
0146     assert_vblank_disabled(&crtc->base);
0147 }
0148
0149 struct intel_crtc_state *intel_crtc_state_alloc(struct intel_crtc *crtc)
0150 {
0151     struct intel_crtc_state *crtc_state;
0152
0153     crtc_state = kmalloc(sizeof(*crtc_state), GFP_KERNEL);
0154
0155     if (crtc_state)
0156         intel_crtc_state_reset(crtc_state, crtc);
0157
0158     return crtc_state;
0159 }
0160
0161 void intel_crtc_state_reset(struct intel_crtc_state *crtc_state,
0162                 struct intel_crtc *crtc)
0163 {
0164     memset(crtc_state, 0, sizeof(*crtc_state));
0165
0166     __drm_atomic_helper_crtc_state_reset(&crtc_state->uapi, &crtc->base);
0167
0168     crtc_state->cpu_transcoder = INVALID_TRANSCODER;
0169     crtc_state->master_transcoder = INVALID_TRANSCODER;
0170     crtc_state->hsw_workaround_pipe = INVALID_PIPE;
0171     crtc_state->scaler_state.scaler_id = -1;
0172     crtc_state->mst_master_transcoder = INVALID_TRANSCODER;
0173 }
0174
0175 static struct intel_crtc *intel_crtc_alloc(void)
0176 {
0177     struct intel_crtc_state *crtc_state;
0178     struct intel_crtc *crtc;
0179
0180     crtc = kzalloc(sizeof(*crtc), GFP_KERNEL);
0181     if (!crtc)
0182         return ERR_PTR(-ENOMEM);
0183
0184     crtc_state = intel_crtc_state_alloc(crtc);
0185     if (!crtc_state) {
0186         kfree(crtc);
0187         return ERR_PTR(-ENOMEM);
0188     }
0189
0190     crtc->base.state = &crtc_state->uapi;
0191     crtc->config = crtc_state;
0192
0193     return crtc;
0194 }
0195
0196 static void intel_crtc_free(struct intel_crtc *crtc)
0197 {
0198     intel_crtc_destroy_state(&crtc->base, crtc->base.state);
0199     kfree(crtc);
0200 }
0201
0202 static void intel_crtc_destroy(struct drm_crtc *_crtc)
0203 {
0204     struct intel_crtc *crtc = to_intel_crtc(_crtc);
0205
0206     cpu_latency_qos_remove_request(&crtc->vblank_pm_qos);
0207
0208     drm_crtc_cleanup(&crtc->base);
0209     kfree(crtc);
0210 }
0211
0212 static int intel_crtc_late_register(struct drm_crtc *crtc)
0213 {
0214     intel_crtc_debugfs_add(crtc);
0215     return 0;
0216 }
0217
0218 #define INTEL_CRTC_FUNCS \
0219     .set_config = drm_atomic_helper_set_config, \
0220     .destroy = intel_crtc_destroy, \
0221     .page_flip = drm_atomic_helper_page_flip, \
0222     .atomic_duplicate_state = intel_crtc_duplicate_state, \
0223     .atomic_destroy_state = intel_crtc_destroy_state, \
0224     .set_crc_source = intel_crtc_set_crc_source, \
0225     .verify_crc_source = intel_crtc_verify_crc_source, \
0226     .get_crc_sources = intel_crtc_get_crc_sources, \
0227     .late_register = intel_crtc_late_register
0228
0229 static const struct drm_crtc_funcs bdw_crtc_funcs = {
0230     INTEL_CRTC_FUNCS,
0231
0232     .get_vblank_counter = g4x_get_vblank_counter,
0233     .enable_vblank = bdw_enable_vblank,
0234     .disable_vblank = bdw_disable_vblank,
0235     .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
0236 };
0237
0238 static const struct drm_crtc_funcs ilk_crtc_funcs = {
0239     INTEL_CRTC_FUNCS,
0240
0241     .get_vblank_counter = g4x_get_vblank_counter,
0242     .enable_vblank = ilk_enable_vblank,
0243     .disable_vblank = ilk_disable_vblank,
0244     .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
0245 };
0246
0247 static const struct drm_crtc_funcs g4x_crtc_funcs = {
0248     INTEL_CRTC_FUNCS,
0249
0250     .get_vblank_counter = g4x_get_vblank_counter,
0251     .enable_vblank = i965_enable_vblank,
0252     .disable_vblank = i965_disable_vblank,
0253     .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
0254 };
0255
0256 static const struct drm_crtc_funcs i965_crtc_funcs = {
0257     INTEL_CRTC_FUNCS,
0258
0259     .get_vblank_counter = i915_get_vblank_counter,
0260     .enable_vblank = i965_enable_vblank,
0261     .disable_vblank = i965_disable_vblank,
0262     .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
0263 };
0264
0265 static const struct drm_crtc_funcs i915gm_crtc_funcs = {
0266     INTEL_CRTC_FUNCS,
0267
0268     .get_vblank_counter = i915_get_vblank_counter,
0269     .enable_vblank = i915gm_enable_vblank,
0270     .disable_vblank = i915gm_disable_vblank,
0271     .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
0272 };
0273
0274 static const struct drm_crtc_funcs i915_crtc_funcs = {
0275     INTEL_CRTC_FUNCS,
0276
0277     .get_vblank_counter = i915_get_vblank_counter,
0278     .enable_vblank = i8xx_enable_vblank,
0279     .disable_vblank = i8xx_disable_vblank,
0280     .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
0281 };
0282
0283 static const struct drm_crtc_funcs i8xx_crtc_funcs = {
0284     INTEL_CRTC_FUNCS,
0285
0286     /* no hw vblank counter */
0287     .enable_vblank = i8xx_enable_vblank,
0288     .disable_vblank = i8xx_disable_vblank,
0289     .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
0290 };
0291
0292 int intel_crtc_init(struct drm_i915_private *dev_priv, enum pipe pipe)
0293 {
0294     struct intel_plane *primary, *cursor;
0295     const struct drm_crtc_funcs *funcs;
0296     struct intel_crtc *crtc;
0297     int sprite, ret;
0298
0299     crtc = intel_crtc_alloc();
0300     if (IS_ERR(crtc))
0301         return PTR_ERR(crtc);
0302
0303     crtc->pipe = pipe;
0304     crtc->num_scalers = RUNTIME_INFO(dev_priv)->num_scalers[pipe];
0305
0306     if (DISPLAY_VER(dev_priv) >= 9)
0307         primary = skl_universal_plane_create(dev_priv, pipe,
0308                              PLANE_PRIMARY);
0309     else
0310         primary = intel_primary_plane_create(dev_priv, pipe);
0311     if (IS_ERR(primary)) {
0312         ret = PTR_ERR(primary);
0313         goto fail;
0314     }
0315     crtc->plane_ids_mask |= BIT(primary->id);
0316
0317     for_each_sprite(dev_priv, pipe, sprite) {
0318         struct intel_plane *plane;
0319
0320         if (DISPLAY_VER(dev_priv) >= 9)
0321             plane = skl_universal_plane_create(dev_priv, pipe,
0322                                PLANE_SPRITE0 + sprite);
0323         else
0324             plane = intel_sprite_plane_create(dev_priv, pipe, sprite);
0325         if (IS_ERR(plane)) {
0326             ret = PTR_ERR(plane);
0327             goto fail;
0328         }
0329         crtc->plane_ids_mask |= BIT(plane->id);
0330     }
0331
0332     cursor = intel_cursor_plane_create(dev_priv, pipe);
0333     if (IS_ERR(cursor)) {
0334         ret = PTR_ERR(cursor);
0335         goto fail;
0336     }
0337     crtc->plane_ids_mask |= BIT(cursor->id);
0338
0339     if (HAS_GMCH(dev_priv)) {
0340         if (IS_CHERRYVIEW(dev_priv) ||
0341             IS_VALLEYVIEW(dev_priv) || IS_G4X(dev_priv))
0342             funcs = &g4x_crtc_funcs;
0343         else if (DISPLAY_VER(dev_priv) == 4)
0344             funcs = &i965_crtc_funcs;
0345         else if (IS_I945GM(dev_priv) || IS_I915GM(dev_priv))
0346             funcs = &i915gm_crtc_funcs;
0347         else if (DISPLAY_VER(dev_priv) == 3)
0348             funcs = &i915_crtc_funcs;
0349         else
0350             funcs = &i8xx_crtc_funcs;
0351     } else {
0352         if (DISPLAY_VER(dev_priv) >= 8)
0353             funcs = &bdw_crtc_funcs;
0354         else
0355             funcs = &ilk_crtc_funcs;
0356     }
0357
0358     ret = drm_crtc_init_with_planes(&dev_priv->drm, &crtc->base,
0359                     &primary->base, &cursor->base,
0360                     funcs, "pipe %c", pipe_name(pipe));
0361     if (ret)
0362         goto fail;
0363
0364     if (DISPLAY_VER(dev_priv) >= 11)
0365         drm_crtc_create_scaling_filter_property(&crtc->base,
0366                         BIT(DRM_SCALING_FILTER_DEFAULT) |
0367                         BIT(DRM_SCALING_FILTER_NEAREST_NEIGHBOR));
0368
0369     intel_color_init(crtc);
0370
0371     intel_crtc_drrs_init(crtc);
0372     intel_crtc_crc_init(crtc);
0373
0374     cpu_latency_qos_add_request(&crtc->vblank_pm_qos, PM_QOS_DEFAULT_VALUE);
0375
0376     drm_WARN_ON(&dev_priv->drm, drm_crtc_index(&crtc->base) != crtc->pipe);
0377
0378     return 0;
0379
0380 fail:
0381     intel_crtc_free(crtc);
0382
0383     return ret;
0384 }
0385
0386 static bool intel_crtc_needs_vblank_work(const struct intel_crtc_state *crtc_state)
0387 {
0388     return crtc_state->hw.active &&
0389         !intel_crtc_needs_modeset(crtc_state) &&
0390         !crtc_state->preload_luts &&
0391         (crtc_state->uapi.color_mgmt_changed ||
0392          crtc_state->update_pipe);
0393 }
0394
0395 static void intel_crtc_vblank_work(struct kthread_work *base)
0396 {
0397     struct drm_vblank_work *work = to_drm_vblank_work(base);
0398     struct intel_crtc_state *crtc_state =
0399         container_of(work, typeof(*crtc_state), vblank_work);
0400     struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
0401
0402     trace_intel_crtc_vblank_work_start(crtc);
0403
0404     intel_color_load_luts(crtc_state);
0405
0406     if (crtc_state->uapi.event) {
0407         spin_lock_irq(&crtc->base.dev->event_lock);
0408         drm_crtc_send_vblank_event(&crtc->base, crtc_state->uapi.event);
0409         crtc_state->uapi.event = NULL;
0410         spin_unlock_irq(&crtc->base.dev->event_lock);
0411     }
0412
0413     trace_intel_crtc_vblank_work_end(crtc);
0414 }
0415
0416 static void intel_crtc_vblank_work_init(struct intel_crtc_state *crtc_state)
0417 {
0418     struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
0419
0420     drm_vblank_work_init(&crtc_state->vblank_work, &crtc->base,
0421                  intel_crtc_vblank_work);
0422     /*
0423      * Interrupt latency is critical for getting the vblank
0424      * work executed as early as possible during the vblank.
0425      */
0426     cpu_latency_qos_update_request(&crtc->vblank_pm_qos, 0);
0427 }
0428
0429 void intel_wait_for_vblank_workers(struct intel_atomic_state *state)
0430 {
0431     struct intel_crtc_state *crtc_state;
0432     struct intel_crtc *crtc;
0433     int i;
0434
0435     for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
0436         if (!intel_crtc_needs_vblank_work(crtc_state))
0437             continue;
0438
0439         drm_vblank_work_flush(&crtc_state->vblank_work);
0440         cpu_latency_qos_update_request(&crtc->vblank_pm_qos,
0441                            PM_QOS_DEFAULT_VALUE);
0442     }
0443 }
0444
0445 int intel_usecs_to_scanlines(const struct drm_display_mode *adjusted_mode,
0446                  int usecs)
0447 {
0448     /* paranoia */
0449     if (!adjusted_mode->crtc_htotal)
0450         return 1;
0451
0452     return DIV_ROUND_UP(usecs * adjusted_mode->crtc_clock,
0453                 1000 * adjusted_mode->crtc_htotal);
0454 }
0455
0456 static int intel_mode_vblank_start(const struct drm_display_mode *mode)
0457 {
0458     int vblank_start = mode->crtc_vblank_start;
0459
0460     if (mode->flags & DRM_MODE_FLAG_INTERLACE)
0461         vblank_start = DIV_ROUND_UP(vblank_start, 2);
0462
0463     return vblank_start;
0464 }
0465
0466 /**
0467  * intel_pipe_update_start() - start update of a set of display registers
0468  * @new_crtc_state: the new crtc state
0469  *
0470  * Mark the start of an update to pipe registers that should be updated
0471  * atomically regarding vblank. If the next vblank will happens within
0472  * the next 100 us, this function waits until the vblank passes.
0473  *
0474  * After a successful call to this function, interrupts will be disabled
0475  * until a subsequent call to intel_pipe_update_end(). That is done to
0476  * avoid random delays.
0477  */
0478 void intel_pipe_update_start(struct intel_crtc_state *new_crtc_state)
0479 {
0480     struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
0481     struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
0482     const struct drm_display_mode *adjusted_mode = &new_crtc_state->hw.adjusted_mode;
0483     long timeout = msecs_to_jiffies_timeout(1);
0484     int scanline, min, max, vblank_start;
0485     wait_queue_head_t *wq = drm_crtc_vblank_waitqueue(&crtc->base);
0486     bool need_vlv_dsi_wa = (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
0487         intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI);
0488     DEFINE_WAIT(wait);
0489
0490     intel_psr_lock(new_crtc_state);
0491
0492     if (new_crtc_state->do_async_flip)
0493         return;
0494
0495     if (intel_crtc_needs_vblank_work(new_crtc_state))
0496         intel_crtc_vblank_work_init(new_crtc_state);
0497
0498     if (new_crtc_state->vrr.enable) {
0499         if (intel_vrr_is_push_sent(new_crtc_state))
0500             vblank_start = intel_vrr_vmin_vblank_start(new_crtc_state);
0501         else
0502             vblank_start = intel_vrr_vmax_vblank_start(new_crtc_state);
0503     } else {
0504         vblank_start = intel_mode_vblank_start(adjusted_mode);
0505     }
0506
0507     /* FIXME needs to be calibrated sensibly */
0508     min = vblank_start - intel_usecs_to_scanlines(adjusted_mode,
0509                               VBLANK_EVASION_TIME_US);
0510     max = vblank_start - 1;
0511
0512     if (min <= 0 || max <= 0)
0513         goto irq_disable;
0514
0515     if (drm_WARN_ON(&dev_priv->drm, drm_crtc_vblank_get(&crtc->base)))
0516         goto irq_disable;
0517
0518     /*
0519      * Wait for psr to idle out after enabling the VBL interrupts
0520      * VBL interrupts will start the PSR exit and prevent a PSR
0521      * re-entry as well.
0522      */
0523     intel_psr_wait_for_idle_locked(new_crtc_state);
0524
0525     local_irq_disable();
0526
0527     crtc->debug.min_vbl = min;
0528     crtc->debug.max_vbl = max;
0529     trace_intel_pipe_update_start(crtc);
0530
0531     for (;;) {
0532         /*
0533          * prepare_to_wait() has a memory barrier, which guarantees
0534          * other CPUs can see the task state update by the time we
0535          * read the scanline.
0536          */
0537         prepare_to_wait(wq, &wait, TASK_UNINTERRUPTIBLE);
0538
0539         scanline = intel_get_crtc_scanline(crtc);
0540         if (scanline < min || scanline > max)
0541             break;
0542
0543         if (!timeout) {
0544             drm_err(&dev_priv->drm,
0545                 "Potential atomic update failure on pipe %c\n",
0546                 pipe_name(crtc->pipe));
0547             break;
0548         }
0549
0550         local_irq_enable();
0551
0552         timeout = schedule_timeout(timeout);
0553
0554         local_irq_disable();
0555     }
0556
0557     finish_wait(wq, &wait);
0558
0559     drm_crtc_vblank_put(&crtc->base);
0560
0561     /*
0562      * On VLV/CHV DSI the scanline counter would appear to
0563      * increment approx. 1/3 of a scanline before start of vblank.
0564      * The registers still get latched at start of vblank however.
0565      * This means we must not write any registers on the first
0566      * line of vblank (since not the whole line is actually in
0567      * vblank). And unfortunately we can't use the interrupt to
0568      * wait here since it will fire too soon. We could use the
0569      * frame start interrupt instead since it will fire after the
0570      * critical scanline, but that would require more changes
0571      * in the interrupt code. So for now we'll just do the nasty
0572      * thing and poll for the bad scanline to pass us by.
0573      *
0574      * FIXME figure out if BXT+ DSI suffers from this as well
0575      */
0576     while (need_vlv_dsi_wa && scanline == vblank_start)
0577         scanline = intel_get_crtc_scanline(crtc);
0578
0579     crtc->debug.scanline_start = scanline;
0580     crtc->debug.start_vbl_time = ktime_get();
0581     crtc->debug.start_vbl_count = intel_crtc_get_vblank_counter(crtc);
0582
0583     trace_intel_pipe_update_vblank_evaded(crtc);
0584     return;
0585
0586 irq_disable:
0587     local_irq_disable();
0588 }
0589
0590 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_VBLANK_EVADE)
0591 static void dbg_vblank_evade(struct intel_crtc *crtc, ktime_t end)
0592 {
0593     u64 delta = ktime_to_ns(ktime_sub(end, crtc->debug.start_vbl_time));
0594     unsigned int h;
0595
0596     h = ilog2(delta >> 9);
0597     if (h >= ARRAY_SIZE(crtc->debug.vbl.times))
0598         h = ARRAY_SIZE(crtc->debug.vbl.times) - 1;
0599     crtc->debug.vbl.times[h]++;
0600
0601     crtc->debug.vbl.sum += delta;
0602     if (!crtc->debug.vbl.min || delta < crtc->debug.vbl.min)
0603         crtc->debug.vbl.min = delta;
0604     if (delta > crtc->debug.vbl.max)
0605         crtc->debug.vbl.max = delta;
0606
0607     if (delta > 1000 * VBLANK_EVASION_TIME_US) {
0608         drm_dbg_kms(crtc->base.dev,
0609                 "Atomic update on pipe (%c) took %lld us, max time under evasion is %u us\n",
0610                 pipe_name(crtc->pipe),
0611                 div_u64(delta, 1000),
0612                 VBLANK_EVASION_TIME_US);
0613         crtc->debug.vbl.over++;
0614     }
0615 }
0616 #else
0617 static void dbg_vblank_evade(struct intel_crtc *crtc, ktime_t end) {}
0618 #endif
0619
0620 /**
0621  * intel_pipe_update_end() - end update of a set of display registers
0622  * @new_crtc_state: the new crtc state
0623  *
0624  * Mark the end of an update started with intel_pipe_update_start(). This
0625  * re-enables interrupts and verifies the update was actually completed
0626  * before a vblank.
0627  */
0628 void intel_pipe_update_end(struct intel_crtc_state *new_crtc_state)
0629 {
0630     struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
0631     enum pipe pipe = crtc->pipe;
0632     int scanline_end = intel_get_crtc_scanline(crtc);
0633     u32 end_vbl_count = intel_crtc_get_vblank_counter(crtc);
0634     ktime_t end_vbl_time = ktime_get();
0635     struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
0636
0637     intel_psr_unlock(new_crtc_state);
0638
0639     if (new_crtc_state->do_async_flip)
0640         return;
0641
0642     trace_intel_pipe_update_end(crtc, end_vbl_count, scanline_end);
0643
0644     /*
0645      * Incase of mipi dsi command mode, we need to set frame update
0646      * request for every commit.
0647      */
0648     if (DISPLAY_VER(dev_priv) >= 11 &&
0649         intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI))
0650         icl_dsi_frame_update(new_crtc_state);
0651
0652     /* We're still in the vblank-evade critical section, this can't race.
0653      * Would be slightly nice to just grab the vblank count and arm the
0654      * event outside of the critical section - the spinlock might spin for a
0655      * while ... */
0656     if (intel_crtc_needs_vblank_work(new_crtc_state)) {
0657         drm_vblank_work_schedule(&new_crtc_state->vblank_work,
0658                      drm_crtc_accurate_vblank_count(&crtc->base) + 1,
0659                      false);
0660     } else if (new_crtc_state->uapi.event) {
0661         drm_WARN_ON(&dev_priv->drm,
0662                 drm_crtc_vblank_get(&crtc->base) != 0);
0663
0664         spin_lock(&crtc->base.dev->event_lock);
0665         drm_crtc_arm_vblank_event(&crtc->base,
0666                       new_crtc_state->uapi.event);
0667         spin_unlock(&crtc->base.dev->event_lock);
0668
0669         new_crtc_state->uapi.event = NULL;
0670     }
0671
0672     /*
0673      * Send VRR Push to terminate Vblank. If we are already in vblank
0674      * this has to be done _after_ sampling the frame counter, as
0675      * otherwise the push would immediately terminate the vblank and
0676      * the sampled frame counter would correspond to the next frame
0677      * instead of the current frame.
0678      *
0679      * There is a tiny race here (iff vblank evasion failed us) where
0680      * we might sample the frame counter just before vmax vblank start
0681      * but the push would be sent just after it. That would cause the
0682      * push to affect the next frame instead of the current frame,
0683      * which would cause the next frame to terminate already at vmin
0684      * vblank start instead of vmax vblank start.
0685      */
0686     intel_vrr_send_push(new_crtc_state);
0687
0688     local_irq_enable();
0689
0690     if (intel_vgpu_active(dev_priv))
0691         return;
0692
0693     if (crtc->debug.start_vbl_count &&
0694         crtc->debug.start_vbl_count != end_vbl_count) {
0695         drm_err(&dev_priv->drm,
0696             "Atomic update failure on pipe %c (start=%u end=%u) time %lld us, min %d, max %d, scanline start %d, end %d\n",
0697             pipe_name(pipe), crtc->debug.start_vbl_count,
0698             end_vbl_count,
0699             ktime_us_delta(end_vbl_time,
0700                        crtc->debug.start_vbl_time),
0701             crtc->debug.min_vbl, crtc->debug.max_vbl,
0702             crtc->debug.scanline_start, scanline_end);
0703     }
0704
0705     dbg_vblank_evade(crtc, end_vbl_time);
0706 }