Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: MIT
0002 /*
0003  * Copyright © 2019 Intel Corporation
0004  */
0005 
0006 #include <linux/string_helpers.h>
0007 
0008 #include <drm/i915_drm.h>
0009 
0010 #include "i915_drv.h"
0011 #include "i915_irq.h"
0012 #include "intel_breadcrumbs.h"
0013 #include "intel_gt.h"
0014 #include "intel_gt_clock_utils.h"
0015 #include "intel_gt_irq.h"
0016 #include "intel_gt_pm_irq.h"
0017 #include "intel_gt_regs.h"
0018 #include "intel_mchbar_regs.h"
0019 #include "intel_pcode.h"
0020 #include "intel_rps.h"
0021 #include "vlv_sideband.h"
0022 #include "../../../platform/x86/intel_ips.h"
0023 
0024 #define BUSY_MAX_EI 20u /* ms */
0025 
0026 /*
0027  * Lock protecting IPS related data structures
0028  */
0029 static DEFINE_SPINLOCK(mchdev_lock);
0030 
0031 static struct intel_gt *rps_to_gt(struct intel_rps *rps)
0032 {
0033     return container_of(rps, struct intel_gt, rps);
0034 }
0035 
0036 static struct drm_i915_private *rps_to_i915(struct intel_rps *rps)
0037 {
0038     return rps_to_gt(rps)->i915;
0039 }
0040 
0041 static struct intel_uncore *rps_to_uncore(struct intel_rps *rps)
0042 {
0043     return rps_to_gt(rps)->uncore;
0044 }
0045 
0046 static struct intel_guc_slpc *rps_to_slpc(struct intel_rps *rps)
0047 {
0048     struct intel_gt *gt = rps_to_gt(rps);
0049 
0050     return &gt->uc.guc.slpc;
0051 }
0052 
0053 static bool rps_uses_slpc(struct intel_rps *rps)
0054 {
0055     struct intel_gt *gt = rps_to_gt(rps);
0056 
0057     return intel_uc_uses_guc_slpc(&gt->uc);
0058 }
0059 
0060 static u32 rps_pm_sanitize_mask(struct intel_rps *rps, u32 mask)
0061 {
0062     return mask & ~rps->pm_intrmsk_mbz;
0063 }
0064 
0065 static void set(struct intel_uncore *uncore, i915_reg_t reg, u32 val)
0066 {
0067     intel_uncore_write_fw(uncore, reg, val);
0068 }
0069 
0070 static void rps_timer(struct timer_list *t)
0071 {
0072     struct intel_rps *rps = from_timer(rps, t, timer);
0073     struct intel_engine_cs *engine;
0074     ktime_t dt, last, timestamp;
0075     enum intel_engine_id id;
0076     s64 max_busy[3] = {};
0077 
0078     timestamp = 0;
0079     for_each_engine(engine, rps_to_gt(rps), id) {
0080         s64 busy;
0081         int i;
0082 
0083         dt = intel_engine_get_busy_time(engine, &timestamp);
0084         last = engine->stats.rps;
0085         engine->stats.rps = dt;
0086 
0087         busy = ktime_to_ns(ktime_sub(dt, last));
0088         for (i = 0; i < ARRAY_SIZE(max_busy); i++) {
0089             if (busy > max_busy[i])
0090                 swap(busy, max_busy[i]);
0091         }
0092     }
0093     last = rps->pm_timestamp;
0094     rps->pm_timestamp = timestamp;
0095 
0096     if (intel_rps_is_active(rps)) {
0097         s64 busy;
0098         int i;
0099 
0100         dt = ktime_sub(timestamp, last);
0101 
0102         /*
0103          * Our goal is to evaluate each engine independently, so we run
0104          * at the lowest clocks required to sustain the heaviest
0105          * workload. However, a task may be split into sequential
0106          * dependent operations across a set of engines, such that
0107          * the independent contributions do not account for high load,
0108          * but overall the task is GPU bound. For example, consider
0109          * video decode on vcs followed by colour post-processing
0110          * on vecs, followed by general post-processing on rcs.
0111          * Since multi-engines being active does imply a single
0112          * continuous workload across all engines, we hedge our
0113          * bets by only contributing a factor of the distributed
0114          * load into our busyness calculation.
0115          */
0116         busy = max_busy[0];
0117         for (i = 1; i < ARRAY_SIZE(max_busy); i++) {
0118             if (!max_busy[i])
0119                 break;
0120 
0121             busy += div_u64(max_busy[i], 1 << i);
0122         }
0123         GT_TRACE(rps_to_gt(rps),
0124              "busy:%lld [%d%%], max:[%lld, %lld, %lld], interval:%d\n",
0125              busy, (int)div64_u64(100 * busy, dt),
0126              max_busy[0], max_busy[1], max_busy[2],
0127              rps->pm_interval);
0128 
0129         if (100 * busy > rps->power.up_threshold * dt &&
0130             rps->cur_freq < rps->max_freq_softlimit) {
0131             rps->pm_iir |= GEN6_PM_RP_UP_THRESHOLD;
0132             rps->pm_interval = 1;
0133             schedule_work(&rps->work);
0134         } else if (100 * busy < rps->power.down_threshold * dt &&
0135                rps->cur_freq > rps->min_freq_softlimit) {
0136             rps->pm_iir |= GEN6_PM_RP_DOWN_THRESHOLD;
0137             rps->pm_interval = 1;
0138             schedule_work(&rps->work);
0139         } else {
0140             rps->last_adj = 0;
0141         }
0142 
0143         mod_timer(&rps->timer,
0144               jiffies + msecs_to_jiffies(rps->pm_interval));
0145         rps->pm_interval = min(rps->pm_interval * 2, BUSY_MAX_EI);
0146     }
0147 }
0148 
0149 static void rps_start_timer(struct intel_rps *rps)
0150 {
0151     rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp);
0152     rps->pm_interval = 1;
0153     mod_timer(&rps->timer, jiffies + 1);
0154 }
0155 
0156 static void rps_stop_timer(struct intel_rps *rps)
0157 {
0158     del_timer_sync(&rps->timer);
0159     rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp);
0160     cancel_work_sync(&rps->work);
0161 }
0162 
0163 static u32 rps_pm_mask(struct intel_rps *rps, u8 val)
0164 {
0165     u32 mask = 0;
0166 
0167     /* We use UP_EI_EXPIRED interrupts for both up/down in manual mode */
0168     if (val > rps->min_freq_softlimit)
0169         mask |= (GEN6_PM_RP_UP_EI_EXPIRED |
0170              GEN6_PM_RP_DOWN_THRESHOLD |
0171              GEN6_PM_RP_DOWN_TIMEOUT);
0172 
0173     if (val < rps->max_freq_softlimit)
0174         mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD;
0175 
0176     mask &= rps->pm_events;
0177 
0178     return rps_pm_sanitize_mask(rps, ~mask);
0179 }
0180 
0181 static void rps_reset_ei(struct intel_rps *rps)
0182 {
0183     memset(&rps->ei, 0, sizeof(rps->ei));
0184 }
0185 
0186 static void rps_enable_interrupts(struct intel_rps *rps)
0187 {
0188     struct intel_gt *gt = rps_to_gt(rps);
0189 
0190     GEM_BUG_ON(rps_uses_slpc(rps));
0191 
0192     GT_TRACE(gt, "interrupts:on rps->pm_events: %x, rps_pm_mask:%x\n",
0193          rps->pm_events, rps_pm_mask(rps, rps->last_freq));
0194 
0195     rps_reset_ei(rps);
0196 
0197     spin_lock_irq(&gt->irq_lock);
0198     gen6_gt_pm_enable_irq(gt, rps->pm_events);
0199     spin_unlock_irq(&gt->irq_lock);
0200 
0201     intel_uncore_write(gt->uncore,
0202                GEN6_PMINTRMSK, rps_pm_mask(rps, rps->last_freq));
0203 }
0204 
0205 static void gen6_rps_reset_interrupts(struct intel_rps *rps)
0206 {
0207     gen6_gt_pm_reset_iir(rps_to_gt(rps), GEN6_PM_RPS_EVENTS);
0208 }
0209 
0210 static void gen11_rps_reset_interrupts(struct intel_rps *rps)
0211 {
0212     while (gen11_gt_reset_one_iir(rps_to_gt(rps), 0, GEN11_GTPM))
0213         ;
0214 }
0215 
0216 static void rps_reset_interrupts(struct intel_rps *rps)
0217 {
0218     struct intel_gt *gt = rps_to_gt(rps);
0219 
0220     spin_lock_irq(&gt->irq_lock);
0221     if (GRAPHICS_VER(gt->i915) >= 11)
0222         gen11_rps_reset_interrupts(rps);
0223     else
0224         gen6_rps_reset_interrupts(rps);
0225 
0226     rps->pm_iir = 0;
0227     spin_unlock_irq(&gt->irq_lock);
0228 }
0229 
0230 static void rps_disable_interrupts(struct intel_rps *rps)
0231 {
0232     struct intel_gt *gt = rps_to_gt(rps);
0233 
0234     intel_uncore_write(gt->uncore,
0235                GEN6_PMINTRMSK, rps_pm_sanitize_mask(rps, ~0u));
0236 
0237     spin_lock_irq(&gt->irq_lock);
0238     gen6_gt_pm_disable_irq(gt, GEN6_PM_RPS_EVENTS);
0239     spin_unlock_irq(&gt->irq_lock);
0240 
0241     intel_synchronize_irq(gt->i915);
0242 
0243     /*
0244      * Now that we will not be generating any more work, flush any
0245      * outstanding tasks. As we are called on the RPS idle path,
0246      * we will reset the GPU to minimum frequencies, so the current
0247      * state of the worker can be discarded.
0248      */
0249     cancel_work_sync(&rps->work);
0250 
0251     rps_reset_interrupts(rps);
0252     GT_TRACE(gt, "interrupts:off\n");
0253 }
0254 
0255 static const struct cparams {
0256     u16 i;
0257     u16 t;
0258     u16 m;
0259     u16 c;
0260 } cparams[] = {
0261     { 1, 1333, 301, 28664 },
0262     { 1, 1066, 294, 24460 },
0263     { 1, 800, 294, 25192 },
0264     { 0, 1333, 276, 27605 },
0265     { 0, 1066, 276, 27605 },
0266     { 0, 800, 231, 23784 },
0267 };
0268 
0269 static void gen5_rps_init(struct intel_rps *rps)
0270 {
0271     struct drm_i915_private *i915 = rps_to_i915(rps);
0272     struct intel_uncore *uncore = rps_to_uncore(rps);
0273     u8 fmax, fmin, fstart;
0274     u32 rgvmodectl;
0275     int c_m, i;
0276 
0277     if (i915->fsb_freq <= 3200)
0278         c_m = 0;
0279     else if (i915->fsb_freq <= 4800)
0280         c_m = 1;
0281     else
0282         c_m = 2;
0283 
0284     for (i = 0; i < ARRAY_SIZE(cparams); i++) {
0285         if (cparams[i].i == c_m && cparams[i].t == i915->mem_freq) {
0286             rps->ips.m = cparams[i].m;
0287             rps->ips.c = cparams[i].c;
0288             break;
0289         }
0290     }
0291 
0292     rgvmodectl = intel_uncore_read(uncore, MEMMODECTL);
0293 
0294     /* Set up min, max, and cur for interrupt handling */
0295     fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
0296     fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
0297     fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
0298         MEMMODE_FSTART_SHIFT;
0299     drm_dbg(&i915->drm, "fmax: %d, fmin: %d, fstart: %d\n",
0300         fmax, fmin, fstart);
0301 
0302     rps->min_freq = fmax;
0303     rps->efficient_freq = fstart;
0304     rps->max_freq = fmin;
0305 }
0306 
0307 static unsigned long
0308 __ips_chipset_val(struct intel_ips *ips)
0309 {
0310     struct intel_uncore *uncore =
0311         rps_to_uncore(container_of(ips, struct intel_rps, ips));
0312     unsigned long now = jiffies_to_msecs(jiffies), dt;
0313     unsigned long result;
0314     u64 total, delta;
0315 
0316     lockdep_assert_held(&mchdev_lock);
0317 
0318     /*
0319      * Prevent division-by-zero if we are asking too fast.
0320      * Also, we don't get interesting results if we are polling
0321      * faster than once in 10ms, so just return the saved value
0322      * in such cases.
0323      */
0324     dt = now - ips->last_time1;
0325     if (dt <= 10)
0326         return ips->chipset_power;
0327 
0328     /* FIXME: handle per-counter overflow */
0329     total = intel_uncore_read(uncore, DMIEC);
0330     total += intel_uncore_read(uncore, DDREC);
0331     total += intel_uncore_read(uncore, CSIEC);
0332 
0333     delta = total - ips->last_count1;
0334 
0335     result = div_u64(div_u64(ips->m * delta, dt) + ips->c, 10);
0336 
0337     ips->last_count1 = total;
0338     ips->last_time1 = now;
0339 
0340     ips->chipset_power = result;
0341 
0342     return result;
0343 }
0344 
0345 static unsigned long ips_mch_val(struct intel_uncore *uncore)
0346 {
0347     unsigned int m, x, b;
0348     u32 tsfs;
0349 
0350     tsfs = intel_uncore_read(uncore, TSFS);
0351     x = intel_uncore_read8(uncore, TR1);
0352 
0353     b = tsfs & TSFS_INTR_MASK;
0354     m = (tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT;
0355 
0356     return m * x / 127 - b;
0357 }
0358 
0359 static int _pxvid_to_vd(u8 pxvid)
0360 {
0361     if (pxvid == 0)
0362         return 0;
0363 
0364     if (pxvid >= 8 && pxvid < 31)
0365         pxvid = 31;
0366 
0367     return (pxvid + 2) * 125;
0368 }
0369 
0370 static u32 pvid_to_extvid(struct drm_i915_private *i915, u8 pxvid)
0371 {
0372     const int vd = _pxvid_to_vd(pxvid);
0373 
0374     if (INTEL_INFO(i915)->is_mobile)
0375         return max(vd - 1125, 0);
0376 
0377     return vd;
0378 }
0379 
0380 static void __gen5_ips_update(struct intel_ips *ips)
0381 {
0382     struct intel_uncore *uncore =
0383         rps_to_uncore(container_of(ips, struct intel_rps, ips));
0384     u64 now, delta, dt;
0385     u32 count;
0386 
0387     lockdep_assert_held(&mchdev_lock);
0388 
0389     now = ktime_get_raw_ns();
0390     dt = now - ips->last_time2;
0391     do_div(dt, NSEC_PER_MSEC);
0392 
0393     /* Don't divide by 0 */
0394     if (dt <= 10)
0395         return;
0396 
0397     count = intel_uncore_read(uncore, GFXEC);
0398     delta = count - ips->last_count2;
0399 
0400     ips->last_count2 = count;
0401     ips->last_time2 = now;
0402 
0403     /* More magic constants... */
0404     ips->gfx_power = div_u64(delta * 1181, dt * 10);
0405 }
0406 
0407 static void gen5_rps_update(struct intel_rps *rps)
0408 {
0409     spin_lock_irq(&mchdev_lock);
0410     __gen5_ips_update(&rps->ips);
0411     spin_unlock_irq(&mchdev_lock);
0412 }
0413 
0414 static unsigned int gen5_invert_freq(struct intel_rps *rps,
0415                      unsigned int val)
0416 {
0417     /* Invert the frequency bin into an ips delay */
0418     val = rps->max_freq - val;
0419     val = rps->min_freq + val;
0420 
0421     return val;
0422 }
0423 
0424 static int __gen5_rps_set(struct intel_rps *rps, u8 val)
0425 {
0426     struct intel_uncore *uncore = rps_to_uncore(rps);
0427     u16 rgvswctl;
0428 
0429     lockdep_assert_held(&mchdev_lock);
0430 
0431     rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
0432     if (rgvswctl & MEMCTL_CMD_STS) {
0433         DRM_DEBUG("gpu busy, RCS change rejected\n");
0434         return -EBUSY; /* still busy with another command */
0435     }
0436 
0437     /* Invert the frequency bin into an ips delay */
0438     val = gen5_invert_freq(rps, val);
0439 
0440     rgvswctl =
0441         (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) |
0442         (val << MEMCTL_FREQ_SHIFT) |
0443         MEMCTL_SFCAVM;
0444     intel_uncore_write16(uncore, MEMSWCTL, rgvswctl);
0445     intel_uncore_posting_read16(uncore, MEMSWCTL);
0446 
0447     rgvswctl |= MEMCTL_CMD_STS;
0448     intel_uncore_write16(uncore, MEMSWCTL, rgvswctl);
0449 
0450     return 0;
0451 }
0452 
0453 static int gen5_rps_set(struct intel_rps *rps, u8 val)
0454 {
0455     int err;
0456 
0457     spin_lock_irq(&mchdev_lock);
0458     err = __gen5_rps_set(rps, val);
0459     spin_unlock_irq(&mchdev_lock);
0460 
0461     return err;
0462 }
0463 
0464 static unsigned long intel_pxfreq(u32 vidfreq)
0465 {
0466     int div = (vidfreq & 0x3f0000) >> 16;
0467     int post = (vidfreq & 0x3000) >> 12;
0468     int pre = (vidfreq & 0x7);
0469 
0470     if (!pre)
0471         return 0;
0472 
0473     return div * 133333 / (pre << post);
0474 }
0475 
0476 static unsigned int init_emon(struct intel_uncore *uncore)
0477 {
0478     u8 pxw[16];
0479     int i;
0480 
0481     /* Disable to program */
0482     intel_uncore_write(uncore, ECR, 0);
0483     intel_uncore_posting_read(uncore, ECR);
0484 
0485     /* Program energy weights for various events */
0486     intel_uncore_write(uncore, SDEW, 0x15040d00);
0487     intel_uncore_write(uncore, CSIEW0, 0x007f0000);
0488     intel_uncore_write(uncore, CSIEW1, 0x1e220004);
0489     intel_uncore_write(uncore, CSIEW2, 0x04000004);
0490 
0491     for (i = 0; i < 5; i++)
0492         intel_uncore_write(uncore, PEW(i), 0);
0493     for (i = 0; i < 3; i++)
0494         intel_uncore_write(uncore, DEW(i), 0);
0495 
0496     /* Program P-state weights to account for frequency power adjustment */
0497     for (i = 0; i < 16; i++) {
0498         u32 pxvidfreq = intel_uncore_read(uncore, PXVFREQ(i));
0499         unsigned int freq = intel_pxfreq(pxvidfreq);
0500         unsigned int vid =
0501             (pxvidfreq & PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT;
0502         unsigned int val;
0503 
0504         val = vid * vid * freq / 1000 * 255;
0505         val /= 127 * 127 * 900;
0506 
0507         pxw[i] = val;
0508     }
0509     /* Render standby states get 0 weight */
0510     pxw[14] = 0;
0511     pxw[15] = 0;
0512 
0513     for (i = 0; i < 4; i++) {
0514         intel_uncore_write(uncore, PXW(i),
0515                    pxw[i * 4 + 0] << 24 |
0516                    pxw[i * 4 + 1] << 16 |
0517                    pxw[i * 4 + 2] <<  8 |
0518                    pxw[i * 4 + 3] <<  0);
0519     }
0520 
0521     /* Adjust magic regs to magic values (more experimental results) */
0522     intel_uncore_write(uncore, OGW0, 0);
0523     intel_uncore_write(uncore, OGW1, 0);
0524     intel_uncore_write(uncore, EG0, 0x00007f00);
0525     intel_uncore_write(uncore, EG1, 0x0000000e);
0526     intel_uncore_write(uncore, EG2, 0x000e0000);
0527     intel_uncore_write(uncore, EG3, 0x68000300);
0528     intel_uncore_write(uncore, EG4, 0x42000000);
0529     intel_uncore_write(uncore, EG5, 0x00140031);
0530     intel_uncore_write(uncore, EG6, 0);
0531     intel_uncore_write(uncore, EG7, 0);
0532 
0533     for (i = 0; i < 8; i++)
0534         intel_uncore_write(uncore, PXWL(i), 0);
0535 
0536     /* Enable PMON + select events */
0537     intel_uncore_write(uncore, ECR, 0x80000019);
0538 
0539     return intel_uncore_read(uncore, LCFUSE02) & LCFUSE_HIV_MASK;
0540 }
0541 
0542 static bool gen5_rps_enable(struct intel_rps *rps)
0543 {
0544     struct drm_i915_private *i915 = rps_to_i915(rps);
0545     struct intel_uncore *uncore = rps_to_uncore(rps);
0546     u8 fstart, vstart;
0547     u32 rgvmodectl;
0548 
0549     spin_lock_irq(&mchdev_lock);
0550 
0551     rgvmodectl = intel_uncore_read(uncore, MEMMODECTL);
0552 
0553     /* Enable temp reporting */
0554     intel_uncore_write16(uncore, PMMISC,
0555                  intel_uncore_read16(uncore, PMMISC) | MCPPCE_EN);
0556     intel_uncore_write16(uncore, TSC1,
0557                  intel_uncore_read16(uncore, TSC1) | TSE);
0558 
0559     /* 100ms RC evaluation intervals */
0560     intel_uncore_write(uncore, RCUPEI, 100000);
0561     intel_uncore_write(uncore, RCDNEI, 100000);
0562 
0563     /* Set max/min thresholds to 90ms and 80ms respectively */
0564     intel_uncore_write(uncore, RCBMAXAVG, 90000);
0565     intel_uncore_write(uncore, RCBMINAVG, 80000);
0566 
0567     intel_uncore_write(uncore, MEMIHYST, 1);
0568 
0569     /* Set up min, max, and cur for interrupt handling */
0570     fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
0571         MEMMODE_FSTART_SHIFT;
0572 
0573     vstart = (intel_uncore_read(uncore, PXVFREQ(fstart)) &
0574           PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT;
0575 
0576     intel_uncore_write(uncore,
0577                MEMINTREN,
0578                MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN);
0579 
0580     intel_uncore_write(uncore, VIDSTART, vstart);
0581     intel_uncore_posting_read(uncore, VIDSTART);
0582 
0583     rgvmodectl |= MEMMODE_SWMODE_EN;
0584     intel_uncore_write(uncore, MEMMODECTL, rgvmodectl);
0585 
0586     if (wait_for_atomic((intel_uncore_read(uncore, MEMSWCTL) &
0587                  MEMCTL_CMD_STS) == 0, 10))
0588         drm_err(&uncore->i915->drm,
0589             "stuck trying to change perf mode\n");
0590     mdelay(1);
0591 
0592     __gen5_rps_set(rps, rps->cur_freq);
0593 
0594     rps->ips.last_count1 = intel_uncore_read(uncore, DMIEC);
0595     rps->ips.last_count1 += intel_uncore_read(uncore, DDREC);
0596     rps->ips.last_count1 += intel_uncore_read(uncore, CSIEC);
0597     rps->ips.last_time1 = jiffies_to_msecs(jiffies);
0598 
0599     rps->ips.last_count2 = intel_uncore_read(uncore, GFXEC);
0600     rps->ips.last_time2 = ktime_get_raw_ns();
0601 
0602     spin_lock(&i915->irq_lock);
0603     ilk_enable_display_irq(i915, DE_PCU_EVENT);
0604     spin_unlock(&i915->irq_lock);
0605 
0606     spin_unlock_irq(&mchdev_lock);
0607 
0608     rps->ips.corr = init_emon(uncore);
0609 
0610     return true;
0611 }
0612 
0613 static void gen5_rps_disable(struct intel_rps *rps)
0614 {
0615     struct drm_i915_private *i915 = rps_to_i915(rps);
0616     struct intel_uncore *uncore = rps_to_uncore(rps);
0617     u16 rgvswctl;
0618 
0619     spin_lock_irq(&mchdev_lock);
0620 
0621     spin_lock(&i915->irq_lock);
0622     ilk_disable_display_irq(i915, DE_PCU_EVENT);
0623     spin_unlock(&i915->irq_lock);
0624 
0625     rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
0626 
0627     /* Ack interrupts, disable EFC interrupt */
0628     intel_uncore_write(uncore, MEMINTREN,
0629                intel_uncore_read(uncore, MEMINTREN) &
0630                ~MEMINT_EVAL_CHG_EN);
0631     intel_uncore_write(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);
0632 
0633     /* Go back to the starting frequency */
0634     __gen5_rps_set(rps, rps->idle_freq);
0635     mdelay(1);
0636     rgvswctl |= MEMCTL_CMD_STS;
0637     intel_uncore_write(uncore, MEMSWCTL, rgvswctl);
0638     mdelay(1);
0639 
0640     spin_unlock_irq(&mchdev_lock);
0641 }
0642 
0643 static u32 rps_limits(struct intel_rps *rps, u8 val)
0644 {
0645     u32 limits;
0646 
0647     /*
0648      * Only set the down limit when we've reached the lowest level to avoid
0649      * getting more interrupts, otherwise leave this clear. This prevents a
0650      * race in the hw when coming out of rc6: There's a tiny window where
0651      * the hw runs at the minimal clock before selecting the desired
0652      * frequency, if the down threshold expires in that window we will not
0653      * receive a down interrupt.
0654      */
0655     if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) {
0656         limits = rps->max_freq_softlimit << 23;
0657         if (val <= rps->min_freq_softlimit)
0658             limits |= rps->min_freq_softlimit << 14;
0659     } else {
0660         limits = rps->max_freq_softlimit << 24;
0661         if (val <= rps->min_freq_softlimit)
0662             limits |= rps->min_freq_softlimit << 16;
0663     }
0664 
0665     return limits;
0666 }
0667 
0668 static void rps_set_power(struct intel_rps *rps, int new_power)
0669 {
0670     struct intel_gt *gt = rps_to_gt(rps);
0671     struct intel_uncore *uncore = gt->uncore;
0672     u32 threshold_up = 0, threshold_down = 0; /* in % */
0673     u32 ei_up = 0, ei_down = 0;
0674 
0675     lockdep_assert_held(&rps->power.mutex);
0676 
0677     if (new_power == rps->power.mode)
0678         return;
0679 
0680     threshold_up = 95;
0681     threshold_down = 85;
0682 
0683     /* Note the units here are not exactly 1us, but 1280ns. */
0684     switch (new_power) {
0685     case LOW_POWER:
0686         ei_up = 16000;
0687         ei_down = 32000;
0688         break;
0689 
0690     case BETWEEN:
0691         ei_up = 13000;
0692         ei_down = 32000;
0693         break;
0694 
0695     case HIGH_POWER:
0696         ei_up = 10000;
0697         ei_down = 32000;
0698         break;
0699     }
0700 
0701     /* When byt can survive without system hang with dynamic
0702      * sw freq adjustments, this restriction can be lifted.
0703      */
0704     if (IS_VALLEYVIEW(gt->i915))
0705         goto skip_hw_write;
0706 
0707     GT_TRACE(gt,
0708          "changing power mode [%d], up %d%% @ %dus, down %d%% @ %dus\n",
0709          new_power, threshold_up, ei_up, threshold_down, ei_down);
0710 
0711     set(uncore, GEN6_RP_UP_EI,
0712         intel_gt_ns_to_pm_interval(gt, ei_up * 1000));
0713     set(uncore, GEN6_RP_UP_THRESHOLD,
0714         intel_gt_ns_to_pm_interval(gt, ei_up * threshold_up * 10));
0715 
0716     set(uncore, GEN6_RP_DOWN_EI,
0717         intel_gt_ns_to_pm_interval(gt, ei_down * 1000));
0718     set(uncore, GEN6_RP_DOWN_THRESHOLD,
0719         intel_gt_ns_to_pm_interval(gt, ei_down * threshold_down * 10));
0720 
0721     set(uncore, GEN6_RP_CONTROL,
0722         (GRAPHICS_VER(gt->i915) > 9 ? 0 : GEN6_RP_MEDIA_TURBO) |
0723         GEN6_RP_MEDIA_HW_NORMAL_MODE |
0724         GEN6_RP_MEDIA_IS_GFX |
0725         GEN6_RP_ENABLE |
0726         GEN6_RP_UP_BUSY_AVG |
0727         GEN6_RP_DOWN_IDLE_AVG);
0728 
0729 skip_hw_write:
0730     rps->power.mode = new_power;
0731     rps->power.up_threshold = threshold_up;
0732     rps->power.down_threshold = threshold_down;
0733 }
0734 
0735 static void gen6_rps_set_thresholds(struct intel_rps *rps, u8 val)
0736 {
0737     int new_power;
0738 
0739     new_power = rps->power.mode;
0740     switch (rps->power.mode) {
0741     case LOW_POWER:
0742         if (val > rps->efficient_freq + 1 &&
0743             val > rps->cur_freq)
0744             new_power = BETWEEN;
0745         break;
0746 
0747     case BETWEEN:
0748         if (val <= rps->efficient_freq &&
0749             val < rps->cur_freq)
0750             new_power = LOW_POWER;
0751         else if (val >= rps->rp0_freq &&
0752              val > rps->cur_freq)
0753             new_power = HIGH_POWER;
0754         break;
0755 
0756     case HIGH_POWER:
0757         if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 &&
0758             val < rps->cur_freq)
0759             new_power = BETWEEN;
0760         break;
0761     }
0762     /* Max/min bins are special */
0763     if (val <= rps->min_freq_softlimit)
0764         new_power = LOW_POWER;
0765     if (val >= rps->max_freq_softlimit)
0766         new_power = HIGH_POWER;
0767 
0768     mutex_lock(&rps->power.mutex);
0769     if (rps->power.interactive)
0770         new_power = HIGH_POWER;
0771     rps_set_power(rps, new_power);
0772     mutex_unlock(&rps->power.mutex);
0773 }
0774 
0775 void intel_rps_mark_interactive(struct intel_rps *rps, bool interactive)
0776 {
0777     GT_TRACE(rps_to_gt(rps), "mark interactive: %s\n",
0778          str_yes_no(interactive));
0779 
0780     mutex_lock(&rps->power.mutex);
0781     if (interactive) {
0782         if (!rps->power.interactive++ && intel_rps_is_active(rps))
0783             rps_set_power(rps, HIGH_POWER);
0784     } else {
0785         GEM_BUG_ON(!rps->power.interactive);
0786         rps->power.interactive--;
0787     }
0788     mutex_unlock(&rps->power.mutex);
0789 }
0790 
0791 static int gen6_rps_set(struct intel_rps *rps, u8 val)
0792 {
0793     struct intel_uncore *uncore = rps_to_uncore(rps);
0794     struct drm_i915_private *i915 = rps_to_i915(rps);
0795     u32 swreq;
0796 
0797     GEM_BUG_ON(rps_uses_slpc(rps));
0798 
0799     if (GRAPHICS_VER(i915) >= 9)
0800         swreq = GEN9_FREQUENCY(val);
0801     else if (IS_HASWELL(i915) || IS_BROADWELL(i915))
0802         swreq = HSW_FREQUENCY(val);
0803     else
0804         swreq = (GEN6_FREQUENCY(val) |
0805              GEN6_OFFSET(0) |
0806              GEN6_AGGRESSIVE_TURBO);
0807     set(uncore, GEN6_RPNSWREQ, swreq);
0808 
0809     GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d, swreq:%x\n",
0810          val, intel_gpu_freq(rps, val), swreq);
0811 
0812     return 0;
0813 }
0814 
0815 static int vlv_rps_set(struct intel_rps *rps, u8 val)
0816 {
0817     struct drm_i915_private *i915 = rps_to_i915(rps);
0818     int err;
0819 
0820     vlv_punit_get(i915);
0821     err = vlv_punit_write(i915, PUNIT_REG_GPU_FREQ_REQ, val);
0822     vlv_punit_put(i915);
0823 
0824     GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d\n",
0825          val, intel_gpu_freq(rps, val));
0826 
0827     return err;
0828 }
0829 
0830 static int rps_set(struct intel_rps *rps, u8 val, bool update)
0831 {
0832     struct drm_i915_private *i915 = rps_to_i915(rps);
0833     int err;
0834 
0835     if (val == rps->last_freq)
0836         return 0;
0837 
0838     if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
0839         err = vlv_rps_set(rps, val);
0840     else if (GRAPHICS_VER(i915) >= 6)
0841         err = gen6_rps_set(rps, val);
0842     else
0843         err = gen5_rps_set(rps, val);
0844     if (err)
0845         return err;
0846 
0847     if (update && GRAPHICS_VER(i915) >= 6)
0848         gen6_rps_set_thresholds(rps, val);
0849     rps->last_freq = val;
0850 
0851     return 0;
0852 }
0853 
0854 void intel_rps_unpark(struct intel_rps *rps)
0855 {
0856     if (!intel_rps_is_enabled(rps))
0857         return;
0858 
0859     GT_TRACE(rps_to_gt(rps), "unpark:%x\n", rps->cur_freq);
0860 
0861     /*
0862      * Use the user's desired frequency as a guide, but for better
0863      * performance, jump directly to RPe as our starting frequency.
0864      */
0865     mutex_lock(&rps->lock);
0866 
0867     intel_rps_set_active(rps);
0868     intel_rps_set(rps,
0869               clamp(rps->cur_freq,
0870                 rps->min_freq_softlimit,
0871                 rps->max_freq_softlimit));
0872 
0873     mutex_unlock(&rps->lock);
0874 
0875     rps->pm_iir = 0;
0876     if (intel_rps_has_interrupts(rps))
0877         rps_enable_interrupts(rps);
0878     if (intel_rps_uses_timer(rps))
0879         rps_start_timer(rps);
0880 
0881     if (GRAPHICS_VER(rps_to_i915(rps)) == 5)
0882         gen5_rps_update(rps);
0883 }
0884 
0885 void intel_rps_park(struct intel_rps *rps)
0886 {
0887     int adj;
0888 
0889     if (!intel_rps_is_enabled(rps))
0890         return;
0891 
0892     if (!intel_rps_clear_active(rps))
0893         return;
0894 
0895     if (intel_rps_uses_timer(rps))
0896         rps_stop_timer(rps);
0897     if (intel_rps_has_interrupts(rps))
0898         rps_disable_interrupts(rps);
0899 
0900     if (rps->last_freq <= rps->idle_freq)
0901         return;
0902 
0903     /*
0904      * The punit delays the write of the frequency and voltage until it
0905      * determines the GPU is awake. During normal usage we don't want to
0906      * waste power changing the frequency if the GPU is sleeping (rc6).
0907      * However, the GPU and driver is now idle and we do not want to delay
0908      * switching to minimum voltage (reducing power whilst idle) as we do
0909      * not expect to be woken in the near future and so must flush the
0910      * change by waking the device.
0911      *
0912      * We choose to take the media powerwell (either would do to trick the
0913      * punit into committing the voltage change) as that takes a lot less
0914      * power than the render powerwell.
0915      */
0916     intel_uncore_forcewake_get(rps_to_uncore(rps), FORCEWAKE_MEDIA);
0917     rps_set(rps, rps->idle_freq, false);
0918     intel_uncore_forcewake_put(rps_to_uncore(rps), FORCEWAKE_MEDIA);
0919 
0920     /*
0921      * Since we will try and restart from the previously requested
0922      * frequency on unparking, treat this idle point as a downclock
0923      * interrupt and reduce the frequency for resume. If we park/unpark
0924      * more frequently than the rps worker can run, we will not respond
0925      * to any EI and never see a change in frequency.
0926      *
0927      * (Note we accommodate Cherryview's limitation of only using an
0928      * even bin by applying it to all.)
0929      */
0930     adj = rps->last_adj;
0931     if (adj < 0)
0932         adj *= 2;
0933     else /* CHV needs even encode values */
0934         adj = -2;
0935     rps->last_adj = adj;
0936     rps->cur_freq = max_t(int, rps->cur_freq + adj, rps->min_freq);
0937     if (rps->cur_freq < rps->efficient_freq) {
0938         rps->cur_freq = rps->efficient_freq;
0939         rps->last_adj = 0;
0940     }
0941 
0942     GT_TRACE(rps_to_gt(rps), "park:%x\n", rps->cur_freq);
0943 }
0944 
0945 u32 intel_rps_get_boost_frequency(struct intel_rps *rps)
0946 {
0947     struct intel_guc_slpc *slpc;
0948 
0949     if (rps_uses_slpc(rps)) {
0950         slpc = rps_to_slpc(rps);
0951 
0952         return slpc->boost_freq;
0953     } else {
0954         return intel_gpu_freq(rps, rps->boost_freq);
0955     }
0956 }
0957 
0958 static int rps_set_boost_freq(struct intel_rps *rps, u32 val)
0959 {
0960     bool boost = false;
0961 
0962     /* Validate against (static) hardware limits */
0963     val = intel_freq_opcode(rps, val);
0964     if (val < rps->min_freq || val > rps->max_freq)
0965         return -EINVAL;
0966 
0967     mutex_lock(&rps->lock);
0968     if (val != rps->boost_freq) {
0969         rps->boost_freq = val;
0970         boost = atomic_read(&rps->num_waiters);
0971     }
0972     mutex_unlock(&rps->lock);
0973     if (boost)
0974         schedule_work(&rps->work);
0975 
0976     return 0;
0977 }
0978 
0979 int intel_rps_set_boost_frequency(struct intel_rps *rps, u32 freq)
0980 {
0981     struct intel_guc_slpc *slpc;
0982 
0983     if (rps_uses_slpc(rps)) {
0984         slpc = rps_to_slpc(rps);
0985 
0986         return intel_guc_slpc_set_boost_freq(slpc, freq);
0987     } else {
0988         return rps_set_boost_freq(rps, freq);
0989     }
0990 }
0991 
0992 void intel_rps_dec_waiters(struct intel_rps *rps)
0993 {
0994     struct intel_guc_slpc *slpc;
0995 
0996     if (rps_uses_slpc(rps)) {
0997         slpc = rps_to_slpc(rps);
0998 
0999         intel_guc_slpc_dec_waiters(slpc);
1000     } else {
1001         atomic_dec(&rps->num_waiters);
1002     }
1003 }
1004 
1005 void intel_rps_boost(struct i915_request *rq)
1006 {
1007     struct intel_guc_slpc *slpc;
1008 
1009     if (i915_request_signaled(rq) || i915_request_has_waitboost(rq))
1010         return;
1011 
1012     /* Serializes with i915_request_retire() */
1013     if (!test_and_set_bit(I915_FENCE_FLAG_BOOST, &rq->fence.flags)) {
1014         struct intel_rps *rps = &READ_ONCE(rq->engine)->gt->rps;
1015 
1016         if (rps_uses_slpc(rps)) {
1017             slpc = rps_to_slpc(rps);
1018 
1019             /* Return if old value is non zero */
1020             if (!atomic_fetch_inc(&slpc->num_waiters))
1021                 schedule_work(&slpc->boost_work);
1022 
1023             return;
1024         }
1025 
1026         if (atomic_fetch_inc(&rps->num_waiters))
1027             return;
1028 
1029         if (!intel_rps_is_active(rps))
1030             return;
1031 
1032         GT_TRACE(rps_to_gt(rps), "boost fence:%llx:%llx\n",
1033              rq->fence.context, rq->fence.seqno);
1034 
1035         if (READ_ONCE(rps->cur_freq) < rps->boost_freq)
1036             schedule_work(&rps->work);
1037 
1038         WRITE_ONCE(rps->boosts, rps->boosts + 1); /* debug only */
1039     }
1040 }
1041 
1042 int intel_rps_set(struct intel_rps *rps, u8 val)
1043 {
1044     int err;
1045 
1046     lockdep_assert_held(&rps->lock);
1047     GEM_BUG_ON(val > rps->max_freq);
1048     GEM_BUG_ON(val < rps->min_freq);
1049 
1050     if (intel_rps_is_active(rps)) {
1051         err = rps_set(rps, val, true);
1052         if (err)
1053             return err;
1054 
1055         /*
1056          * Make sure we continue to get interrupts
1057          * until we hit the minimum or maximum frequencies.
1058          */
1059         if (intel_rps_has_interrupts(rps)) {
1060             struct intel_uncore *uncore = rps_to_uncore(rps);
1061 
1062             set(uncore,
1063                 GEN6_RP_INTERRUPT_LIMITS, rps_limits(rps, val));
1064 
1065             set(uncore, GEN6_PMINTRMSK, rps_pm_mask(rps, val));
1066         }
1067     }
1068 
1069     rps->cur_freq = val;
1070     return 0;
1071 }
1072 
1073 static u32 intel_rps_read_state_cap(struct intel_rps *rps)
1074 {
1075     struct drm_i915_private *i915 = rps_to_i915(rps);
1076     struct intel_uncore *uncore = rps_to_uncore(rps);
1077 
1078     if (IS_PONTEVECCHIO(i915))
1079         return intel_uncore_read(uncore, PVC_RP_STATE_CAP);
1080     else if (IS_XEHPSDV(i915))
1081         return intel_uncore_read(uncore, XEHPSDV_RP_STATE_CAP);
1082     else if (IS_GEN9_LP(i915))
1083         return intel_uncore_read(uncore, BXT_RP_STATE_CAP);
1084     else
1085         return intel_uncore_read(uncore, GEN6_RP_STATE_CAP);
1086 }
1087 
1088 /**
1089  * gen6_rps_get_freq_caps - Get freq caps exposed by HW
1090  * @rps: the intel_rps structure
1091  * @caps: returned freq caps
1092  *
1093  * Returned "caps" frequencies should be converted to MHz using
1094  * intel_gpu_freq()
1095  */
1096 void gen6_rps_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps)
1097 {
1098     struct drm_i915_private *i915 = rps_to_i915(rps);
1099     u32 rp_state_cap;
1100 
1101     rp_state_cap = intel_rps_read_state_cap(rps);
1102 
1103     /* static values from HW: RP0 > RP1 > RPn (min_freq) */
1104     if (IS_GEN9_LP(i915)) {
1105         caps->rp0_freq = (rp_state_cap >> 16) & 0xff;
1106         caps->rp1_freq = (rp_state_cap >>  8) & 0xff;
1107         caps->min_freq = (rp_state_cap >>  0) & 0xff;
1108     } else {
1109         caps->rp0_freq = (rp_state_cap >>  0) & 0xff;
1110         caps->rp1_freq = (rp_state_cap >>  8) & 0xff;
1111         caps->min_freq = (rp_state_cap >> 16) & 0xff;
1112     }
1113 
1114     if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) {
1115         /*
1116          * In this case rp_state_cap register reports frequencies in
1117          * units of 50 MHz. Convert these to the actual "hw unit", i.e.
1118          * units of 16.67 MHz
1119          */
1120         caps->rp0_freq *= GEN9_FREQ_SCALER;
1121         caps->rp1_freq *= GEN9_FREQ_SCALER;
1122         caps->min_freq *= GEN9_FREQ_SCALER;
1123     }
1124 }
1125 
1126 static void gen6_rps_init(struct intel_rps *rps)
1127 {
1128     struct drm_i915_private *i915 = rps_to_i915(rps);
1129     struct intel_rps_freq_caps caps;
1130 
1131     gen6_rps_get_freq_caps(rps, &caps);
1132     rps->rp0_freq = caps.rp0_freq;
1133     rps->rp1_freq = caps.rp1_freq;
1134     rps->min_freq = caps.min_freq;
1135 
1136     /* hw_max = RP0 until we check for overclocking */
1137     rps->max_freq = rps->rp0_freq;
1138 
1139     rps->efficient_freq = rps->rp1_freq;
1140     if (IS_HASWELL(i915) || IS_BROADWELL(i915) ||
1141         IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) {
1142         u32 ddcc_status = 0;
1143         u32 mult = 1;
1144 
1145         if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11)
1146             mult = GEN9_FREQ_SCALER;
1147         if (snb_pcode_read(rps_to_gt(rps)->uncore,
1148                    HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,
1149                    &ddcc_status, NULL) == 0)
1150             rps->efficient_freq =
1151                 clamp_t(u32,
1152                     ((ddcc_status >> 8) & 0xff) * mult,
1153                     rps->min_freq,
1154                     rps->max_freq);
1155     }
1156 }
1157 
1158 static bool rps_reset(struct intel_rps *rps)
1159 {
1160     struct drm_i915_private *i915 = rps_to_i915(rps);
1161 
1162     /* force a reset */
1163     rps->power.mode = -1;
1164     rps->last_freq = -1;
1165 
1166     if (rps_set(rps, rps->min_freq, true)) {
1167         drm_err(&i915->drm, "Failed to reset RPS to initial values\n");
1168         return false;
1169     }
1170 
1171     rps->cur_freq = rps->min_freq;
1172     return true;
1173 }
1174 
1175 /* See the Gen9_GT_PM_Programming_Guide doc for the below */
1176 static bool gen9_rps_enable(struct intel_rps *rps)
1177 {
1178     struct intel_gt *gt = rps_to_gt(rps);
1179     struct intel_uncore *uncore = gt->uncore;
1180 
1181     /* Program defaults and thresholds for RPS */
1182     if (GRAPHICS_VER(gt->i915) == 9)
1183         intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ,
1184                       GEN9_FREQUENCY(rps->rp1_freq));
1185 
1186     intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 0xa);
1187 
1188     rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD;
1189 
1190     return rps_reset(rps);
1191 }
1192 
1193 static bool gen8_rps_enable(struct intel_rps *rps)
1194 {
1195     struct intel_uncore *uncore = rps_to_uncore(rps);
1196 
1197     intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ,
1198                   HSW_FREQUENCY(rps->rp1_freq));
1199 
1200     intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1201 
1202     rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD;
1203 
1204     return rps_reset(rps);
1205 }
1206 
1207 static bool gen6_rps_enable(struct intel_rps *rps)
1208 {
1209     struct intel_uncore *uncore = rps_to_uncore(rps);
1210 
1211     /* Power down if completely idle for over 50ms */
1212     intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 50000);
1213     intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1214 
1215     rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD |
1216               GEN6_PM_RP_DOWN_THRESHOLD |
1217               GEN6_PM_RP_DOWN_TIMEOUT);
1218 
1219     return rps_reset(rps);
1220 }
1221 
1222 static int chv_rps_max_freq(struct intel_rps *rps)
1223 {
1224     struct drm_i915_private *i915 = rps_to_i915(rps);
1225     struct intel_gt *gt = rps_to_gt(rps);
1226     u32 val;
1227 
1228     val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE);
1229 
1230     switch (gt->info.sseu.eu_total) {
1231     case 8:
1232         /* (2 * 4) config */
1233         val >>= FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT;
1234         break;
1235     case 12:
1236         /* (2 * 6) config */
1237         val >>= FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT;
1238         break;
1239     case 16:
1240         /* (2 * 8) config */
1241     default:
1242         /* Setting (2 * 8) Min RP0 for any other combination */
1243         val >>= FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT;
1244         break;
1245     }
1246 
1247     return val & FB_GFX_FREQ_FUSE_MASK;
1248 }
1249 
1250 static int chv_rps_rpe_freq(struct intel_rps *rps)
1251 {
1252     struct drm_i915_private *i915 = rps_to_i915(rps);
1253     u32 val;
1254 
1255     val = vlv_punit_read(i915, PUNIT_GPU_DUTYCYCLE_REG);
1256     val >>= PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT;
1257 
1258     return val & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK;
1259 }
1260 
1261 static int chv_rps_guar_freq(struct intel_rps *rps)
1262 {
1263     struct drm_i915_private *i915 = rps_to_i915(rps);
1264     u32 val;
1265 
1266     val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE);
1267 
1268     return val & FB_GFX_FREQ_FUSE_MASK;
1269 }
1270 
1271 static u32 chv_rps_min_freq(struct intel_rps *rps)
1272 {
1273     struct drm_i915_private *i915 = rps_to_i915(rps);
1274     u32 val;
1275 
1276     val = vlv_punit_read(i915, FB_GFX_FMIN_AT_VMIN_FUSE);
1277     val >>= FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT;
1278 
1279     return val & FB_GFX_FREQ_FUSE_MASK;
1280 }
1281 
1282 static bool chv_rps_enable(struct intel_rps *rps)
1283 {
1284     struct intel_uncore *uncore = rps_to_uncore(rps);
1285     struct drm_i915_private *i915 = rps_to_i915(rps);
1286     u32 val;
1287 
1288     /* 1: Program defaults and thresholds for RPS*/
1289     intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000);
1290     intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400);
1291     intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000);
1292     intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000);
1293     intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000);
1294 
1295     intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1296 
1297     /* 2: Enable RPS */
1298     intel_uncore_write_fw(uncore, GEN6_RP_CONTROL,
1299                   GEN6_RP_MEDIA_HW_NORMAL_MODE |
1300                   GEN6_RP_MEDIA_IS_GFX |
1301                   GEN6_RP_ENABLE |
1302                   GEN6_RP_UP_BUSY_AVG |
1303                   GEN6_RP_DOWN_IDLE_AVG);
1304 
1305     rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD |
1306               GEN6_PM_RP_DOWN_THRESHOLD |
1307               GEN6_PM_RP_DOWN_TIMEOUT);
1308 
1309     /* Setting Fixed Bias */
1310     vlv_punit_get(i915);
1311 
1312     val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | CHV_BIAS_CPU_50_SOC_50;
1313     vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val);
1314 
1315     val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
1316 
1317     vlv_punit_put(i915);
1318 
1319     /* RPS code assumes GPLL is used */
1320     drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0,
1321               "GPLL not enabled\n");
1322 
1323     drm_dbg(&i915->drm, "GPLL enabled? %s\n",
1324         str_yes_no(val & GPLLENABLE));
1325     drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val);
1326 
1327     return rps_reset(rps);
1328 }
1329 
1330 static int vlv_rps_guar_freq(struct intel_rps *rps)
1331 {
1332     struct drm_i915_private *i915 = rps_to_i915(rps);
1333     u32 val, rp1;
1334 
1335     val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE);
1336 
1337     rp1 = val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK;
1338     rp1 >>= FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT;
1339 
1340     return rp1;
1341 }
1342 
1343 static int vlv_rps_max_freq(struct intel_rps *rps)
1344 {
1345     struct drm_i915_private *i915 = rps_to_i915(rps);
1346     u32 val, rp0;
1347 
1348     val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE);
1349 
1350     rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
1351     /* Clamp to max */
1352     rp0 = min_t(u32, rp0, 0xea);
1353 
1354     return rp0;
1355 }
1356 
1357 static int vlv_rps_rpe_freq(struct intel_rps *rps)
1358 {
1359     struct drm_i915_private *i915 = rps_to_i915(rps);
1360     u32 val, rpe;
1361 
1362     val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
1363     rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
1364     val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
1365     rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
1366 
1367     return rpe;
1368 }
1369 
1370 static int vlv_rps_min_freq(struct intel_rps *rps)
1371 {
1372     struct drm_i915_private *i915 = rps_to_i915(rps);
1373     u32 val;
1374 
1375     val = vlv_punit_read(i915, PUNIT_REG_GPU_LFM) & 0xff;
1376     /*
1377      * According to the BYT Punit GPU turbo HAS 1.1.6.3 the minimum value
1378      * for the minimum frequency in GPLL mode is 0xc1. Contrary to this on
1379      * a BYT-M B0 the above register contains 0xbf. Moreover when setting
1380      * a frequency Punit will not allow values below 0xc0. Clamp it 0xc0
1381      * to make sure it matches what Punit accepts.
1382      */
1383     return max_t(u32, val, 0xc0);
1384 }
1385 
1386 static bool vlv_rps_enable(struct intel_rps *rps)
1387 {
1388     struct intel_uncore *uncore = rps_to_uncore(rps);
1389     struct drm_i915_private *i915 = rps_to_i915(rps);
1390     u32 val;
1391 
1392     intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000);
1393     intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400);
1394     intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000);
1395     intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000);
1396     intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000);
1397 
1398     intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1399 
1400     intel_uncore_write_fw(uncore, GEN6_RP_CONTROL,
1401                   GEN6_RP_MEDIA_TURBO |
1402                   GEN6_RP_MEDIA_HW_NORMAL_MODE |
1403                   GEN6_RP_MEDIA_IS_GFX |
1404                   GEN6_RP_ENABLE |
1405                   GEN6_RP_UP_BUSY_AVG |
1406                   GEN6_RP_DOWN_IDLE_CONT);
1407 
1408     /* WaGsvRC0ResidencyMethod:vlv */
1409     rps->pm_events = GEN6_PM_RP_UP_EI_EXPIRED;
1410 
1411     vlv_punit_get(i915);
1412 
1413     /* Setting Fixed Bias */
1414     val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | VLV_BIAS_CPU_125_SOC_875;
1415     vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val);
1416 
1417     val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
1418 
1419     vlv_punit_put(i915);
1420 
1421     /* RPS code assumes GPLL is used */
1422     drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0,
1423               "GPLL not enabled\n");
1424 
1425     drm_dbg(&i915->drm, "GPLL enabled? %s\n",
1426         str_yes_no(val & GPLLENABLE));
1427     drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val);
1428 
1429     return rps_reset(rps);
1430 }
1431 
1432 static unsigned long __ips_gfx_val(struct intel_ips *ips)
1433 {
1434     struct intel_rps *rps = container_of(ips, typeof(*rps), ips);
1435     struct intel_uncore *uncore = rps_to_uncore(rps);
1436     unsigned int t, state1, state2;
1437     u32 pxvid, ext_v;
1438     u64 corr, corr2;
1439 
1440     lockdep_assert_held(&mchdev_lock);
1441 
1442     pxvid = intel_uncore_read(uncore, PXVFREQ(rps->cur_freq));
1443     pxvid = (pxvid >> 24) & 0x7f;
1444     ext_v = pvid_to_extvid(rps_to_i915(rps), pxvid);
1445 
1446     state1 = ext_v;
1447 
1448     /* Revel in the empirically derived constants */
1449 
1450     /* Correction factor in 1/100000 units */
1451     t = ips_mch_val(uncore);
1452     if (t > 80)
1453         corr = t * 2349 + 135940;
1454     else if (t >= 50)
1455         corr = t * 964 + 29317;
1456     else /* < 50 */
1457         corr = t * 301 + 1004;
1458 
1459     corr = div_u64(corr * 150142 * state1, 10000) - 78642;
1460     corr2 = div_u64(corr, 100000) * ips->corr;
1461 
1462     state2 = div_u64(corr2 * state1, 10000);
1463     state2 /= 100; /* convert to mW */
1464 
1465     __gen5_ips_update(ips);
1466 
1467     return ips->gfx_power + state2;
1468 }
1469 
1470 static bool has_busy_stats(struct intel_rps *rps)
1471 {
1472     struct intel_engine_cs *engine;
1473     enum intel_engine_id id;
1474 
1475     for_each_engine(engine, rps_to_gt(rps), id) {
1476         if (!intel_engine_supports_stats(engine))
1477             return false;
1478     }
1479 
1480     return true;
1481 }
1482 
1483 void intel_rps_enable(struct intel_rps *rps)
1484 {
1485     struct drm_i915_private *i915 = rps_to_i915(rps);
1486     struct intel_uncore *uncore = rps_to_uncore(rps);
1487     bool enabled = false;
1488 
1489     if (!HAS_RPS(i915))
1490         return;
1491 
1492     if (rps_uses_slpc(rps))
1493         return;
1494 
1495     intel_gt_check_clock_frequency(rps_to_gt(rps));
1496 
1497     intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
1498     if (rps->max_freq <= rps->min_freq)
1499         /* leave disabled, no room for dynamic reclocking */;
1500     else if (IS_CHERRYVIEW(i915))
1501         enabled = chv_rps_enable(rps);
1502     else if (IS_VALLEYVIEW(i915))
1503         enabled = vlv_rps_enable(rps);
1504     else if (GRAPHICS_VER(i915) >= 9)
1505         enabled = gen9_rps_enable(rps);
1506     else if (GRAPHICS_VER(i915) >= 8)
1507         enabled = gen8_rps_enable(rps);
1508     else if (GRAPHICS_VER(i915) >= 6)
1509         enabled = gen6_rps_enable(rps);
1510     else if (IS_IRONLAKE_M(i915))
1511         enabled = gen5_rps_enable(rps);
1512     else
1513         MISSING_CASE(GRAPHICS_VER(i915));
1514     intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
1515     if (!enabled)
1516         return;
1517 
1518     GT_TRACE(rps_to_gt(rps),
1519          "min:%x, max:%x, freq:[%d, %d]\n",
1520          rps->min_freq, rps->max_freq,
1521          intel_gpu_freq(rps, rps->min_freq),
1522          intel_gpu_freq(rps, rps->max_freq));
1523 
1524     GEM_BUG_ON(rps->max_freq < rps->min_freq);
1525     GEM_BUG_ON(rps->idle_freq > rps->max_freq);
1526 
1527     GEM_BUG_ON(rps->efficient_freq < rps->min_freq);
1528     GEM_BUG_ON(rps->efficient_freq > rps->max_freq);
1529 
1530     if (has_busy_stats(rps))
1531         intel_rps_set_timer(rps);
1532     else if (GRAPHICS_VER(i915) >= 6 && GRAPHICS_VER(i915) <= 11)
1533         intel_rps_set_interrupts(rps);
1534     else
1535         /* Ironlake currently uses intel_ips.ko */ {}
1536 
1537     intel_rps_set_enabled(rps);
1538 }
1539 
1540 static void gen6_rps_disable(struct intel_rps *rps)
1541 {
1542     set(rps_to_uncore(rps), GEN6_RP_CONTROL, 0);
1543 }
1544 
1545 void intel_rps_disable(struct intel_rps *rps)
1546 {
1547     struct drm_i915_private *i915 = rps_to_i915(rps);
1548 
1549     intel_rps_clear_enabled(rps);
1550     intel_rps_clear_interrupts(rps);
1551     intel_rps_clear_timer(rps);
1552 
1553     if (GRAPHICS_VER(i915) >= 6)
1554         gen6_rps_disable(rps);
1555     else if (IS_IRONLAKE_M(i915))
1556         gen5_rps_disable(rps);
1557 }
1558 
1559 static int byt_gpu_freq(struct intel_rps *rps, int val)
1560 {
1561     /*
1562      * N = val - 0xb7
1563      * Slow = Fast = GPLL ref * N
1564      */
1565     return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000);
1566 }
1567 
1568 static int byt_freq_opcode(struct intel_rps *rps, int val)
1569 {
1570     return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7;
1571 }
1572 
1573 static int chv_gpu_freq(struct intel_rps *rps, int val)
1574 {
1575     /*
1576      * N = val / 2
1577      * CU (slow) = CU2x (fast) / 2 = GPLL ref * N / 2
1578      */
1579     return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000);
1580 }
1581 
1582 static int chv_freq_opcode(struct intel_rps *rps, int val)
1583 {
1584     /* CHV needs even values */
1585     return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2;
1586 }
1587 
1588 int intel_gpu_freq(struct intel_rps *rps, int val)
1589 {
1590     struct drm_i915_private *i915 = rps_to_i915(rps);
1591 
1592     if (GRAPHICS_VER(i915) >= 9)
1593         return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER,
1594                      GEN9_FREQ_SCALER);
1595     else if (IS_CHERRYVIEW(i915))
1596         return chv_gpu_freq(rps, val);
1597     else if (IS_VALLEYVIEW(i915))
1598         return byt_gpu_freq(rps, val);
1599     else if (GRAPHICS_VER(i915) >= 6)
1600         return val * GT_FREQUENCY_MULTIPLIER;
1601     else
1602         return val;
1603 }
1604 
1605 int intel_freq_opcode(struct intel_rps *rps, int val)
1606 {
1607     struct drm_i915_private *i915 = rps_to_i915(rps);
1608 
1609     if (GRAPHICS_VER(i915) >= 9)
1610         return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER,
1611                      GT_FREQUENCY_MULTIPLIER);
1612     else if (IS_CHERRYVIEW(i915))
1613         return chv_freq_opcode(rps, val);
1614     else if (IS_VALLEYVIEW(i915))
1615         return byt_freq_opcode(rps, val);
1616     else if (GRAPHICS_VER(i915) >= 6)
1617         return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER);
1618     else
1619         return val;
1620 }
1621 
1622 static void vlv_init_gpll_ref_freq(struct intel_rps *rps)
1623 {
1624     struct drm_i915_private *i915 = rps_to_i915(rps);
1625 
1626     rps->gpll_ref_freq =
1627         vlv_get_cck_clock(i915, "GPLL ref",
1628                   CCK_GPLL_CLOCK_CONTROL,
1629                   i915->czclk_freq);
1630 
1631     drm_dbg(&i915->drm, "GPLL reference freq: %d kHz\n",
1632         rps->gpll_ref_freq);
1633 }
1634 
1635 static void vlv_rps_init(struct intel_rps *rps)
1636 {
1637     struct drm_i915_private *i915 = rps_to_i915(rps);
1638     u32 val;
1639 
1640     vlv_iosf_sb_get(i915,
1641             BIT(VLV_IOSF_SB_PUNIT) |
1642             BIT(VLV_IOSF_SB_NC) |
1643             BIT(VLV_IOSF_SB_CCK));
1644 
1645     vlv_init_gpll_ref_freq(rps);
1646 
1647     val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
1648     switch ((val >> 6) & 3) {
1649     case 0:
1650     case 1:
1651         i915->mem_freq = 800;
1652         break;
1653     case 2:
1654         i915->mem_freq = 1066;
1655         break;
1656     case 3:
1657         i915->mem_freq = 1333;
1658         break;
1659     }
1660     drm_dbg(&i915->drm, "DDR speed: %d MHz\n", i915->mem_freq);
1661 
1662     rps->max_freq = vlv_rps_max_freq(rps);
1663     rps->rp0_freq = rps->max_freq;
1664     drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n",
1665         intel_gpu_freq(rps, rps->max_freq), rps->max_freq);
1666 
1667     rps->efficient_freq = vlv_rps_rpe_freq(rps);
1668     drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n",
1669         intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq);
1670 
1671     rps->rp1_freq = vlv_rps_guar_freq(rps);
1672     drm_dbg(&i915->drm, "RP1(Guar Freq) GPU freq: %d MHz (%u)\n",
1673         intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq);
1674 
1675     rps->min_freq = vlv_rps_min_freq(rps);
1676     drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n",
1677         intel_gpu_freq(rps, rps->min_freq), rps->min_freq);
1678 
1679     vlv_iosf_sb_put(i915,
1680             BIT(VLV_IOSF_SB_PUNIT) |
1681             BIT(VLV_IOSF_SB_NC) |
1682             BIT(VLV_IOSF_SB_CCK));
1683 }
1684 
1685 static void chv_rps_init(struct intel_rps *rps)
1686 {
1687     struct drm_i915_private *i915 = rps_to_i915(rps);
1688     u32 val;
1689 
1690     vlv_iosf_sb_get(i915,
1691             BIT(VLV_IOSF_SB_PUNIT) |
1692             BIT(VLV_IOSF_SB_NC) |
1693             BIT(VLV_IOSF_SB_CCK));
1694 
1695     vlv_init_gpll_ref_freq(rps);
1696 
1697     val = vlv_cck_read(i915, CCK_FUSE_REG);
1698 
1699     switch ((val >> 2) & 0x7) {
1700     case 3:
1701         i915->mem_freq = 2000;
1702         break;
1703     default:
1704         i915->mem_freq = 1600;
1705         break;
1706     }
1707     drm_dbg(&i915->drm, "DDR speed: %d MHz\n", i915->mem_freq);
1708 
1709     rps->max_freq = chv_rps_max_freq(rps);
1710     rps->rp0_freq = rps->max_freq;
1711     drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n",
1712         intel_gpu_freq(rps, rps->max_freq), rps->max_freq);
1713 
1714     rps->efficient_freq = chv_rps_rpe_freq(rps);
1715     drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n",
1716         intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq);
1717 
1718     rps->rp1_freq = chv_rps_guar_freq(rps);
1719     drm_dbg(&i915->drm, "RP1(Guar) GPU freq: %d MHz (%u)\n",
1720         intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq);
1721 
1722     rps->min_freq = chv_rps_min_freq(rps);
1723     drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n",
1724         intel_gpu_freq(rps, rps->min_freq), rps->min_freq);
1725 
1726     vlv_iosf_sb_put(i915,
1727             BIT(VLV_IOSF_SB_PUNIT) |
1728             BIT(VLV_IOSF_SB_NC) |
1729             BIT(VLV_IOSF_SB_CCK));
1730 
1731     drm_WARN_ONCE(&i915->drm, (rps->max_freq | rps->efficient_freq |
1732                    rps->rp1_freq | rps->min_freq) & 1,
1733               "Odd GPU freq values\n");
1734 }
1735 
1736 static void vlv_c0_read(struct intel_uncore *uncore, struct intel_rps_ei *ei)
1737 {
1738     ei->ktime = ktime_get_raw();
1739     ei->render_c0 = intel_uncore_read(uncore, VLV_RENDER_C0_COUNT);
1740     ei->media_c0 = intel_uncore_read(uncore, VLV_MEDIA_C0_COUNT);
1741 }
1742 
1743 static u32 vlv_wa_c0_ei(struct intel_rps *rps, u32 pm_iir)
1744 {
1745     struct intel_uncore *uncore = rps_to_uncore(rps);
1746     const struct intel_rps_ei *prev = &rps->ei;
1747     struct intel_rps_ei now;
1748     u32 events = 0;
1749 
1750     if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0)
1751         return 0;
1752 
1753     vlv_c0_read(uncore, &now);
1754 
1755     if (prev->ktime) {
1756         u64 time, c0;
1757         u32 render, media;
1758 
1759         time = ktime_us_delta(now.ktime, prev->ktime);
1760 
1761         time *= rps_to_i915(rps)->czclk_freq;
1762 
1763         /* Workload can be split between render + media,
1764          * e.g. SwapBuffers being blitted in X after being rendered in
1765          * mesa. To account for this we need to combine both engines
1766          * into our activity counter.
1767          */
1768         render = now.render_c0 - prev->render_c0;
1769         media = now.media_c0 - prev->media_c0;
1770         c0 = max(render, media);
1771         c0 *= 1000 * 100 << 8; /* to usecs and scale to threshold% */
1772 
1773         if (c0 > time * rps->power.up_threshold)
1774             events = GEN6_PM_RP_UP_THRESHOLD;
1775         else if (c0 < time * rps->power.down_threshold)
1776             events = GEN6_PM_RP_DOWN_THRESHOLD;
1777     }
1778 
1779     rps->ei = now;
1780     return events;
1781 }
1782 
1783 static void rps_work(struct work_struct *work)
1784 {
1785     struct intel_rps *rps = container_of(work, typeof(*rps), work);
1786     struct intel_gt *gt = rps_to_gt(rps);
1787     struct drm_i915_private *i915 = rps_to_i915(rps);
1788     bool client_boost = false;
1789     int new_freq, adj, min, max;
1790     u32 pm_iir = 0;
1791 
1792     spin_lock_irq(&gt->irq_lock);
1793     pm_iir = fetch_and_zero(&rps->pm_iir) & rps->pm_events;
1794     client_boost = atomic_read(&rps->num_waiters);
1795     spin_unlock_irq(&gt->irq_lock);
1796 
1797     /* Make sure we didn't queue anything we're not going to process. */
1798     if (!pm_iir && !client_boost)
1799         goto out;
1800 
1801     mutex_lock(&rps->lock);
1802     if (!intel_rps_is_active(rps)) {
1803         mutex_unlock(&rps->lock);
1804         return;
1805     }
1806 
1807     pm_iir |= vlv_wa_c0_ei(rps, pm_iir);
1808 
1809     adj = rps->last_adj;
1810     new_freq = rps->cur_freq;
1811     min = rps->min_freq_softlimit;
1812     max = rps->max_freq_softlimit;
1813     if (client_boost)
1814         max = rps->max_freq;
1815 
1816     GT_TRACE(gt,
1817          "pm_iir:%x, client_boost:%s, last:%d, cur:%x, min:%x, max:%x\n",
1818          pm_iir, str_yes_no(client_boost),
1819          adj, new_freq, min, max);
1820 
1821     if (client_boost && new_freq < rps->boost_freq) {
1822         new_freq = rps->boost_freq;
1823         adj = 0;
1824     } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
1825         if (adj > 0)
1826             adj *= 2;
1827         else /* CHV needs even encode values */
1828             adj = IS_CHERRYVIEW(gt->i915) ? 2 : 1;
1829 
1830         if (new_freq >= rps->max_freq_softlimit)
1831             adj = 0;
1832     } else if (client_boost) {
1833         adj = 0;
1834     } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
1835         if (rps->cur_freq > rps->efficient_freq)
1836             new_freq = rps->efficient_freq;
1837         else if (rps->cur_freq > rps->min_freq_softlimit)
1838             new_freq = rps->min_freq_softlimit;
1839         adj = 0;
1840     } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
1841         if (adj < 0)
1842             adj *= 2;
1843         else /* CHV needs even encode values */
1844             adj = IS_CHERRYVIEW(gt->i915) ? -2 : -1;
1845 
1846         if (new_freq <= rps->min_freq_softlimit)
1847             adj = 0;
1848     } else { /* unknown event */
1849         adj = 0;
1850     }
1851 
1852     /*
1853      * sysfs frequency limits may have snuck in while
1854      * servicing the interrupt
1855      */
1856     new_freq += adj;
1857     new_freq = clamp_t(int, new_freq, min, max);
1858 
1859     if (intel_rps_set(rps, new_freq)) {
1860         drm_dbg(&i915->drm, "Failed to set new GPU frequency\n");
1861         adj = 0;
1862     }
1863     rps->last_adj = adj;
1864 
1865     mutex_unlock(&rps->lock);
1866 
1867 out:
1868     spin_lock_irq(&gt->irq_lock);
1869     gen6_gt_pm_unmask_irq(gt, rps->pm_events);
1870     spin_unlock_irq(&gt->irq_lock);
1871 }
1872 
1873 void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)
1874 {
1875     struct intel_gt *gt = rps_to_gt(rps);
1876     const u32 events = rps->pm_events & pm_iir;
1877 
1878     lockdep_assert_held(&gt->irq_lock);
1879 
1880     if (unlikely(!events))
1881         return;
1882 
1883     GT_TRACE(gt, "irq events:%x\n", events);
1884 
1885     gen6_gt_pm_mask_irq(gt, events);
1886 
1887     rps->pm_iir |= events;
1888     schedule_work(&rps->work);
1889 }
1890 
1891 void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)
1892 {
1893     struct intel_gt *gt = rps_to_gt(rps);
1894     u32 events;
1895 
1896     events = pm_iir & rps->pm_events;
1897     if (events) {
1898         spin_lock(&gt->irq_lock);
1899 
1900         GT_TRACE(gt, "irq events:%x\n", events);
1901 
1902         gen6_gt_pm_mask_irq(gt, events);
1903         rps->pm_iir |= events;
1904 
1905         schedule_work(&rps->work);
1906         spin_unlock(&gt->irq_lock);
1907     }
1908 
1909     if (GRAPHICS_VER(gt->i915) >= 8)
1910         return;
1911 
1912     if (pm_iir & PM_VEBOX_USER_INTERRUPT)
1913         intel_engine_cs_irq(gt->engine[VECS0], pm_iir >> 10);
1914 
1915     if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
1916         DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir);
1917 }
1918 
1919 void gen5_rps_irq_handler(struct intel_rps *rps)
1920 {
1921     struct intel_uncore *uncore = rps_to_uncore(rps);
1922     u32 busy_up, busy_down, max_avg, min_avg;
1923     u8 new_freq;
1924 
1925     spin_lock(&mchdev_lock);
1926 
1927     intel_uncore_write16(uncore,
1928                  MEMINTRSTS,
1929                  intel_uncore_read(uncore, MEMINTRSTS));
1930 
1931     intel_uncore_write16(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);
1932     busy_up = intel_uncore_read(uncore, RCPREVBSYTUPAVG);
1933     busy_down = intel_uncore_read(uncore, RCPREVBSYTDNAVG);
1934     max_avg = intel_uncore_read(uncore, RCBMAXAVG);
1935     min_avg = intel_uncore_read(uncore, RCBMINAVG);
1936 
1937     /* Handle RCS change request from hw */
1938     new_freq = rps->cur_freq;
1939     if (busy_up > max_avg)
1940         new_freq++;
1941     else if (busy_down < min_avg)
1942         new_freq--;
1943     new_freq = clamp(new_freq,
1944              rps->min_freq_softlimit,
1945              rps->max_freq_softlimit);
1946 
1947     if (new_freq != rps->cur_freq && !__gen5_rps_set(rps, new_freq))
1948         rps->cur_freq = new_freq;
1949 
1950     spin_unlock(&mchdev_lock);
1951 }
1952 
1953 void intel_rps_init_early(struct intel_rps *rps)
1954 {
1955     mutex_init(&rps->lock);
1956     mutex_init(&rps->power.mutex);
1957 
1958     INIT_WORK(&rps->work, rps_work);
1959     timer_setup(&rps->timer, rps_timer, 0);
1960 
1961     atomic_set(&rps->num_waiters, 0);
1962 }
1963 
1964 void intel_rps_init(struct intel_rps *rps)
1965 {
1966     struct drm_i915_private *i915 = rps_to_i915(rps);
1967 
1968     if (rps_uses_slpc(rps))
1969         return;
1970 
1971     if (IS_CHERRYVIEW(i915))
1972         chv_rps_init(rps);
1973     else if (IS_VALLEYVIEW(i915))
1974         vlv_rps_init(rps);
1975     else if (GRAPHICS_VER(i915) >= 6)
1976         gen6_rps_init(rps);
1977     else if (IS_IRONLAKE_M(i915))
1978         gen5_rps_init(rps);
1979 
1980     /* Derive initial user preferences/limits from the hardware limits */
1981     rps->max_freq_softlimit = rps->max_freq;
1982     rps->min_freq_softlimit = rps->min_freq;
1983 
1984     /* After setting max-softlimit, find the overclock max freq */
1985     if (GRAPHICS_VER(i915) == 6 || IS_IVYBRIDGE(i915) || IS_HASWELL(i915)) {
1986         u32 params = 0;
1987 
1988         snb_pcode_read(rps_to_gt(rps)->uncore, GEN6_READ_OC_PARAMS, &params, NULL);
1989         if (params & BIT(31)) { /* OC supported */
1990             drm_dbg(&i915->drm,
1991                 "Overclocking supported, max: %dMHz, overclock: %dMHz\n",
1992                 (rps->max_freq & 0xff) * 50,
1993                 (params & 0xff) * 50);
1994             rps->max_freq = params & 0xff;
1995         }
1996     }
1997 
1998     /* Finally allow us to boost to max by default */
1999     rps->boost_freq = rps->max_freq;
2000     rps->idle_freq = rps->min_freq;
2001 
2002     /* Start in the middle, from here we will autotune based on workload */
2003     rps->cur_freq = rps->efficient_freq;
2004 
2005     rps->pm_intrmsk_mbz = 0;
2006 
2007     /*
2008      * SNB,IVB,HSW can while VLV,CHV may hard hang on looping batchbuffer
2009      * if GEN6_PM_UP_EI_EXPIRED is masked.
2010      *
2011      * TODO: verify if this can be reproduced on VLV,CHV.
2012      */
2013     if (GRAPHICS_VER(i915) <= 7)
2014         rps->pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED;
2015 
2016     if (GRAPHICS_VER(i915) >= 8 && GRAPHICS_VER(i915) < 11)
2017         rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC;
2018 
2019     /* GuC needs ARAT expired interrupt unmasked */
2020     if (intel_uc_uses_guc_submission(&rps_to_gt(rps)->uc))
2021         rps->pm_intrmsk_mbz |= ARAT_EXPIRED_INTRMSK;
2022 }
2023 
2024 void intel_rps_sanitize(struct intel_rps *rps)
2025 {
2026     if (rps_uses_slpc(rps))
2027         return;
2028 
2029     if (GRAPHICS_VER(rps_to_i915(rps)) >= 6)
2030         rps_disable_interrupts(rps);
2031 }
2032 
2033 u32 intel_rps_get_cagf(struct intel_rps *rps, u32 rpstat)
2034 {
2035     struct drm_i915_private *i915 = rps_to_i915(rps);
2036     u32 cagf;
2037 
2038     if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
2039         cagf = (rpstat >> 8) & 0xff;
2040     else if (GRAPHICS_VER(i915) >= 9)
2041         cagf = (rpstat & GEN9_CAGF_MASK) >> GEN9_CAGF_SHIFT;
2042     else if (IS_HASWELL(i915) || IS_BROADWELL(i915))
2043         cagf = (rpstat & HSW_CAGF_MASK) >> HSW_CAGF_SHIFT;
2044     else if (GRAPHICS_VER(i915) >= 6)
2045         cagf = (rpstat & GEN6_CAGF_MASK) >> GEN6_CAGF_SHIFT;
2046     else
2047         cagf = gen5_invert_freq(rps, (rpstat & MEMSTAT_PSTATE_MASK) >>
2048                     MEMSTAT_PSTATE_SHIFT);
2049 
2050     return cagf;
2051 }
2052 
2053 static u32 read_cagf(struct intel_rps *rps)
2054 {
2055     struct drm_i915_private *i915 = rps_to_i915(rps);
2056     struct intel_uncore *uncore = rps_to_uncore(rps);
2057     u32 freq;
2058 
2059     if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) {
2060         vlv_punit_get(i915);
2061         freq = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
2062         vlv_punit_put(i915);
2063     } else if (GRAPHICS_VER(i915) >= 6) {
2064         freq = intel_uncore_read(uncore, GEN6_RPSTAT1);
2065     } else {
2066         freq = intel_uncore_read(uncore, MEMSTAT_ILK);
2067     }
2068 
2069     return intel_rps_get_cagf(rps, freq);
2070 }
2071 
2072 u32 intel_rps_read_actual_frequency(struct intel_rps *rps)
2073 {
2074     struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm;
2075     intel_wakeref_t wakeref;
2076     u32 freq = 0;
2077 
2078     with_intel_runtime_pm_if_in_use(rpm, wakeref)
2079         freq = intel_gpu_freq(rps, read_cagf(rps));
2080 
2081     return freq;
2082 }
2083 
2084 u32 intel_rps_read_punit_req(struct intel_rps *rps)
2085 {
2086     struct intel_uncore *uncore = rps_to_uncore(rps);
2087     struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm;
2088     intel_wakeref_t wakeref;
2089     u32 freq = 0;
2090 
2091     with_intel_runtime_pm_if_in_use(rpm, wakeref)
2092         freq = intel_uncore_read(uncore, GEN6_RPNSWREQ);
2093 
2094     return freq;
2095 }
2096 
2097 static u32 intel_rps_get_req(u32 pureq)
2098 {
2099     u32 req = pureq >> GEN9_SW_REQ_UNSLICE_RATIO_SHIFT;
2100 
2101     return req;
2102 }
2103 
2104 u32 intel_rps_read_punit_req_frequency(struct intel_rps *rps)
2105 {
2106     u32 freq = intel_rps_get_req(intel_rps_read_punit_req(rps));
2107 
2108     return intel_gpu_freq(rps, freq);
2109 }
2110 
2111 u32 intel_rps_get_requested_frequency(struct intel_rps *rps)
2112 {
2113     if (rps_uses_slpc(rps))
2114         return intel_rps_read_punit_req_frequency(rps);
2115     else
2116         return intel_gpu_freq(rps, rps->cur_freq);
2117 }
2118 
2119 u32 intel_rps_get_max_frequency(struct intel_rps *rps)
2120 {
2121     struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2122 
2123     if (rps_uses_slpc(rps))
2124         return slpc->max_freq_softlimit;
2125     else
2126         return intel_gpu_freq(rps, rps->max_freq_softlimit);
2127 }
2128 
2129 /**
2130  * intel_rps_get_max_raw_freq - returns the max frequency in some raw format.
2131  * @rps: the intel_rps structure
2132  *
2133  * Returns the max frequency in a raw format. In newer platforms raw is in
2134  * units of 50 MHz.
2135  */
2136 u32 intel_rps_get_max_raw_freq(struct intel_rps *rps)
2137 {
2138     struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2139     u32 freq;
2140 
2141     if (rps_uses_slpc(rps)) {
2142         return DIV_ROUND_CLOSEST(slpc->rp0_freq,
2143                      GT_FREQUENCY_MULTIPLIER);
2144     } else {
2145         freq = rps->max_freq;
2146         if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) {
2147             /* Convert GT frequency to 50 MHz units */
2148             freq /= GEN9_FREQ_SCALER;
2149         }
2150         return freq;
2151     }
2152 }
2153 
2154 u32 intel_rps_get_rp0_frequency(struct intel_rps *rps)
2155 {
2156     struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2157 
2158     if (rps_uses_slpc(rps))
2159         return slpc->rp0_freq;
2160     else
2161         return intel_gpu_freq(rps, rps->rp0_freq);
2162 }
2163 
2164 u32 intel_rps_get_rp1_frequency(struct intel_rps *rps)
2165 {
2166     struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2167 
2168     if (rps_uses_slpc(rps))
2169         return slpc->rp1_freq;
2170     else
2171         return intel_gpu_freq(rps, rps->rp1_freq);
2172 }
2173 
2174 u32 intel_rps_get_rpn_frequency(struct intel_rps *rps)
2175 {
2176     struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2177 
2178     if (rps_uses_slpc(rps))
2179         return slpc->min_freq;
2180     else
2181         return intel_gpu_freq(rps, rps->min_freq);
2182 }
2183 
2184 static int set_max_freq(struct intel_rps *rps, u32 val)
2185 {
2186     struct drm_i915_private *i915 = rps_to_i915(rps);
2187     int ret = 0;
2188 
2189     mutex_lock(&rps->lock);
2190 
2191     val = intel_freq_opcode(rps, val);
2192     if (val < rps->min_freq ||
2193         val > rps->max_freq ||
2194         val < rps->min_freq_softlimit) {
2195         ret = -EINVAL;
2196         goto unlock;
2197     }
2198 
2199     if (val > rps->rp0_freq)
2200         drm_dbg(&i915->drm, "User requested overclocking to %d\n",
2201             intel_gpu_freq(rps, val));
2202 
2203     rps->max_freq_softlimit = val;
2204 
2205     val = clamp_t(int, rps->cur_freq,
2206               rps->min_freq_softlimit,
2207               rps->max_freq_softlimit);
2208 
2209     /*
2210      * We still need *_set_rps to process the new max_delay and
2211      * update the interrupt limits and PMINTRMSK even though
2212      * frequency request may be unchanged.
2213      */
2214     intel_rps_set(rps, val);
2215 
2216 unlock:
2217     mutex_unlock(&rps->lock);
2218 
2219     return ret;
2220 }
2221 
2222 int intel_rps_set_max_frequency(struct intel_rps *rps, u32 val)
2223 {
2224     struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2225 
2226     if (rps_uses_slpc(rps))
2227         return intel_guc_slpc_set_max_freq(slpc, val);
2228     else
2229         return set_max_freq(rps, val);
2230 }
2231 
2232 u32 intel_rps_get_min_frequency(struct intel_rps *rps)
2233 {
2234     struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2235 
2236     if (rps_uses_slpc(rps))
2237         return slpc->min_freq_softlimit;
2238     else
2239         return intel_gpu_freq(rps, rps->min_freq_softlimit);
2240 }
2241 
2242 /**
2243  * intel_rps_get_min_raw_freq - returns the min frequency in some raw format.
2244  * @rps: the intel_rps structure
2245  *
2246  * Returns the min frequency in a raw format. In newer platforms raw is in
2247  * units of 50 MHz.
2248  */
2249 u32 intel_rps_get_min_raw_freq(struct intel_rps *rps)
2250 {
2251     struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2252     u32 freq;
2253 
2254     if (rps_uses_slpc(rps)) {
2255         return DIV_ROUND_CLOSEST(slpc->min_freq,
2256                      GT_FREQUENCY_MULTIPLIER);
2257     } else {
2258         freq = rps->min_freq;
2259         if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) {
2260             /* Convert GT frequency to 50 MHz units */
2261             freq /= GEN9_FREQ_SCALER;
2262         }
2263         return freq;
2264     }
2265 }
2266 
2267 static int set_min_freq(struct intel_rps *rps, u32 val)
2268 {
2269     int ret = 0;
2270 
2271     mutex_lock(&rps->lock);
2272 
2273     val = intel_freq_opcode(rps, val);
2274     if (val < rps->min_freq ||
2275         val > rps->max_freq ||
2276         val > rps->max_freq_softlimit) {
2277         ret = -EINVAL;
2278         goto unlock;
2279     }
2280 
2281     rps->min_freq_softlimit = val;
2282 
2283     val = clamp_t(int, rps->cur_freq,
2284               rps->min_freq_softlimit,
2285               rps->max_freq_softlimit);
2286 
2287     /*
2288      * We still need *_set_rps to process the new min_delay and
2289      * update the interrupt limits and PMINTRMSK even though
2290      * frequency request may be unchanged.
2291      */
2292     intel_rps_set(rps, val);
2293 
2294 unlock:
2295     mutex_unlock(&rps->lock);
2296 
2297     return ret;
2298 }
2299 
2300 int intel_rps_set_min_frequency(struct intel_rps *rps, u32 val)
2301 {
2302     struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2303 
2304     if (rps_uses_slpc(rps))
2305         return intel_guc_slpc_set_min_freq(slpc, val);
2306     else
2307         return set_min_freq(rps, val);
2308 }
2309 
2310 static void intel_rps_set_manual(struct intel_rps *rps, bool enable)
2311 {
2312     struct intel_uncore *uncore = rps_to_uncore(rps);
2313     u32 state = enable ? GEN9_RPSWCTL_ENABLE : GEN9_RPSWCTL_DISABLE;
2314 
2315     /* Allow punit to process software requests */
2316     intel_uncore_write(uncore, GEN6_RP_CONTROL, state);
2317 }
2318 
2319 void intel_rps_raise_unslice(struct intel_rps *rps)
2320 {
2321     struct intel_uncore *uncore = rps_to_uncore(rps);
2322 
2323     mutex_lock(&rps->lock);
2324 
2325     if (rps_uses_slpc(rps)) {
2326         /* RP limits have not been initialized yet for SLPC path */
2327         struct intel_rps_freq_caps caps;
2328 
2329         gen6_rps_get_freq_caps(rps, &caps);
2330 
2331         intel_rps_set_manual(rps, true);
2332         intel_uncore_write(uncore, GEN6_RPNSWREQ,
2333                    ((caps.rp0_freq <<
2334                    GEN9_SW_REQ_UNSLICE_RATIO_SHIFT) |
2335                    GEN9_IGNORE_SLICE_RATIO));
2336         intel_rps_set_manual(rps, false);
2337     } else {
2338         intel_rps_set(rps, rps->rp0_freq);
2339     }
2340 
2341     mutex_unlock(&rps->lock);
2342 }
2343 
2344 void intel_rps_lower_unslice(struct intel_rps *rps)
2345 {
2346     struct intel_uncore *uncore = rps_to_uncore(rps);
2347 
2348     mutex_lock(&rps->lock);
2349 
2350     if (rps_uses_slpc(rps)) {
2351         /* RP limits have not been initialized yet for SLPC path */
2352         struct intel_rps_freq_caps caps;
2353 
2354         gen6_rps_get_freq_caps(rps, &caps);
2355 
2356         intel_rps_set_manual(rps, true);
2357         intel_uncore_write(uncore, GEN6_RPNSWREQ,
2358                    ((caps.min_freq <<
2359                    GEN9_SW_REQ_UNSLICE_RATIO_SHIFT) |
2360                    GEN9_IGNORE_SLICE_RATIO));
2361         intel_rps_set_manual(rps, false);
2362     } else {
2363         intel_rps_set(rps, rps->min_freq);
2364     }
2365 
2366     mutex_unlock(&rps->lock);
2367 }
2368 
2369 static u32 rps_read_mmio(struct intel_rps *rps, i915_reg_t reg32)
2370 {
2371     struct intel_gt *gt = rps_to_gt(rps);
2372     intel_wakeref_t wakeref;
2373     u32 val;
2374 
2375     with_intel_runtime_pm(gt->uncore->rpm, wakeref)
2376         val = intel_uncore_read(gt->uncore, reg32);
2377 
2378     return val;
2379 }
2380 
2381 bool rps_read_mask_mmio(struct intel_rps *rps,
2382             i915_reg_t reg32, u32 mask)
2383 {
2384     return rps_read_mmio(rps, reg32) & mask;
2385 }
2386 
2387 /* External interface for intel_ips.ko */
2388 
2389 static struct drm_i915_private __rcu *ips_mchdev;
2390 
2391 /**
2392  * Tells the intel_ips driver that the i915 driver is now loaded, if
2393  * IPS got loaded first.
2394  *
2395  * This awkward dance is so that neither module has to depend on the
2396  * other in order for IPS to do the appropriate communication of
2397  * GPU turbo limits to i915.
2398  */
2399 static void
2400 ips_ping_for_i915_load(void)
2401 {
2402     void (*link)(void);
2403 
2404     link = symbol_get(ips_link_to_i915_driver);
2405     if (link) {
2406         link();
2407         symbol_put(ips_link_to_i915_driver);
2408     }
2409 }
2410 
2411 void intel_rps_driver_register(struct intel_rps *rps)
2412 {
2413     struct intel_gt *gt = rps_to_gt(rps);
2414 
2415     /*
2416      * We only register the i915 ips part with intel-ips once everything is
2417      * set up, to avoid intel-ips sneaking in and reading bogus values.
2418      */
2419     if (GRAPHICS_VER(gt->i915) == 5) {
2420         GEM_BUG_ON(ips_mchdev);
2421         rcu_assign_pointer(ips_mchdev, gt->i915);
2422         ips_ping_for_i915_load();
2423     }
2424 }
2425 
2426 void intel_rps_driver_unregister(struct intel_rps *rps)
2427 {
2428     if (rcu_access_pointer(ips_mchdev) == rps_to_i915(rps))
2429         rcu_assign_pointer(ips_mchdev, NULL);
2430 }
2431 
2432 static struct drm_i915_private *mchdev_get(void)
2433 {
2434     struct drm_i915_private *i915;
2435 
2436     rcu_read_lock();
2437     i915 = rcu_dereference(ips_mchdev);
2438     if (i915 && !kref_get_unless_zero(&i915->drm.ref))
2439         i915 = NULL;
2440     rcu_read_unlock();
2441 
2442     return i915;
2443 }
2444 
2445 /**
2446  * i915_read_mch_val - return value for IPS use
2447  *
2448  * Calculate and return a value for the IPS driver to use when deciding whether
2449  * we have thermal and power headroom to increase CPU or GPU power budget.
2450  */
2451 unsigned long i915_read_mch_val(void)
2452 {
2453     struct drm_i915_private *i915;
2454     unsigned long chipset_val = 0;
2455     unsigned long graphics_val = 0;
2456     intel_wakeref_t wakeref;
2457 
2458     i915 = mchdev_get();
2459     if (!i915)
2460         return 0;
2461 
2462     with_intel_runtime_pm(&i915->runtime_pm, wakeref) {
2463         struct intel_ips *ips = &to_gt(i915)->rps.ips;
2464 
2465         spin_lock_irq(&mchdev_lock);
2466         chipset_val = __ips_chipset_val(ips);
2467         graphics_val = __ips_gfx_val(ips);
2468         spin_unlock_irq(&mchdev_lock);
2469     }
2470 
2471     drm_dev_put(&i915->drm);
2472     return chipset_val + graphics_val;
2473 }
2474 EXPORT_SYMBOL_GPL(i915_read_mch_val);
2475 
2476 /**
2477  * i915_gpu_raise - raise GPU frequency limit
2478  *
2479  * Raise the limit; IPS indicates we have thermal headroom.
2480  */
2481 bool i915_gpu_raise(void)
2482 {
2483     struct drm_i915_private *i915;
2484     struct intel_rps *rps;
2485 
2486     i915 = mchdev_get();
2487     if (!i915)
2488         return false;
2489 
2490     rps = &to_gt(i915)->rps;
2491 
2492     spin_lock_irq(&mchdev_lock);
2493     if (rps->max_freq_softlimit < rps->max_freq)
2494         rps->max_freq_softlimit++;
2495     spin_unlock_irq(&mchdev_lock);
2496 
2497     drm_dev_put(&i915->drm);
2498     return true;
2499 }
2500 EXPORT_SYMBOL_GPL(i915_gpu_raise);
2501 
2502 /**
2503  * i915_gpu_lower - lower GPU frequency limit
2504  *
2505  * IPS indicates we're close to a thermal limit, so throttle back the GPU
2506  * frequency maximum.
2507  */
2508 bool i915_gpu_lower(void)
2509 {
2510     struct drm_i915_private *i915;
2511     struct intel_rps *rps;
2512 
2513     i915 = mchdev_get();
2514     if (!i915)
2515         return false;
2516 
2517     rps = &to_gt(i915)->rps;
2518 
2519     spin_lock_irq(&mchdev_lock);
2520     if (rps->max_freq_softlimit > rps->min_freq)
2521         rps->max_freq_softlimit--;
2522     spin_unlock_irq(&mchdev_lock);
2523 
2524     drm_dev_put(&i915->drm);
2525     return true;
2526 }
2527 EXPORT_SYMBOL_GPL(i915_gpu_lower);
2528 
2529 /**
2530  * i915_gpu_busy - indicate GPU business to IPS
2531  *
2532  * Tell the IPS driver whether or not the GPU is busy.
2533  */
2534 bool i915_gpu_busy(void)
2535 {
2536     struct drm_i915_private *i915;
2537     bool ret;
2538 
2539     i915 = mchdev_get();
2540     if (!i915)
2541         return false;
2542 
2543     ret = to_gt(i915)->awake;
2544 
2545     drm_dev_put(&i915->drm);
2546     return ret;
2547 }
2548 EXPORT_SYMBOL_GPL(i915_gpu_busy);
2549 
2550 /**
2551  * i915_gpu_turbo_disable - disable graphics turbo
2552  *
2553  * Disable graphics turbo by resetting the max frequency and setting the
2554  * current frequency to the default.
2555  */
2556 bool i915_gpu_turbo_disable(void)
2557 {
2558     struct drm_i915_private *i915;
2559     struct intel_rps *rps;
2560     bool ret;
2561 
2562     i915 = mchdev_get();
2563     if (!i915)
2564         return false;
2565 
2566     rps = &to_gt(i915)->rps;
2567 
2568     spin_lock_irq(&mchdev_lock);
2569     rps->max_freq_softlimit = rps->min_freq;
2570     ret = !__gen5_rps_set(&to_gt(i915)->rps, rps->min_freq);
2571     spin_unlock_irq(&mchdev_lock);
2572 
2573     drm_dev_put(&i915->drm);
2574     return ret;
2575 }
2576 EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
2577 
2578 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
2579 #include "selftest_rps.c"
2580 #include "selftest_slpc.c"
2581 #endif
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!