Lines Matching refs:rps
26 static struct intel_gt *rps_to_gt(struct intel_rps *rps) in rps_to_gt() argument
28 return container_of(rps, struct intel_gt, rps); in rps_to_gt()
31 static struct drm_i915_private *rps_to_i915(struct intel_rps *rps) in rps_to_i915() argument
33 return rps_to_gt(rps)->i915; in rps_to_i915()
36 static struct intel_uncore *rps_to_uncore(struct intel_rps *rps) in rps_to_uncore() argument
38 return rps_to_gt(rps)->uncore; in rps_to_uncore()
41 static struct intel_guc_slpc *rps_to_slpc(struct intel_rps *rps) in rps_to_slpc() argument
43 struct intel_gt *gt = rps_to_gt(rps); in rps_to_slpc()
48 static bool rps_uses_slpc(struct intel_rps *rps) in rps_uses_slpc() argument
50 struct intel_gt *gt = rps_to_gt(rps); in rps_uses_slpc()
55 static u32 rps_pm_sanitize_mask(struct intel_rps *rps, u32 mask) in rps_pm_sanitize_mask() argument
57 return mask & ~rps->pm_intrmsk_mbz; in rps_pm_sanitize_mask()
67 struct intel_rps *rps = from_timer(rps, t, timer); in rps_timer() local
74 for_each_engine(engine, rps_to_gt(rps), id) { in rps_timer()
79 last = engine->stats.rps; in rps_timer()
80 engine->stats.rps = dt; in rps_timer()
88 last = rps->pm_timestamp; in rps_timer()
89 rps->pm_timestamp = timestamp; in rps_timer()
91 if (intel_rps_is_active(rps)) { in rps_timer()
118 GT_TRACE(rps_to_gt(rps), in rps_timer()
122 rps->pm_interval); in rps_timer()
124 if (100 * busy > rps->power.up_threshold * dt && in rps_timer()
125 rps->cur_freq < rps->max_freq_softlimit) { in rps_timer()
126 rps->pm_iir |= GEN6_PM_RP_UP_THRESHOLD; in rps_timer()
127 rps->pm_interval = 1; in rps_timer()
128 schedule_work(&rps->work); in rps_timer()
129 } else if (100 * busy < rps->power.down_threshold * dt && in rps_timer()
130 rps->cur_freq > rps->min_freq_softlimit) { in rps_timer()
131 rps->pm_iir |= GEN6_PM_RP_DOWN_THRESHOLD; in rps_timer()
132 rps->pm_interval = 1; in rps_timer()
133 schedule_work(&rps->work); in rps_timer()
135 rps->last_adj = 0; in rps_timer()
138 mod_timer(&rps->timer, in rps_timer()
139 jiffies + msecs_to_jiffies(rps->pm_interval)); in rps_timer()
140 rps->pm_interval = min(rps->pm_interval * 2, BUSY_MAX_EI); in rps_timer()
144 static void rps_start_timer(struct intel_rps *rps) in rps_start_timer() argument
146 rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp); in rps_start_timer()
147 rps->pm_interval = 1; in rps_start_timer()
148 mod_timer(&rps->timer, jiffies + 1); in rps_start_timer()
151 static void rps_stop_timer(struct intel_rps *rps) in rps_stop_timer() argument
153 del_timer_sync(&rps->timer); in rps_stop_timer()
154 rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp); in rps_stop_timer()
155 cancel_work_sync(&rps->work); in rps_stop_timer()
158 static u32 rps_pm_mask(struct intel_rps *rps, u8 val) in rps_pm_mask() argument
163 if (val > rps->min_freq_softlimit) in rps_pm_mask()
168 if (val < rps->max_freq_softlimit) in rps_pm_mask()
171 mask &= rps->pm_events; in rps_pm_mask()
173 return rps_pm_sanitize_mask(rps, ~mask); in rps_pm_mask()
176 static void rps_reset_ei(struct intel_rps *rps) in rps_reset_ei() argument
178 memset(&rps->ei, 0, sizeof(rps->ei)); in rps_reset_ei()
181 static void rps_enable_interrupts(struct intel_rps *rps) in rps_enable_interrupts() argument
183 struct intel_gt *gt = rps_to_gt(rps); in rps_enable_interrupts()
185 GEM_BUG_ON(rps_uses_slpc(rps)); in rps_enable_interrupts()
188 rps->pm_events, rps_pm_mask(rps, rps->last_freq)); in rps_enable_interrupts()
190 rps_reset_ei(rps); in rps_enable_interrupts()
193 gen6_gt_pm_enable_irq(gt, rps->pm_events); in rps_enable_interrupts()
197 GEN6_PMINTRMSK, rps_pm_mask(rps, rps->last_freq)); in rps_enable_interrupts()
200 static void gen6_rps_reset_interrupts(struct intel_rps *rps) in gen6_rps_reset_interrupts() argument
202 gen6_gt_pm_reset_iir(rps_to_gt(rps), GEN6_PM_RPS_EVENTS); in gen6_rps_reset_interrupts()
205 static void gen11_rps_reset_interrupts(struct intel_rps *rps) in gen11_rps_reset_interrupts() argument
207 while (gen11_gt_reset_one_iir(rps_to_gt(rps), 0, GEN11_GTPM)) in gen11_rps_reset_interrupts()
211 static void rps_reset_interrupts(struct intel_rps *rps) in rps_reset_interrupts() argument
213 struct intel_gt *gt = rps_to_gt(rps); in rps_reset_interrupts()
217 gen11_rps_reset_interrupts(rps); in rps_reset_interrupts()
219 gen6_rps_reset_interrupts(rps); in rps_reset_interrupts()
221 rps->pm_iir = 0; in rps_reset_interrupts()
225 static void rps_disable_interrupts(struct intel_rps *rps) in rps_disable_interrupts() argument
227 struct intel_gt *gt = rps_to_gt(rps); in rps_disable_interrupts()
230 GEN6_PMINTRMSK, rps_pm_sanitize_mask(rps, ~0u)); in rps_disable_interrupts()
244 cancel_work_sync(&rps->work); in rps_disable_interrupts()
246 rps_reset_interrupts(rps); in rps_disable_interrupts()
264 static void gen5_rps_init(struct intel_rps *rps) in gen5_rps_init() argument
266 struct drm_i915_private *i915 = rps_to_i915(rps); in gen5_rps_init()
267 struct intel_uncore *uncore = rps_to_uncore(rps); in gen5_rps_init()
281 rps->ips.m = cparams[i].m; in gen5_rps_init()
282 rps->ips.c = cparams[i].c; in gen5_rps_init()
297 rps->min_freq = fmax; in gen5_rps_init()
298 rps->efficient_freq = fstart; in gen5_rps_init()
299 rps->max_freq = fmin; in gen5_rps_init()
402 static void gen5_rps_update(struct intel_rps *rps) in gen5_rps_update() argument
405 __gen5_ips_update(&rps->ips); in gen5_rps_update()
409 static unsigned int gen5_invert_freq(struct intel_rps *rps, in gen5_invert_freq() argument
413 val = rps->max_freq - val; in gen5_invert_freq()
414 val = rps->min_freq + val; in gen5_invert_freq()
419 static int __gen5_rps_set(struct intel_rps *rps, u8 val) in __gen5_rps_set() argument
421 struct intel_uncore *uncore = rps_to_uncore(rps); in __gen5_rps_set()
433 val = gen5_invert_freq(rps, val); in __gen5_rps_set()
448 static int gen5_rps_set(struct intel_rps *rps, u8 val) in gen5_rps_set() argument
453 err = __gen5_rps_set(rps, val); in gen5_rps_set()
537 static bool gen5_rps_enable(struct intel_rps *rps) in gen5_rps_enable() argument
539 struct drm_i915_private *i915 = rps_to_i915(rps); in gen5_rps_enable()
540 struct intel_uncore *uncore = rps_to_uncore(rps); in gen5_rps_enable()
587 __gen5_rps_set(rps, rps->cur_freq); in gen5_rps_enable()
589 rps->ips.last_count1 = intel_uncore_read(uncore, DMIEC); in gen5_rps_enable()
590 rps->ips.last_count1 += intel_uncore_read(uncore, DDREC); in gen5_rps_enable()
591 rps->ips.last_count1 += intel_uncore_read(uncore, CSIEC); in gen5_rps_enable()
592 rps->ips.last_time1 = jiffies_to_msecs(jiffies); in gen5_rps_enable()
594 rps->ips.last_count2 = intel_uncore_read(uncore, GFXEC); in gen5_rps_enable()
595 rps->ips.last_time2 = ktime_get_raw_ns(); in gen5_rps_enable()
603 rps->ips.corr = init_emon(uncore); in gen5_rps_enable()
608 static void gen5_rps_disable(struct intel_rps *rps) in gen5_rps_disable() argument
610 struct drm_i915_private *i915 = rps_to_i915(rps); in gen5_rps_disable()
611 struct intel_uncore *uncore = rps_to_uncore(rps); in gen5_rps_disable()
629 __gen5_rps_set(rps, rps->idle_freq); in gen5_rps_disable()
638 static u32 rps_limits(struct intel_rps *rps, u8 val) in rps_limits() argument
650 if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) { in rps_limits()
651 limits = rps->max_freq_softlimit << 23; in rps_limits()
652 if (val <= rps->min_freq_softlimit) in rps_limits()
653 limits |= rps->min_freq_softlimit << 14; in rps_limits()
655 limits = rps->max_freq_softlimit << 24; in rps_limits()
656 if (val <= rps->min_freq_softlimit) in rps_limits()
657 limits |= rps->min_freq_softlimit << 16; in rps_limits()
663 static void rps_set_power(struct intel_rps *rps, int new_power) in rps_set_power() argument
665 struct intel_gt *gt = rps_to_gt(rps); in rps_set_power()
670 lockdep_assert_held(&rps->power.mutex); in rps_set_power()
672 if (new_power == rps->power.mode) in rps_set_power()
725 rps->power.mode = new_power; in rps_set_power()
726 rps->power.up_threshold = threshold_up; in rps_set_power()
727 rps->power.down_threshold = threshold_down; in rps_set_power()
730 static void gen6_rps_set_thresholds(struct intel_rps *rps, u8 val) in gen6_rps_set_thresholds() argument
734 new_power = rps->power.mode; in gen6_rps_set_thresholds()
735 switch (rps->power.mode) { in gen6_rps_set_thresholds()
737 if (val > rps->efficient_freq + 1 && in gen6_rps_set_thresholds()
738 val > rps->cur_freq) in gen6_rps_set_thresholds()
743 if (val <= rps->efficient_freq && in gen6_rps_set_thresholds()
744 val < rps->cur_freq) in gen6_rps_set_thresholds()
746 else if (val >= rps->rp0_freq && in gen6_rps_set_thresholds()
747 val > rps->cur_freq) in gen6_rps_set_thresholds()
752 if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 && in gen6_rps_set_thresholds()
753 val < rps->cur_freq) in gen6_rps_set_thresholds()
758 if (val <= rps->min_freq_softlimit) in gen6_rps_set_thresholds()
760 if (val >= rps->max_freq_softlimit) in gen6_rps_set_thresholds()
763 mutex_lock(&rps->power.mutex); in gen6_rps_set_thresholds()
764 if (rps->power.interactive) in gen6_rps_set_thresholds()
766 rps_set_power(rps, new_power); in gen6_rps_set_thresholds()
767 mutex_unlock(&rps->power.mutex); in gen6_rps_set_thresholds()
770 void intel_rps_mark_interactive(struct intel_rps *rps, bool interactive) in intel_rps_mark_interactive() argument
772 GT_TRACE(rps_to_gt(rps), "mark interactive: %s\n", yesno(interactive)); in intel_rps_mark_interactive()
774 mutex_lock(&rps->power.mutex); in intel_rps_mark_interactive()
776 if (!rps->power.interactive++ && intel_rps_is_active(rps)) in intel_rps_mark_interactive()
777 rps_set_power(rps, HIGH_POWER); in intel_rps_mark_interactive()
779 GEM_BUG_ON(!rps->power.interactive); in intel_rps_mark_interactive()
780 rps->power.interactive--; in intel_rps_mark_interactive()
782 mutex_unlock(&rps->power.mutex); in intel_rps_mark_interactive()
785 static int gen6_rps_set(struct intel_rps *rps, u8 val) in gen6_rps_set() argument
787 struct intel_uncore *uncore = rps_to_uncore(rps); in gen6_rps_set()
788 struct drm_i915_private *i915 = rps_to_i915(rps); in gen6_rps_set()
791 GEM_BUG_ON(rps_uses_slpc(rps)); in gen6_rps_set()
803 GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d, swreq:%x\n", in gen6_rps_set()
804 val, intel_gpu_freq(rps, val), swreq); in gen6_rps_set()
809 static int vlv_rps_set(struct intel_rps *rps, u8 val) in vlv_rps_set() argument
811 struct drm_i915_private *i915 = rps_to_i915(rps); in vlv_rps_set()
818 GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d\n", in vlv_rps_set()
819 val, intel_gpu_freq(rps, val)); in vlv_rps_set()
824 static int rps_set(struct intel_rps *rps, u8 val, bool update) in rps_set() argument
826 struct drm_i915_private *i915 = rps_to_i915(rps); in rps_set()
829 if (val == rps->last_freq) in rps_set()
833 err = vlv_rps_set(rps, val); in rps_set()
835 err = gen6_rps_set(rps, val); in rps_set()
837 err = gen5_rps_set(rps, val); in rps_set()
842 gen6_rps_set_thresholds(rps, val); in rps_set()
843 rps->last_freq = val; in rps_set()
848 void intel_rps_unpark(struct intel_rps *rps) in intel_rps_unpark() argument
850 if (!intel_rps_is_enabled(rps)) in intel_rps_unpark()
853 GT_TRACE(rps_to_gt(rps), "unpark:%x\n", rps->cur_freq); in intel_rps_unpark()
859 mutex_lock(&rps->lock); in intel_rps_unpark()
861 intel_rps_set_active(rps); in intel_rps_unpark()
862 intel_rps_set(rps, in intel_rps_unpark()
863 clamp(rps->cur_freq, in intel_rps_unpark()
864 rps->min_freq_softlimit, in intel_rps_unpark()
865 rps->max_freq_softlimit)); in intel_rps_unpark()
867 mutex_unlock(&rps->lock); in intel_rps_unpark()
869 rps->pm_iir = 0; in intel_rps_unpark()
870 if (intel_rps_has_interrupts(rps)) in intel_rps_unpark()
871 rps_enable_interrupts(rps); in intel_rps_unpark()
872 if (intel_rps_uses_timer(rps)) in intel_rps_unpark()
873 rps_start_timer(rps); in intel_rps_unpark()
875 if (GRAPHICS_VER(rps_to_i915(rps)) == 5) in intel_rps_unpark()
876 gen5_rps_update(rps); in intel_rps_unpark()
879 void intel_rps_park(struct intel_rps *rps) in intel_rps_park() argument
883 if (!intel_rps_is_enabled(rps)) in intel_rps_park()
886 if (!intel_rps_clear_active(rps)) in intel_rps_park()
889 if (intel_rps_uses_timer(rps)) in intel_rps_park()
890 rps_stop_timer(rps); in intel_rps_park()
891 if (intel_rps_has_interrupts(rps)) in intel_rps_park()
892 rps_disable_interrupts(rps); in intel_rps_park()
894 if (rps->last_freq <= rps->idle_freq) in intel_rps_park()
910 intel_uncore_forcewake_get(rps_to_uncore(rps), FORCEWAKE_MEDIA); in intel_rps_park()
911 rps_set(rps, rps->idle_freq, false); in intel_rps_park()
912 intel_uncore_forcewake_put(rps_to_uncore(rps), FORCEWAKE_MEDIA); in intel_rps_park()
924 adj = rps->last_adj; in intel_rps_park()
929 rps->last_adj = adj; in intel_rps_park()
930 rps->cur_freq = max_t(int, rps->cur_freq + adj, rps->min_freq); in intel_rps_park()
931 if (rps->cur_freq < rps->efficient_freq) { in intel_rps_park()
932 rps->cur_freq = rps->efficient_freq; in intel_rps_park()
933 rps->last_adj = 0; in intel_rps_park()
936 GT_TRACE(rps_to_gt(rps), "park:%x\n", rps->cur_freq); in intel_rps_park()
946 struct intel_rps *rps = &READ_ONCE(rq->engine)->gt->rps; in intel_rps_boost() local
948 if (atomic_fetch_inc(&rps->num_waiters)) in intel_rps_boost()
951 if (!intel_rps_is_active(rps)) in intel_rps_boost()
954 GT_TRACE(rps_to_gt(rps), "boost fence:%llx:%llx\n", in intel_rps_boost()
957 if (READ_ONCE(rps->cur_freq) < rps->boost_freq) in intel_rps_boost()
958 schedule_work(&rps->work); in intel_rps_boost()
960 WRITE_ONCE(rps->boosts, rps->boosts + 1); /* debug only */ in intel_rps_boost()
964 int intel_rps_set(struct intel_rps *rps, u8 val) in intel_rps_set() argument
968 lockdep_assert_held(&rps->lock); in intel_rps_set()
969 GEM_BUG_ON(val > rps->max_freq); in intel_rps_set()
970 GEM_BUG_ON(val < rps->min_freq); in intel_rps_set()
972 if (intel_rps_is_active(rps)) { in intel_rps_set()
973 err = rps_set(rps, val, true); in intel_rps_set()
981 if (intel_rps_has_interrupts(rps)) { in intel_rps_set()
982 struct intel_uncore *uncore = rps_to_uncore(rps); in intel_rps_set()
985 GEN6_RP_INTERRUPT_LIMITS, rps_limits(rps, val)); in intel_rps_set()
987 set(uncore, GEN6_PMINTRMSK, rps_pm_mask(rps, val)); in intel_rps_set()
991 rps->cur_freq = val; in intel_rps_set()
995 static void gen6_rps_init(struct intel_rps *rps) in gen6_rps_init() argument
997 struct drm_i915_private *i915 = rps_to_i915(rps); in gen6_rps_init()
998 u32 rp_state_cap = intel_rps_read_state_cap(rps); in gen6_rps_init()
1004 rps->rp0_freq = (rp_state_cap >> 16) & 0xff; in gen6_rps_init()
1005 rps->rp1_freq = (rp_state_cap >> 8) & 0xff; in gen6_rps_init()
1006 rps->min_freq = (rp_state_cap >> 0) & 0xff; in gen6_rps_init()
1008 rps->rp0_freq = (rp_state_cap >> 0) & 0xff; in gen6_rps_init()
1009 rps->rp1_freq = (rp_state_cap >> 8) & 0xff; in gen6_rps_init()
1010 rps->min_freq = (rp_state_cap >> 16) & 0xff; in gen6_rps_init()
1014 rps->max_freq = rps->rp0_freq; in gen6_rps_init()
1016 rps->efficient_freq = rps->rp1_freq; in gen6_rps_init()
1024 rps->efficient_freq = in gen6_rps_init()
1027 rps->min_freq, in gen6_rps_init()
1028 rps->max_freq); in gen6_rps_init()
1035 rps->rp0_freq *= GEN9_FREQ_SCALER; in gen6_rps_init()
1036 rps->rp1_freq *= GEN9_FREQ_SCALER; in gen6_rps_init()
1037 rps->min_freq *= GEN9_FREQ_SCALER; in gen6_rps_init()
1038 rps->max_freq *= GEN9_FREQ_SCALER; in gen6_rps_init()
1039 rps->efficient_freq *= GEN9_FREQ_SCALER; in gen6_rps_init()
1043 static bool rps_reset(struct intel_rps *rps) in rps_reset() argument
1045 struct drm_i915_private *i915 = rps_to_i915(rps); in rps_reset()
1048 rps->power.mode = -1; in rps_reset()
1049 rps->last_freq = -1; in rps_reset()
1051 if (rps_set(rps, rps->min_freq, true)) { in rps_reset()
1056 rps->cur_freq = rps->min_freq; in rps_reset()
1061 static bool gen9_rps_enable(struct intel_rps *rps) in gen9_rps_enable() argument
1063 struct intel_gt *gt = rps_to_gt(rps); in gen9_rps_enable()
1069 GEN9_FREQUENCY(rps->rp1_freq)); in gen9_rps_enable()
1073 rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD; in gen9_rps_enable()
1075 return rps_reset(rps); in gen9_rps_enable()
1078 static bool gen8_rps_enable(struct intel_rps *rps) in gen8_rps_enable() argument
1080 struct intel_uncore *uncore = rps_to_uncore(rps); in gen8_rps_enable()
1083 HSW_FREQUENCY(rps->rp1_freq)); in gen8_rps_enable()
1087 rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD; in gen8_rps_enable()
1089 return rps_reset(rps); in gen8_rps_enable()
1092 static bool gen6_rps_enable(struct intel_rps *rps) in gen6_rps_enable() argument
1094 struct intel_uncore *uncore = rps_to_uncore(rps); in gen6_rps_enable()
1100 rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD | in gen6_rps_enable()
1104 return rps_reset(rps); in gen6_rps_enable()
1107 static int chv_rps_max_freq(struct intel_rps *rps) in chv_rps_max_freq() argument
1109 struct drm_i915_private *i915 = rps_to_i915(rps); in chv_rps_max_freq()
1110 struct intel_gt *gt = rps_to_gt(rps); in chv_rps_max_freq()
1135 static int chv_rps_rpe_freq(struct intel_rps *rps) in chv_rps_rpe_freq() argument
1137 struct drm_i915_private *i915 = rps_to_i915(rps); in chv_rps_rpe_freq()
1146 static int chv_rps_guar_freq(struct intel_rps *rps) in chv_rps_guar_freq() argument
1148 struct drm_i915_private *i915 = rps_to_i915(rps); in chv_rps_guar_freq()
1156 static u32 chv_rps_min_freq(struct intel_rps *rps) in chv_rps_min_freq() argument
1158 struct drm_i915_private *i915 = rps_to_i915(rps); in chv_rps_min_freq()
1167 static bool chv_rps_enable(struct intel_rps *rps) in chv_rps_enable() argument
1169 struct intel_uncore *uncore = rps_to_uncore(rps); in chv_rps_enable()
1170 struct drm_i915_private *i915 = rps_to_i915(rps); in chv_rps_enable()
1190 rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD | in chv_rps_enable()
1211 return rps_reset(rps); in chv_rps_enable()
1214 static int vlv_rps_guar_freq(struct intel_rps *rps) in vlv_rps_guar_freq() argument
1216 struct drm_i915_private *i915 = rps_to_i915(rps); in vlv_rps_guar_freq()
1227 static int vlv_rps_max_freq(struct intel_rps *rps) in vlv_rps_max_freq() argument
1229 struct drm_i915_private *i915 = rps_to_i915(rps); in vlv_rps_max_freq()
1241 static int vlv_rps_rpe_freq(struct intel_rps *rps) in vlv_rps_rpe_freq() argument
1243 struct drm_i915_private *i915 = rps_to_i915(rps); in vlv_rps_rpe_freq()
1254 static int vlv_rps_min_freq(struct intel_rps *rps) in vlv_rps_min_freq() argument
1256 struct drm_i915_private *i915 = rps_to_i915(rps); in vlv_rps_min_freq()
1270 static bool vlv_rps_enable(struct intel_rps *rps) in vlv_rps_enable() argument
1272 struct intel_uncore *uncore = rps_to_uncore(rps); in vlv_rps_enable()
1273 struct drm_i915_private *i915 = rps_to_i915(rps); in vlv_rps_enable()
1293 rps->pm_events = GEN6_PM_RP_UP_EI_EXPIRED; in vlv_rps_enable()
1312 return rps_reset(rps); in vlv_rps_enable()
1317 struct intel_rps *rps = container_of(ips, typeof(*rps), ips); in __ips_gfx_val() local
1318 struct intel_uncore *uncore = rps_to_uncore(rps); in __ips_gfx_val()
1325 pxvid = intel_uncore_read(uncore, PXVFREQ(rps->cur_freq)); in __ips_gfx_val()
1327 ext_v = pvid_to_extvid(rps_to_i915(rps), pxvid); in __ips_gfx_val()
1353 static bool has_busy_stats(struct intel_rps *rps) in has_busy_stats() argument
1358 for_each_engine(engine, rps_to_gt(rps), id) { in has_busy_stats()
1366 void intel_rps_enable(struct intel_rps *rps) in intel_rps_enable() argument
1368 struct drm_i915_private *i915 = rps_to_i915(rps); in intel_rps_enable()
1369 struct intel_uncore *uncore = rps_to_uncore(rps); in intel_rps_enable()
1375 if (rps_uses_slpc(rps)) in intel_rps_enable()
1378 intel_gt_check_clock_frequency(rps_to_gt(rps)); in intel_rps_enable()
1381 if (rps->max_freq <= rps->min_freq) in intel_rps_enable()
1384 enabled = chv_rps_enable(rps); in intel_rps_enable()
1386 enabled = vlv_rps_enable(rps); in intel_rps_enable()
1388 enabled = gen9_rps_enable(rps); in intel_rps_enable()
1390 enabled = gen8_rps_enable(rps); in intel_rps_enable()
1392 enabled = gen6_rps_enable(rps); in intel_rps_enable()
1394 enabled = gen5_rps_enable(rps); in intel_rps_enable()
1401 GT_TRACE(rps_to_gt(rps), in intel_rps_enable()
1403 rps->min_freq, rps->max_freq, in intel_rps_enable()
1404 intel_gpu_freq(rps, rps->min_freq), in intel_rps_enable()
1405 intel_gpu_freq(rps, rps->max_freq)); in intel_rps_enable()
1407 GEM_BUG_ON(rps->max_freq < rps->min_freq); in intel_rps_enable()
1408 GEM_BUG_ON(rps->idle_freq > rps->max_freq); in intel_rps_enable()
1410 GEM_BUG_ON(rps->efficient_freq < rps->min_freq); in intel_rps_enable()
1411 GEM_BUG_ON(rps->efficient_freq > rps->max_freq); in intel_rps_enable()
1413 if (has_busy_stats(rps)) in intel_rps_enable()
1414 intel_rps_set_timer(rps); in intel_rps_enable()
1416 intel_rps_set_interrupts(rps); in intel_rps_enable()
1420 intel_rps_set_enabled(rps); in intel_rps_enable()
1423 static void gen6_rps_disable(struct intel_rps *rps) in gen6_rps_disable() argument
1425 set(rps_to_uncore(rps), GEN6_RP_CONTROL, 0); in gen6_rps_disable()
1428 void intel_rps_disable(struct intel_rps *rps) in intel_rps_disable() argument
1430 struct drm_i915_private *i915 = rps_to_i915(rps); in intel_rps_disable()
1432 intel_rps_clear_enabled(rps); in intel_rps_disable()
1433 intel_rps_clear_interrupts(rps); in intel_rps_disable()
1434 intel_rps_clear_timer(rps); in intel_rps_disable()
1437 gen6_rps_disable(rps); in intel_rps_disable()
1439 gen5_rps_disable(rps); in intel_rps_disable()
1442 static int byt_gpu_freq(struct intel_rps *rps, int val) in byt_gpu_freq() argument
1448 return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000); in byt_gpu_freq()
1451 static int byt_freq_opcode(struct intel_rps *rps, int val) in byt_freq_opcode() argument
1453 return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7; in byt_freq_opcode()
1456 static int chv_gpu_freq(struct intel_rps *rps, int val) in chv_gpu_freq() argument
1462 return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000); in chv_gpu_freq()
1465 static int chv_freq_opcode(struct intel_rps *rps, int val) in chv_freq_opcode() argument
1468 return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2; in chv_freq_opcode()
1471 int intel_gpu_freq(struct intel_rps *rps, int val) in intel_gpu_freq() argument
1473 struct drm_i915_private *i915 = rps_to_i915(rps); in intel_gpu_freq()
1479 return chv_gpu_freq(rps, val); in intel_gpu_freq()
1481 return byt_gpu_freq(rps, val); in intel_gpu_freq()
1488 int intel_freq_opcode(struct intel_rps *rps, int val) in intel_freq_opcode() argument
1490 struct drm_i915_private *i915 = rps_to_i915(rps); in intel_freq_opcode()
1496 return chv_freq_opcode(rps, val); in intel_freq_opcode()
1498 return byt_freq_opcode(rps, val); in intel_freq_opcode()
1505 static void vlv_init_gpll_ref_freq(struct intel_rps *rps) in vlv_init_gpll_ref_freq() argument
1507 struct drm_i915_private *i915 = rps_to_i915(rps); in vlv_init_gpll_ref_freq()
1509 rps->gpll_ref_freq = in vlv_init_gpll_ref_freq()
1515 rps->gpll_ref_freq); in vlv_init_gpll_ref_freq()
1518 static void vlv_rps_init(struct intel_rps *rps) in vlv_rps_init() argument
1520 struct drm_i915_private *i915 = rps_to_i915(rps); in vlv_rps_init()
1528 vlv_init_gpll_ref_freq(rps); in vlv_rps_init()
1545 rps->max_freq = vlv_rps_max_freq(rps); in vlv_rps_init()
1546 rps->rp0_freq = rps->max_freq; in vlv_rps_init()
1548 intel_gpu_freq(rps, rps->max_freq), rps->max_freq); in vlv_rps_init()
1550 rps->efficient_freq = vlv_rps_rpe_freq(rps); in vlv_rps_init()
1552 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq); in vlv_rps_init()
1554 rps->rp1_freq = vlv_rps_guar_freq(rps); in vlv_rps_init()
1556 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq); in vlv_rps_init()
1558 rps->min_freq = vlv_rps_min_freq(rps); in vlv_rps_init()
1560 intel_gpu_freq(rps, rps->min_freq), rps->min_freq); in vlv_rps_init()
1568 static void chv_rps_init(struct intel_rps *rps) in chv_rps_init() argument
1570 struct drm_i915_private *i915 = rps_to_i915(rps); in chv_rps_init()
1578 vlv_init_gpll_ref_freq(rps); in chv_rps_init()
1592 rps->max_freq = chv_rps_max_freq(rps); in chv_rps_init()
1593 rps->rp0_freq = rps->max_freq; in chv_rps_init()
1595 intel_gpu_freq(rps, rps->max_freq), rps->max_freq); in chv_rps_init()
1597 rps->efficient_freq = chv_rps_rpe_freq(rps); in chv_rps_init()
1599 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq); in chv_rps_init()
1601 rps->rp1_freq = chv_rps_guar_freq(rps); in chv_rps_init()
1603 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq); in chv_rps_init()
1605 rps->min_freq = chv_rps_min_freq(rps); in chv_rps_init()
1607 intel_gpu_freq(rps, rps->min_freq), rps->min_freq); in chv_rps_init()
1614 drm_WARN_ONCE(&i915->drm, (rps->max_freq | rps->efficient_freq | in chv_rps_init()
1615 rps->rp1_freq | rps->min_freq) & 1, in chv_rps_init()
1626 static u32 vlv_wa_c0_ei(struct intel_rps *rps, u32 pm_iir) in vlv_wa_c0_ei() argument
1628 struct intel_uncore *uncore = rps_to_uncore(rps); in vlv_wa_c0_ei()
1629 const struct intel_rps_ei *prev = &rps->ei; in vlv_wa_c0_ei()
1644 time *= rps_to_i915(rps)->czclk_freq; in vlv_wa_c0_ei()
1656 if (c0 > time * rps->power.up_threshold) in vlv_wa_c0_ei()
1658 else if (c0 < time * rps->power.down_threshold) in vlv_wa_c0_ei()
1662 rps->ei = now; in vlv_wa_c0_ei()
1668 struct intel_rps *rps = container_of(work, typeof(*rps), work); in rps_work() local
1669 struct intel_gt *gt = rps_to_gt(rps); in rps_work()
1670 struct drm_i915_private *i915 = rps_to_i915(rps); in rps_work()
1676 pm_iir = fetch_and_zero(&rps->pm_iir) & rps->pm_events; in rps_work()
1677 client_boost = atomic_read(&rps->num_waiters); in rps_work()
1684 mutex_lock(&rps->lock); in rps_work()
1685 if (!intel_rps_is_active(rps)) { in rps_work()
1686 mutex_unlock(&rps->lock); in rps_work()
1690 pm_iir |= vlv_wa_c0_ei(rps, pm_iir); in rps_work()
1692 adj = rps->last_adj; in rps_work()
1693 new_freq = rps->cur_freq; in rps_work()
1694 min = rps->min_freq_softlimit; in rps_work()
1695 max = rps->max_freq_softlimit; in rps_work()
1697 max = rps->max_freq; in rps_work()
1704 if (client_boost && new_freq < rps->boost_freq) { in rps_work()
1705 new_freq = rps->boost_freq; in rps_work()
1713 if (new_freq >= rps->max_freq_softlimit) in rps_work()
1718 if (rps->cur_freq > rps->efficient_freq) in rps_work()
1719 new_freq = rps->efficient_freq; in rps_work()
1720 else if (rps->cur_freq > rps->min_freq_softlimit) in rps_work()
1721 new_freq = rps->min_freq_softlimit; in rps_work()
1729 if (new_freq <= rps->min_freq_softlimit) in rps_work()
1742 if (intel_rps_set(rps, new_freq)) { in rps_work()
1746 rps->last_adj = adj; in rps_work()
1748 mutex_unlock(&rps->lock); in rps_work()
1752 gen6_gt_pm_unmask_irq(gt, rps->pm_events); in rps_work()
1756 void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir) in gen11_rps_irq_handler() argument
1758 struct intel_gt *gt = rps_to_gt(rps); in gen11_rps_irq_handler()
1759 const u32 events = rps->pm_events & pm_iir; in gen11_rps_irq_handler()
1770 rps->pm_iir |= events; in gen11_rps_irq_handler()
1771 schedule_work(&rps->work); in gen11_rps_irq_handler()
1774 void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir) in gen6_rps_irq_handler() argument
1776 struct intel_gt *gt = rps_to_gt(rps); in gen6_rps_irq_handler()
1779 events = pm_iir & rps->pm_events; in gen6_rps_irq_handler()
1786 rps->pm_iir |= events; in gen6_rps_irq_handler()
1788 schedule_work(&rps->work); in gen6_rps_irq_handler()
1802 void gen5_rps_irq_handler(struct intel_rps *rps) in gen5_rps_irq_handler() argument
1804 struct intel_uncore *uncore = rps_to_uncore(rps); in gen5_rps_irq_handler()
1821 new_freq = rps->cur_freq; in gen5_rps_irq_handler()
1827 rps->min_freq_softlimit, in gen5_rps_irq_handler()
1828 rps->max_freq_softlimit); in gen5_rps_irq_handler()
1830 if (new_freq != rps->cur_freq && !__gen5_rps_set(rps, new_freq)) in gen5_rps_irq_handler()
1831 rps->cur_freq = new_freq; in gen5_rps_irq_handler()
1836 void intel_rps_init_early(struct intel_rps *rps) in intel_rps_init_early() argument
1838 mutex_init(&rps->lock); in intel_rps_init_early()
1839 mutex_init(&rps->power.mutex); in intel_rps_init_early()
1841 INIT_WORK(&rps->work, rps_work); in intel_rps_init_early()
1842 timer_setup(&rps->timer, rps_timer, 0); in intel_rps_init_early()
1844 atomic_set(&rps->num_waiters, 0); in intel_rps_init_early()
1847 void intel_rps_init(struct intel_rps *rps) in intel_rps_init() argument
1849 struct drm_i915_private *i915 = rps_to_i915(rps); in intel_rps_init()
1851 if (rps_uses_slpc(rps)) in intel_rps_init()
1855 chv_rps_init(rps); in intel_rps_init()
1857 vlv_rps_init(rps); in intel_rps_init()
1859 gen6_rps_init(rps); in intel_rps_init()
1861 gen5_rps_init(rps); in intel_rps_init()
1864 rps->max_freq_softlimit = rps->max_freq; in intel_rps_init()
1865 rps->min_freq_softlimit = rps->min_freq; in intel_rps_init()
1876 (rps->max_freq & 0xff) * 50, in intel_rps_init()
1878 rps->max_freq = params & 0xff; in intel_rps_init()
1883 rps->boost_freq = rps->max_freq; in intel_rps_init()
1884 rps->idle_freq = rps->min_freq; in intel_rps_init()
1887 rps->cur_freq = rps->efficient_freq; in intel_rps_init()
1889 rps->pm_intrmsk_mbz = 0; in intel_rps_init()
1898 rps->pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED; in intel_rps_init()
1901 rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC; in intel_rps_init()
1904 if (intel_uc_uses_guc_submission(&rps_to_gt(rps)->uc)) in intel_rps_init()
1905 rps->pm_intrmsk_mbz |= ARAT_EXPIRED_INTRMSK; in intel_rps_init()
1908 void intel_rps_sanitize(struct intel_rps *rps) in intel_rps_sanitize() argument
1910 if (rps_uses_slpc(rps)) in intel_rps_sanitize()
1913 if (GRAPHICS_VER(rps_to_i915(rps)) >= 6) in intel_rps_sanitize()
1914 rps_disable_interrupts(rps); in intel_rps_sanitize()
1917 u32 intel_rps_get_cagf(struct intel_rps *rps, u32 rpstat) in intel_rps_get_cagf() argument
1919 struct drm_i915_private *i915 = rps_to_i915(rps); in intel_rps_get_cagf()
1931 cagf = gen5_invert_freq(rps, (rpstat & MEMSTAT_PSTATE_MASK) >> in intel_rps_get_cagf()
1937 static u32 read_cagf(struct intel_rps *rps) in read_cagf() argument
1939 struct drm_i915_private *i915 = rps_to_i915(rps); in read_cagf()
1940 struct intel_uncore *uncore = rps_to_uncore(rps); in read_cagf()
1953 return intel_rps_get_cagf(rps, freq); in read_cagf()
1956 u32 intel_rps_read_actual_frequency(struct intel_rps *rps) in intel_rps_read_actual_frequency() argument
1958 struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm; in intel_rps_read_actual_frequency()
1963 freq = intel_gpu_freq(rps, read_cagf(rps)); in intel_rps_read_actual_frequency()
1968 u32 intel_rps_read_punit_req(struct intel_rps *rps) in intel_rps_read_punit_req() argument
1970 struct intel_uncore *uncore = rps_to_uncore(rps); in intel_rps_read_punit_req()
1971 struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm; in intel_rps_read_punit_req()
1988 u32 intel_rps_read_punit_req_frequency(struct intel_rps *rps) in intel_rps_read_punit_req_frequency() argument
1990 u32 freq = intel_rps_get_req(intel_rps_read_punit_req(rps)); in intel_rps_read_punit_req_frequency()
1992 return intel_gpu_freq(rps, freq); in intel_rps_read_punit_req_frequency()
1995 u32 intel_rps_get_requested_frequency(struct intel_rps *rps) in intel_rps_get_requested_frequency() argument
1997 if (rps_uses_slpc(rps)) in intel_rps_get_requested_frequency()
1998 return intel_rps_read_punit_req_frequency(rps); in intel_rps_get_requested_frequency()
2000 return intel_gpu_freq(rps, rps->cur_freq); in intel_rps_get_requested_frequency()
2003 u32 intel_rps_get_max_frequency(struct intel_rps *rps) in intel_rps_get_max_frequency() argument
2005 struct intel_guc_slpc *slpc = rps_to_slpc(rps); in intel_rps_get_max_frequency()
2007 if (rps_uses_slpc(rps)) in intel_rps_get_max_frequency()
2010 return intel_gpu_freq(rps, rps->max_freq_softlimit); in intel_rps_get_max_frequency()
2013 u32 intel_rps_get_rp0_frequency(struct intel_rps *rps) in intel_rps_get_rp0_frequency() argument
2015 struct intel_guc_slpc *slpc = rps_to_slpc(rps); in intel_rps_get_rp0_frequency()
2017 if (rps_uses_slpc(rps)) in intel_rps_get_rp0_frequency()
2020 return intel_gpu_freq(rps, rps->rp0_freq); in intel_rps_get_rp0_frequency()
2023 u32 intel_rps_get_rp1_frequency(struct intel_rps *rps) in intel_rps_get_rp1_frequency() argument
2025 struct intel_guc_slpc *slpc = rps_to_slpc(rps); in intel_rps_get_rp1_frequency()
2027 if (rps_uses_slpc(rps)) in intel_rps_get_rp1_frequency()
2030 return intel_gpu_freq(rps, rps->rp1_freq); in intel_rps_get_rp1_frequency()
2033 u32 intel_rps_get_rpn_frequency(struct intel_rps *rps) in intel_rps_get_rpn_frequency() argument
2035 struct intel_guc_slpc *slpc = rps_to_slpc(rps); in intel_rps_get_rpn_frequency()
2037 if (rps_uses_slpc(rps)) in intel_rps_get_rpn_frequency()
2040 return intel_gpu_freq(rps, rps->min_freq); in intel_rps_get_rpn_frequency()
2043 static int set_max_freq(struct intel_rps *rps, u32 val) in set_max_freq() argument
2045 struct drm_i915_private *i915 = rps_to_i915(rps); in set_max_freq()
2048 mutex_lock(&rps->lock); in set_max_freq()
2050 val = intel_freq_opcode(rps, val); in set_max_freq()
2051 if (val < rps->min_freq || in set_max_freq()
2052 val > rps->max_freq || in set_max_freq()
2053 val < rps->min_freq_softlimit) { in set_max_freq()
2058 if (val > rps->rp0_freq) in set_max_freq()
2060 intel_gpu_freq(rps, val)); in set_max_freq()
2062 rps->max_freq_softlimit = val; in set_max_freq()
2064 val = clamp_t(int, rps->cur_freq, in set_max_freq()
2065 rps->min_freq_softlimit, in set_max_freq()
2066 rps->max_freq_softlimit); in set_max_freq()
2073 intel_rps_set(rps, val); in set_max_freq()
2076 mutex_unlock(&rps->lock); in set_max_freq()
2081 int intel_rps_set_max_frequency(struct intel_rps *rps, u32 val) in intel_rps_set_max_frequency() argument
2083 struct intel_guc_slpc *slpc = rps_to_slpc(rps); in intel_rps_set_max_frequency()
2085 if (rps_uses_slpc(rps)) in intel_rps_set_max_frequency()
2088 return set_max_freq(rps, val); in intel_rps_set_max_frequency()
2091 u32 intel_rps_get_min_frequency(struct intel_rps *rps) in intel_rps_get_min_frequency() argument
2093 struct intel_guc_slpc *slpc = rps_to_slpc(rps); in intel_rps_get_min_frequency()
2095 if (rps_uses_slpc(rps)) in intel_rps_get_min_frequency()
2098 return intel_gpu_freq(rps, rps->min_freq_softlimit); in intel_rps_get_min_frequency()
2101 static int set_min_freq(struct intel_rps *rps, u32 val) in set_min_freq() argument
2105 mutex_lock(&rps->lock); in set_min_freq()
2107 val = intel_freq_opcode(rps, val); in set_min_freq()
2108 if (val < rps->min_freq || in set_min_freq()
2109 val > rps->max_freq || in set_min_freq()
2110 val > rps->max_freq_softlimit) { in set_min_freq()
2115 rps->min_freq_softlimit = val; in set_min_freq()
2117 val = clamp_t(int, rps->cur_freq, in set_min_freq()
2118 rps->min_freq_softlimit, in set_min_freq()
2119 rps->max_freq_softlimit); in set_min_freq()
2126 intel_rps_set(rps, val); in set_min_freq()
2129 mutex_unlock(&rps->lock); in set_min_freq()
2134 int intel_rps_set_min_frequency(struct intel_rps *rps, u32 val) in intel_rps_set_min_frequency() argument
2136 struct intel_guc_slpc *slpc = rps_to_slpc(rps); in intel_rps_set_min_frequency()
2138 if (rps_uses_slpc(rps)) in intel_rps_set_min_frequency()
2141 return set_min_freq(rps, val); in intel_rps_set_min_frequency()
2144 u32 intel_rps_read_state_cap(struct intel_rps *rps) in intel_rps_read_state_cap() argument
2146 struct drm_i915_private *i915 = rps_to_i915(rps); in intel_rps_read_state_cap()
2147 struct intel_uncore *uncore = rps_to_uncore(rps); in intel_rps_read_state_cap()
2181 void intel_rps_driver_register(struct intel_rps *rps) in intel_rps_driver_register() argument
2183 struct intel_gt *gt = rps_to_gt(rps); in intel_rps_driver_register()
2196 void intel_rps_driver_unregister(struct intel_rps *rps) in intel_rps_driver_unregister() argument
2198 if (rcu_access_pointer(ips_mchdev) == rps_to_i915(rps)) in intel_rps_driver_unregister()
2233 struct intel_ips *ips = &i915->gt.rps.ips; in i915_read_mch_val()
2254 struct intel_rps *rps; in i915_gpu_raise() local
2260 rps = &i915->gt.rps; in i915_gpu_raise()
2263 if (rps->max_freq_softlimit < rps->max_freq) in i915_gpu_raise()
2264 rps->max_freq_softlimit++; in i915_gpu_raise()
2281 struct intel_rps *rps; in i915_gpu_lower() local
2287 rps = &i915->gt.rps; in i915_gpu_lower()
2290 if (rps->max_freq_softlimit > rps->min_freq) in i915_gpu_lower()
2291 rps->max_freq_softlimit--; in i915_gpu_lower()
2329 struct intel_rps *rps; in i915_gpu_turbo_disable() local
2336 rps = &i915->gt.rps; in i915_gpu_turbo_disable()
2339 rps->max_freq_softlimit = rps->min_freq; in i915_gpu_turbo_disable()
2340 ret = !__gen5_rps_set(&i915->gt.rps, rps->min_freq); in i915_gpu_turbo_disable()