1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2019 Intel Corporation
4 */
5
6 #include <linux/suspend.h>
7
8 #include "i915_drv.h"
9 #include "i915_params.h"
10 #include "intel_context.h"
11 #include "intel_engine_pm.h"
12 #include "intel_gt.h"
13 #include "intel_gt_clock_utils.h"
14 #include "intel_gt_pm.h"
15 #include "intel_gt_requests.h"
16 #include "intel_llc.h"
17 #include "intel_pm.h"
18 #include "intel_rc6.h"
19 #include "intel_rps.h"
20 #include "intel_wakeref.h"
21 #include "pxp/intel_pxp_pm.h"
22
23 #define I915_GT_SUSPEND_IDLE_TIMEOUT (HZ / 2)
24
user_forcewake(struct intel_gt * gt,bool suspend)25 static void user_forcewake(struct intel_gt *gt, bool suspend)
26 {
27 int count = atomic_read(>->user_wakeref);
28
29 /* Inside suspend/resume so single threaded, no races to worry about. */
30 if (likely(!count))
31 return;
32
33 intel_gt_pm_get(gt);
34 if (suspend) {
35 GEM_BUG_ON(count > atomic_read(>->wakeref.count));
36 atomic_sub(count, >->wakeref.count);
37 } else {
38 atomic_add(count, >->wakeref.count);
39 }
40 intel_gt_pm_put(gt);
41 }
42
runtime_begin(struct intel_gt * gt)43 static void runtime_begin(struct intel_gt *gt)
44 {
45 local_irq_disable();
46 write_seqcount_begin(>->stats.lock);
47 gt->stats.start = ktime_get();
48 gt->stats.active = true;
49 write_seqcount_end(>->stats.lock);
50 local_irq_enable();
51 }
52
runtime_end(struct intel_gt * gt)53 static void runtime_end(struct intel_gt *gt)
54 {
55 local_irq_disable();
56 write_seqcount_begin(>->stats.lock);
57 gt->stats.active = false;
58 gt->stats.total =
59 ktime_add(gt->stats.total,
60 ktime_sub(ktime_get(), gt->stats.start));
61 write_seqcount_end(>->stats.lock);
62 local_irq_enable();
63 }
64
__gt_unpark(struct intel_wakeref * wf)65 static int __gt_unpark(struct intel_wakeref *wf)
66 {
67 struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref);
68 struct drm_i915_private *i915 = gt->i915;
69
70 GT_TRACE(gt, "\n");
71
72 /*
73 * It seems that the DMC likes to transition between the DC states a lot
74 * when there are no connected displays (no active power domains) during
75 * command submission.
76 *
77 * This activity has negative impact on the performance of the chip with
78 * huge latencies observed in the interrupt handler and elsewhere.
79 *
80 * Work around it by grabbing a GT IRQ power domain whilst there is any
81 * GT activity, preventing any DC state transitions.
82 */
83 gt->awake = intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ);
84 GEM_BUG_ON(!gt->awake);
85
86 intel_rc6_unpark(>->rc6);
87 intel_rps_unpark(>->rps);
88 i915_pmu_gt_unparked(i915);
89
90 intel_gt_unpark_requests(gt);
91 runtime_begin(gt);
92
93 return 0;
94 }
95
__gt_park(struct intel_wakeref * wf)96 static int __gt_park(struct intel_wakeref *wf)
97 {
98 struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref);
99 intel_wakeref_t wakeref = fetch_and_zero(>->awake);
100 struct drm_i915_private *i915 = gt->i915;
101
102 GT_TRACE(gt, "\n");
103
104 runtime_end(gt);
105 intel_gt_park_requests(gt);
106
107 i915_vma_parked(gt);
108 i915_pmu_gt_parked(i915);
109 intel_rps_park(>->rps);
110 intel_rc6_park(>->rc6);
111
112 /* Everything switched off, flush any residual interrupt just in case */
113 intel_synchronize_irq(i915);
114
115 /* Defer dropping the display power well for 100ms, it's slow! */
116 GEM_BUG_ON(!wakeref);
117 intel_display_power_put_async(i915, POWER_DOMAIN_GT_IRQ, wakeref);
118
119 return 0;
120 }
121
122 static const struct intel_wakeref_ops wf_ops = {
123 .get = __gt_unpark,
124 .put = __gt_park,
125 };
126
intel_gt_pm_init_early(struct intel_gt * gt)127 void intel_gt_pm_init_early(struct intel_gt *gt)
128 {
129 intel_wakeref_init(>->wakeref, gt->uncore->rpm, &wf_ops);
130 seqcount_mutex_init(>->stats.lock, >->wakeref.mutex);
131 }
132
intel_gt_pm_init(struct intel_gt * gt)133 void intel_gt_pm_init(struct intel_gt *gt)
134 {
135 /*
136 * Enabling power-management should be "self-healing". If we cannot
137 * enable a feature, simply leave it disabled with a notice to the
138 * user.
139 */
140 intel_rc6_init(>->rc6);
141 intel_rps_init(>->rps);
142 }
143
reset_engines(struct intel_gt * gt)144 static bool reset_engines(struct intel_gt *gt)
145 {
146 if (INTEL_INFO(gt->i915)->gpu_reset_clobbers_display)
147 return false;
148
149 return __intel_gt_reset(gt, ALL_ENGINES) == 0;
150 }
151
gt_sanitize(struct intel_gt * gt,bool force)152 static void gt_sanitize(struct intel_gt *gt, bool force)
153 {
154 struct intel_engine_cs *engine;
155 enum intel_engine_id id;
156 intel_wakeref_t wakeref;
157
158 GT_TRACE(gt, "force:%s", yesno(force));
159
160 /* Use a raw wakeref to avoid calling intel_display_power_get early */
161 wakeref = intel_runtime_pm_get(gt->uncore->rpm);
162 intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
163
164 intel_gt_check_clock_frequency(gt);
165
166 /*
167 * As we have just resumed the machine and woken the device up from
168 * deep PCI sleep (presumably D3_cold), assume the HW has been reset
169 * back to defaults, recovering from whatever wedged state we left it
170 * in and so worth trying to use the device once more.
171 */
172 if (intel_gt_is_wedged(gt))
173 intel_gt_unset_wedged(gt);
174
175 for_each_engine(engine, gt, id)
176 if (engine->reset.prepare)
177 engine->reset.prepare(engine);
178
179 intel_uc_reset_prepare(>->uc);
180
181 for_each_engine(engine, gt, id)
182 if (engine->sanitize)
183 engine->sanitize(engine);
184
185 if (reset_engines(gt) || force) {
186 for_each_engine(engine, gt, id)
187 __intel_engine_reset(engine, false);
188 }
189
190 intel_uc_reset(>->uc, false);
191
192 for_each_engine(engine, gt, id)
193 if (engine->reset.finish)
194 engine->reset.finish(engine);
195
196 intel_rps_sanitize(>->rps);
197
198 intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
199 intel_runtime_pm_put(gt->uncore->rpm, wakeref);
200 }
201
intel_gt_pm_fini(struct intel_gt * gt)202 void intel_gt_pm_fini(struct intel_gt *gt)
203 {
204 intel_rc6_fini(>->rc6);
205 }
206
intel_gt_resume(struct intel_gt * gt)207 int intel_gt_resume(struct intel_gt *gt)
208 {
209 struct intel_engine_cs *engine;
210 enum intel_engine_id id;
211 int err;
212
213 err = intel_gt_has_unrecoverable_error(gt);
214 if (err)
215 return err;
216
217 GT_TRACE(gt, "\n");
218
219 /*
220 * After resume, we may need to poke into the pinned kernel
221 * contexts to paper over any damage caused by the sudden suspend.
222 * Only the kernel contexts should remain pinned over suspend,
223 * allowing us to fixup the user contexts on their first pin.
224 */
225 gt_sanitize(gt, true);
226
227 intel_gt_pm_get(gt);
228
229 intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
230 intel_rc6_sanitize(>->rc6);
231 if (intel_gt_is_wedged(gt)) {
232 err = -EIO;
233 goto out_fw;
234 }
235
236 /* Only when the HW is re-initialised, can we replay the requests */
237 err = intel_gt_init_hw(gt);
238 if (err) {
239 i915_probe_error(gt->i915,
240 "Failed to initialize GPU, declaring it wedged!\n");
241 goto err_wedged;
242 }
243
244 intel_uc_reset_finish(>->uc);
245
246 intel_rps_enable(>->rps);
247 intel_llc_enable(>->llc);
248
249 for_each_engine(engine, gt, id) {
250 intel_engine_pm_get(engine);
251
252 engine->serial++; /* kernel context lost */
253 err = intel_engine_resume(engine);
254
255 intel_engine_pm_put(engine);
256 if (err) {
257 drm_err(>->i915->drm,
258 "Failed to restart %s (%d)\n",
259 engine->name, err);
260 goto err_wedged;
261 }
262 }
263
264 intel_rc6_enable(>->rc6);
265
266 intel_uc_resume(>->uc);
267
268 intel_pxp_resume(>->pxp);
269
270 user_forcewake(gt, false);
271
272 out_fw:
273 intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
274 intel_gt_pm_put(gt);
275 return err;
276
277 err_wedged:
278 intel_gt_set_wedged(gt);
279 goto out_fw;
280 }
281
wait_for_suspend(struct intel_gt * gt)282 static void wait_for_suspend(struct intel_gt *gt)
283 {
284 if (!intel_gt_pm_is_awake(gt))
285 return;
286
287 if (intel_gt_wait_for_idle(gt, I915_GT_SUSPEND_IDLE_TIMEOUT) == -ETIME) {
288 /*
289 * Forcibly cancel outstanding work and leave
290 * the gpu quiet.
291 */
292 intel_gt_set_wedged(gt);
293 intel_gt_retire_requests(gt);
294 }
295
296 intel_gt_pm_wait_for_idle(gt);
297 }
298
intel_gt_suspend_prepare(struct intel_gt * gt)299 void intel_gt_suspend_prepare(struct intel_gt *gt)
300 {
301 user_forcewake(gt, true);
302 wait_for_suspend(gt);
303
304 intel_pxp_suspend_prepare(>->pxp);
305 }
306
pm_suspend_target(void)307 static suspend_state_t pm_suspend_target(void)
308 {
309 #if IS_ENABLED(CONFIG_SUSPEND) && IS_ENABLED(CONFIG_PM_SLEEP)
310 return pm_suspend_target_state;
311 #else
312 return PM_SUSPEND_TO_IDLE;
313 #endif
314 }
315
intel_gt_suspend_late(struct intel_gt * gt)316 void intel_gt_suspend_late(struct intel_gt *gt)
317 {
318 intel_wakeref_t wakeref;
319
320 /* We expect to be idle already; but also want to be independent */
321 wait_for_suspend(gt);
322
323 if (is_mock_gt(gt))
324 return;
325
326 GEM_BUG_ON(gt->awake);
327
328 intel_uc_suspend(>->uc);
329 intel_pxp_suspend(>->pxp);
330
331 /*
332 * On disabling the device, we want to turn off HW access to memory
333 * that we no longer own.
334 *
335 * However, not all suspend-states disable the device. S0 (s2idle)
336 * is effectively runtime-suspend, the device is left powered on
337 * but needs to be put into a low power state. We need to keep
338 * powermanagement enabled, but we also retain system state and so
339 * it remains safe to keep on using our allocated memory.
340 */
341 if (pm_suspend_target() == PM_SUSPEND_TO_IDLE)
342 return;
343
344 with_intel_runtime_pm(gt->uncore->rpm, wakeref) {
345 intel_rps_disable(>->rps);
346 intel_rc6_disable(>->rc6);
347 intel_llc_disable(>->llc);
348 }
349
350 gt_sanitize(gt, false);
351
352 GT_TRACE(gt, "\n");
353 }
354
intel_gt_runtime_suspend(struct intel_gt * gt)355 void intel_gt_runtime_suspend(struct intel_gt *gt)
356 {
357 intel_pxp_runtime_suspend(>->pxp);
358 intel_uc_runtime_suspend(>->uc);
359
360 GT_TRACE(gt, "\n");
361 }
362
intel_gt_runtime_resume(struct intel_gt * gt)363 int intel_gt_runtime_resume(struct intel_gt *gt)
364 {
365 int ret;
366
367 GT_TRACE(gt, "\n");
368 intel_gt_init_swizzling(gt);
369 intel_ggtt_restore_fences(gt->ggtt);
370
371 ret = intel_uc_runtime_resume(>->uc);
372 if (ret)
373 return ret;
374
375 intel_pxp_runtime_resume(>->pxp);
376
377 return 0;
378 }
379
__intel_gt_get_awake_time(const struct intel_gt * gt)380 static ktime_t __intel_gt_get_awake_time(const struct intel_gt *gt)
381 {
382 ktime_t total = gt->stats.total;
383
384 if (gt->stats.active)
385 total = ktime_add(total,
386 ktime_sub(ktime_get(), gt->stats.start));
387
388 return total;
389 }
390
intel_gt_get_awake_time(const struct intel_gt * gt)391 ktime_t intel_gt_get_awake_time(const struct intel_gt *gt)
392 {
393 unsigned int seq;
394 ktime_t total;
395
396 do {
397 seq = read_seqcount_begin(>->stats.lock);
398 total = __intel_gt_get_awake_time(gt);
399 } while (read_seqcount_retry(>->stats.lock, seq));
400
401 return total;
402 }
403
404 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
405 #include "selftest_gt_pm.c"
406 #endif
407