1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2016-2018, The Linux Foundation. All rights reserved.
3  */
4 
5 #define pr_fmt(fmt)	"[drm:%s:%d] " fmt, __func__, __LINE__
6 
7 #include <linux/debugfs.h>
8 #include <linux/errno.h>
9 #include <linux/mutex.h>
10 #include <linux/pm_opp.h>
11 #include <linux/sort.h>
12 #include <linux/clk.h>
13 #include <linux/bitmap.h>
14 
15 #include "dpu_kms.h"
16 #include "dpu_trace.h"
17 #include "dpu_crtc.h"
18 #include "dpu_core_perf.h"
19 
20 /**
21  * enum dpu_perf_mode - performance tuning mode
22  * @DPU_PERF_MODE_NORMAL: performance controlled by user mode client
23  * @DPU_PERF_MODE_MINIMUM: performance bounded by minimum setting
24  * @DPU_PERF_MODE_FIXED: performance bounded by fixed setting
25  * @DPU_PERF_MODE_MAX: maximum value, used for error checking
26  */
27 enum dpu_perf_mode {
28 	DPU_PERF_MODE_NORMAL,
29 	DPU_PERF_MODE_MINIMUM,
30 	DPU_PERF_MODE_FIXED,
31 	DPU_PERF_MODE_MAX
32 };
33 
34 /**
35  * _dpu_core_perf_calc_bw() - to calculate BW per crtc
36  * @kms:  pointer to the dpu_kms
37  * @crtc: pointer to a crtc
38  * Return: returns aggregated BW for all planes in crtc.
39  */
_dpu_core_perf_calc_bw(struct dpu_kms * kms,struct drm_crtc * crtc)40 static u64 _dpu_core_perf_calc_bw(struct dpu_kms *kms,
41 		struct drm_crtc *crtc)
42 {
43 	struct drm_plane *plane;
44 	struct dpu_plane_state *pstate;
45 	u64 crtc_plane_bw = 0;
46 	u32 bw_factor;
47 
48 	drm_atomic_crtc_for_each_plane(plane, crtc) {
49 		pstate = to_dpu_plane_state(plane->state);
50 		if (!pstate)
51 			continue;
52 
53 		crtc_plane_bw += pstate->plane_fetch_bw;
54 	}
55 
56 	bw_factor = kms->catalog->perf.bw_inefficiency_factor;
57 	if (bw_factor) {
58 		crtc_plane_bw *= bw_factor;
59 		do_div(crtc_plane_bw, 100);
60 	}
61 
62 	return crtc_plane_bw;
63 }
64 
65 /**
66  * _dpu_core_perf_calc_clk() - to calculate clock per crtc
67  * @kms:  pointer to the dpu_kms
68  * @crtc: pointer to a crtc
69  * @state: pointer to a crtc state
70  * Return: returns max clk for all planes in crtc.
71  */
_dpu_core_perf_calc_clk(struct dpu_kms * kms,struct drm_crtc * crtc,struct drm_crtc_state * state)72 static u64 _dpu_core_perf_calc_clk(struct dpu_kms *kms,
73 		struct drm_crtc *crtc, struct drm_crtc_state *state)
74 {
75 	struct drm_plane *plane;
76 	struct dpu_plane_state *pstate;
77 	struct drm_display_mode *mode;
78 	u64 crtc_clk;
79 	u32 clk_factor;
80 
81 	mode = &state->adjusted_mode;
82 
83 	crtc_clk = mode->vtotal * mode->hdisplay * drm_mode_vrefresh(mode);
84 
85 	drm_atomic_crtc_for_each_plane(plane, crtc) {
86 		pstate = to_dpu_plane_state(plane->state);
87 		if (!pstate)
88 			continue;
89 
90 		crtc_clk = max(pstate->plane_clk, crtc_clk);
91 	}
92 
93 	clk_factor = kms->catalog->perf.clk_inefficiency_factor;
94 	if (clk_factor) {
95 		crtc_clk *= clk_factor;
96 		do_div(crtc_clk, 100);
97 	}
98 
99 	return crtc_clk;
100 }
101 
_dpu_crtc_get_kms(struct drm_crtc * crtc)102 static struct dpu_kms *_dpu_crtc_get_kms(struct drm_crtc *crtc)
103 {
104 	struct msm_drm_private *priv;
105 	priv = crtc->dev->dev_private;
106 	return to_dpu_kms(priv->kms);
107 }
108 
_dpu_core_perf_calc_crtc(struct dpu_kms * kms,struct drm_crtc * crtc,struct drm_crtc_state * state,struct dpu_core_perf_params * perf)109 static void _dpu_core_perf_calc_crtc(struct dpu_kms *kms,
110 		struct drm_crtc *crtc,
111 		struct drm_crtc_state *state,
112 		struct dpu_core_perf_params *perf)
113 {
114 	if (!kms || !kms->catalog || !crtc || !state || !perf) {
115 		DPU_ERROR("invalid parameters\n");
116 		return;
117 	}
118 
119 	memset(perf, 0, sizeof(struct dpu_core_perf_params));
120 
121 	if (kms->perf.perf_tune.mode == DPU_PERF_MODE_MINIMUM) {
122 		perf->bw_ctl = 0;
123 		perf->max_per_pipe_ib = 0;
124 		perf->core_clk_rate = 0;
125 	} else if (kms->perf.perf_tune.mode == DPU_PERF_MODE_FIXED) {
126 		perf->bw_ctl = kms->perf.fix_core_ab_vote;
127 		perf->max_per_pipe_ib = kms->perf.fix_core_ib_vote;
128 		perf->core_clk_rate = kms->perf.fix_core_clk_rate;
129 	} else {
130 		perf->bw_ctl = _dpu_core_perf_calc_bw(kms, crtc);
131 		perf->max_per_pipe_ib = kms->catalog->perf.min_dram_ib;
132 		perf->core_clk_rate = _dpu_core_perf_calc_clk(kms, crtc, state);
133 	}
134 
135 	DRM_DEBUG_ATOMIC(
136 		"crtc=%d clk_rate=%llu core_ib=%llu core_ab=%llu\n",
137 			crtc->base.id, perf->core_clk_rate,
138 			perf->max_per_pipe_ib, perf->bw_ctl);
139 }
140 
dpu_core_perf_crtc_check(struct drm_crtc * crtc,struct drm_crtc_state * state)141 int dpu_core_perf_crtc_check(struct drm_crtc *crtc,
142 		struct drm_crtc_state *state)
143 {
144 	u32 bw, threshold;
145 	u64 bw_sum_of_intfs = 0;
146 	enum dpu_crtc_client_type curr_client_type;
147 	struct dpu_crtc_state *dpu_cstate;
148 	struct drm_crtc *tmp_crtc;
149 	struct dpu_kms *kms;
150 
151 	if (!crtc || !state) {
152 		DPU_ERROR("invalid crtc\n");
153 		return -EINVAL;
154 	}
155 
156 	kms = _dpu_crtc_get_kms(crtc);
157 	if (!kms->catalog) {
158 		DPU_ERROR("invalid parameters\n");
159 		return 0;
160 	}
161 
162 	/* we only need bandwidth check on real-time clients (interfaces) */
163 	if (dpu_crtc_get_client_type(crtc) == NRT_CLIENT)
164 		return 0;
165 
166 	dpu_cstate = to_dpu_crtc_state(state);
167 
168 	/* obtain new values */
169 	_dpu_core_perf_calc_crtc(kms, crtc, state, &dpu_cstate->new_perf);
170 
171 	bw_sum_of_intfs = dpu_cstate->new_perf.bw_ctl;
172 	curr_client_type = dpu_crtc_get_client_type(crtc);
173 
174 	drm_for_each_crtc(tmp_crtc, crtc->dev) {
175 		if (tmp_crtc->enabled &&
176 		    (dpu_crtc_get_client_type(tmp_crtc) ==
177 				curr_client_type) && (tmp_crtc != crtc)) {
178 			struct dpu_crtc_state *tmp_cstate =
179 				to_dpu_crtc_state(tmp_crtc->state);
180 
181 			DRM_DEBUG_ATOMIC("crtc:%d bw:%llu ctrl:%d\n",
182 				tmp_crtc->base.id, tmp_cstate->new_perf.bw_ctl,
183 				tmp_cstate->bw_control);
184 
185 				bw_sum_of_intfs += tmp_cstate->new_perf.bw_ctl;
186 		}
187 
188 		/* convert bandwidth to kb */
189 		bw = DIV_ROUND_UP_ULL(bw_sum_of_intfs, 1000);
190 		DRM_DEBUG_ATOMIC("calculated bandwidth=%uk\n", bw);
191 
192 		threshold = kms->catalog->perf.max_bw_high;
193 
194 		DRM_DEBUG_ATOMIC("final threshold bw limit = %d\n", threshold);
195 
196 		if (!threshold) {
197 			DPU_ERROR("no bandwidth limits specified\n");
198 			return -E2BIG;
199 		} else if (bw > threshold) {
200 			DPU_ERROR("exceeds bandwidth: %ukb > %ukb\n", bw,
201 					threshold);
202 			return -E2BIG;
203 		}
204 	}
205 
206 	return 0;
207 }
208 
_dpu_core_perf_crtc_update_bus(struct dpu_kms * kms,struct drm_crtc * crtc)209 static int _dpu_core_perf_crtc_update_bus(struct dpu_kms *kms,
210 		struct drm_crtc *crtc)
211 {
212 	struct dpu_core_perf_params perf = { 0 };
213 	enum dpu_crtc_client_type curr_client_type
214 					= dpu_crtc_get_client_type(crtc);
215 	struct drm_crtc *tmp_crtc;
216 	struct dpu_crtc_state *dpu_cstate;
217 	int i, ret = 0;
218 	u64 avg_bw;
219 
220 	drm_for_each_crtc(tmp_crtc, crtc->dev) {
221 		if (tmp_crtc->enabled &&
222 			curr_client_type ==
223 				dpu_crtc_get_client_type(tmp_crtc)) {
224 			dpu_cstate = to_dpu_crtc_state(tmp_crtc->state);
225 
226 			perf.max_per_pipe_ib = max(perf.max_per_pipe_ib,
227 					dpu_cstate->new_perf.max_per_pipe_ib);
228 
229 			perf.bw_ctl += dpu_cstate->new_perf.bw_ctl;
230 
231 			DRM_DEBUG_ATOMIC("crtc=%d bw=%llu paths:%d\n",
232 				  tmp_crtc->base.id,
233 				  dpu_cstate->new_perf.bw_ctl, kms->num_paths);
234 		}
235 	}
236 
237 	if (!kms->num_paths)
238 		return 0;
239 
240 	avg_bw = perf.bw_ctl;
241 	do_div(avg_bw, (kms->num_paths * 1000)); /*Bps_to_icc*/
242 
243 	for (i = 0; i < kms->num_paths; i++)
244 		icc_set_bw(kms->path[i], avg_bw, perf.max_per_pipe_ib);
245 
246 	return ret;
247 }
248 
249 /**
250  * dpu_core_perf_crtc_release_bw() - request zero bandwidth
251  * @crtc: pointer to a crtc
252  *
253  * Function checks a state variable for the crtc, if all pending commit
254  * requests are done, meaning no more bandwidth is needed, release
255  * bandwidth request.
256  */
dpu_core_perf_crtc_release_bw(struct drm_crtc * crtc)257 void dpu_core_perf_crtc_release_bw(struct drm_crtc *crtc)
258 {
259 	struct dpu_crtc *dpu_crtc;
260 	struct dpu_kms *kms;
261 
262 	if (!crtc) {
263 		DPU_ERROR("invalid crtc\n");
264 		return;
265 	}
266 
267 	kms = _dpu_crtc_get_kms(crtc);
268 	if (!kms->catalog) {
269 		DPU_ERROR("invalid kms\n");
270 		return;
271 	}
272 
273 	dpu_crtc = to_dpu_crtc(crtc);
274 
275 	if (atomic_dec_return(&kms->bandwidth_ref) > 0)
276 		return;
277 
278 	/* Release the bandwidth */
279 	if (kms->perf.enable_bw_release) {
280 		trace_dpu_cmd_release_bw(crtc->base.id);
281 		DRM_DEBUG_ATOMIC("Release BW crtc=%d\n", crtc->base.id);
282 		dpu_crtc->cur_perf.bw_ctl = 0;
283 		_dpu_core_perf_crtc_update_bus(kms, crtc);
284 	}
285 }
286 
_dpu_core_perf_set_core_clk_rate(struct dpu_kms * kms,u64 rate)287 static int _dpu_core_perf_set_core_clk_rate(struct dpu_kms *kms, u64 rate)
288 {
289 	struct dss_clk *core_clk = kms->perf.core_clk;
290 
291 	if (core_clk->max_rate && (rate > core_clk->max_rate))
292 		rate = core_clk->max_rate;
293 
294 	core_clk->rate = rate;
295 	return dev_pm_opp_set_rate(&kms->pdev->dev, core_clk->rate);
296 }
297 
_dpu_core_perf_get_core_clk_rate(struct dpu_kms * kms)298 static u64 _dpu_core_perf_get_core_clk_rate(struct dpu_kms *kms)
299 {
300 	u64 clk_rate = kms->perf.perf_tune.min_core_clk;
301 	struct drm_crtc *crtc;
302 	struct dpu_crtc_state *dpu_cstate;
303 
304 	drm_for_each_crtc(crtc, kms->dev) {
305 		if (crtc->enabled) {
306 			dpu_cstate = to_dpu_crtc_state(crtc->state);
307 			clk_rate = max(dpu_cstate->new_perf.core_clk_rate,
308 							clk_rate);
309 			clk_rate = clk_round_rate(kms->perf.core_clk->clk,
310 					clk_rate);
311 		}
312 	}
313 
314 	if (kms->perf.perf_tune.mode == DPU_PERF_MODE_FIXED)
315 		clk_rate = kms->perf.fix_core_clk_rate;
316 
317 	DRM_DEBUG_ATOMIC("clk:%llu\n", clk_rate);
318 
319 	return clk_rate;
320 }
321 
dpu_core_perf_crtc_update(struct drm_crtc * crtc,int params_changed,bool stop_req)322 int dpu_core_perf_crtc_update(struct drm_crtc *crtc,
323 		int params_changed, bool stop_req)
324 {
325 	struct dpu_core_perf_params *new, *old;
326 	bool update_bus = false, update_clk = false;
327 	u64 clk_rate = 0;
328 	struct dpu_crtc *dpu_crtc;
329 	struct dpu_crtc_state *dpu_cstate;
330 	struct dpu_kms *kms;
331 	int ret;
332 
333 	if (!crtc) {
334 		DPU_ERROR("invalid crtc\n");
335 		return -EINVAL;
336 	}
337 
338 	kms = _dpu_crtc_get_kms(crtc);
339 	if (!kms->catalog) {
340 		DPU_ERROR("invalid kms\n");
341 		return -EINVAL;
342 	}
343 
344 	dpu_crtc = to_dpu_crtc(crtc);
345 	dpu_cstate = to_dpu_crtc_state(crtc->state);
346 
347 	DRM_DEBUG_ATOMIC("crtc:%d stop_req:%d core_clk:%llu\n",
348 			crtc->base.id, stop_req, kms->perf.core_clk_rate);
349 
350 	old = &dpu_crtc->cur_perf;
351 	new = &dpu_cstate->new_perf;
352 
353 	if (crtc->enabled && !stop_req) {
354 		/*
355 		 * cases for bus bandwidth update.
356 		 * 1. new bandwidth vote - "ab or ib vote" is higher
357 		 *    than current vote for update request.
358 		 * 2. new bandwidth vote - "ab or ib vote" is lower
359 		 *    than current vote at end of commit or stop.
360 		 */
361 		if ((params_changed && ((new->bw_ctl > old->bw_ctl) ||
362 			(new->max_per_pipe_ib > old->max_per_pipe_ib)))	||
363 			(!params_changed && ((new->bw_ctl < old->bw_ctl) ||
364 			(new->max_per_pipe_ib < old->max_per_pipe_ib)))) {
365 			DRM_DEBUG_ATOMIC("crtc=%d p=%d new_bw=%llu,old_bw=%llu\n",
366 				crtc->base.id, params_changed,
367 				new->bw_ctl, old->bw_ctl);
368 			old->bw_ctl = new->bw_ctl;
369 			old->max_per_pipe_ib = new->max_per_pipe_ib;
370 			update_bus = true;
371 		}
372 
373 		if ((params_changed &&
374 			(new->core_clk_rate > old->core_clk_rate)) ||
375 			(!params_changed &&
376 			(new->core_clk_rate < old->core_clk_rate))) {
377 			old->core_clk_rate = new->core_clk_rate;
378 			update_clk = true;
379 		}
380 	} else {
381 		DRM_DEBUG_ATOMIC("crtc=%d disable\n", crtc->base.id);
382 		memset(old, 0, sizeof(*old));
383 		update_bus = true;
384 		update_clk = true;
385 	}
386 
387 	trace_dpu_perf_crtc_update(crtc->base.id, new->bw_ctl,
388 		new->core_clk_rate, stop_req, update_bus, update_clk);
389 
390 	if (update_bus) {
391 		ret = _dpu_core_perf_crtc_update_bus(kms, crtc);
392 		if (ret) {
393 			DPU_ERROR("crtc-%d: failed to update bus bw vote\n",
394 				  crtc->base.id);
395 			return ret;
396 		}
397 	}
398 
399 	/*
400 	 * Update the clock after bandwidth vote to ensure
401 	 * bandwidth is available before clock rate is increased.
402 	 */
403 	if (update_clk) {
404 		clk_rate = _dpu_core_perf_get_core_clk_rate(kms);
405 
406 		trace_dpu_core_perf_update_clk(kms->dev, stop_req, clk_rate);
407 
408 		ret = _dpu_core_perf_set_core_clk_rate(kms, clk_rate);
409 		if (ret) {
410 			DPU_ERROR("failed to set %s clock rate %llu\n",
411 					kms->perf.core_clk->clk_name, clk_rate);
412 			return ret;
413 		}
414 
415 		kms->perf.core_clk_rate = clk_rate;
416 		DRM_DEBUG_ATOMIC("update clk rate = %lld HZ\n", clk_rate);
417 	}
418 	return 0;
419 }
420 
421 #ifdef CONFIG_DEBUG_FS
422 
_dpu_core_perf_mode_write(struct file * file,const char __user * user_buf,size_t count,loff_t * ppos)423 static ssize_t _dpu_core_perf_mode_write(struct file *file,
424 		    const char __user *user_buf, size_t count, loff_t *ppos)
425 {
426 	struct dpu_core_perf *perf = file->private_data;
427 	struct dpu_perf_cfg *cfg = &perf->catalog->perf;
428 	u32 perf_mode = 0;
429 	int ret;
430 
431 	ret = kstrtouint_from_user(user_buf, count, 0, &perf_mode);
432 	if (ret)
433 		return ret;
434 
435 	if (perf_mode >= DPU_PERF_MODE_MAX)
436 		return -EINVAL;
437 
438 	if (perf_mode == DPU_PERF_MODE_FIXED) {
439 		DRM_INFO("fix performance mode\n");
440 	} else if (perf_mode == DPU_PERF_MODE_MINIMUM) {
441 		/* run the driver with max clk and BW vote */
442 		perf->perf_tune.min_core_clk = perf->max_core_clk_rate;
443 		perf->perf_tune.min_bus_vote =
444 				(u64) cfg->max_bw_high * 1000;
445 		DRM_INFO("minimum performance mode\n");
446 	} else if (perf_mode == DPU_PERF_MODE_NORMAL) {
447 		/* reset the perf tune params to 0 */
448 		perf->perf_tune.min_core_clk = 0;
449 		perf->perf_tune.min_bus_vote = 0;
450 		DRM_INFO("normal performance mode\n");
451 	}
452 	perf->perf_tune.mode = perf_mode;
453 
454 	return count;
455 }
456 
_dpu_core_perf_mode_read(struct file * file,char __user * buff,size_t count,loff_t * ppos)457 static ssize_t _dpu_core_perf_mode_read(struct file *file,
458 			char __user *buff, size_t count, loff_t *ppos)
459 {
460 	struct dpu_core_perf *perf = file->private_data;
461 	int len;
462 	char buf[128];
463 
464 	len = scnprintf(buf, sizeof(buf),
465 			"mode %d min_mdp_clk %llu min_bus_vote %llu\n",
466 			perf->perf_tune.mode,
467 			perf->perf_tune.min_core_clk,
468 			perf->perf_tune.min_bus_vote);
469 
470 	return simple_read_from_buffer(buff, count, ppos, buf, len);
471 }
472 
473 static const struct file_operations dpu_core_perf_mode_fops = {
474 	.open = simple_open,
475 	.read = _dpu_core_perf_mode_read,
476 	.write = _dpu_core_perf_mode_write,
477 };
478 
dpu_core_perf_debugfs_init(struct dpu_kms * dpu_kms,struct dentry * parent)479 int dpu_core_perf_debugfs_init(struct dpu_kms *dpu_kms, struct dentry *parent)
480 {
481 	struct dpu_core_perf *perf = &dpu_kms->perf;
482 	struct dpu_mdss_cfg *catalog = perf->catalog;
483 	struct dentry *entry;
484 
485 	entry = debugfs_create_dir("core_perf", parent);
486 
487 	debugfs_create_u64("max_core_clk_rate", 0600, entry,
488 			&perf->max_core_clk_rate);
489 	debugfs_create_u64("core_clk_rate", 0600, entry,
490 			&perf->core_clk_rate);
491 	debugfs_create_u32("enable_bw_release", 0600, entry,
492 			(u32 *)&perf->enable_bw_release);
493 	debugfs_create_u32("threshold_low", 0600, entry,
494 			(u32 *)&catalog->perf.max_bw_low);
495 	debugfs_create_u32("threshold_high", 0600, entry,
496 			(u32 *)&catalog->perf.max_bw_high);
497 	debugfs_create_u32("min_core_ib", 0600, entry,
498 			(u32 *)&catalog->perf.min_core_ib);
499 	debugfs_create_u32("min_llcc_ib", 0600, entry,
500 			(u32 *)&catalog->perf.min_llcc_ib);
501 	debugfs_create_u32("min_dram_ib", 0600, entry,
502 			(u32 *)&catalog->perf.min_dram_ib);
503 	debugfs_create_file("perf_mode", 0600, entry,
504 			(u32 *)perf, &dpu_core_perf_mode_fops);
505 	debugfs_create_u64("fix_core_clk_rate", 0600, entry,
506 			&perf->fix_core_clk_rate);
507 	debugfs_create_u64("fix_core_ib_vote", 0600, entry,
508 			&perf->fix_core_ib_vote);
509 	debugfs_create_u64("fix_core_ab_vote", 0600, entry,
510 			&perf->fix_core_ab_vote);
511 
512 	return 0;
513 }
514 #endif
515 
dpu_core_perf_destroy(struct dpu_core_perf * perf)516 void dpu_core_perf_destroy(struct dpu_core_perf *perf)
517 {
518 	if (!perf) {
519 		DPU_ERROR("invalid parameters\n");
520 		return;
521 	}
522 
523 	perf->max_core_clk_rate = 0;
524 	perf->core_clk = NULL;
525 	perf->catalog = NULL;
526 	perf->dev = NULL;
527 }
528 
dpu_core_perf_init(struct dpu_core_perf * perf,struct drm_device * dev,struct dpu_mdss_cfg * catalog,struct dss_clk * core_clk)529 int dpu_core_perf_init(struct dpu_core_perf *perf,
530 		struct drm_device *dev,
531 		struct dpu_mdss_cfg *catalog,
532 		struct dss_clk *core_clk)
533 {
534 	perf->dev = dev;
535 	perf->catalog = catalog;
536 	perf->core_clk = core_clk;
537 
538 	perf->max_core_clk_rate = core_clk->max_rate;
539 	if (!perf->max_core_clk_rate) {
540 		DPU_DEBUG("optional max core clk rate, use default\n");
541 		perf->max_core_clk_rate = DPU_PERF_DEFAULT_MAX_CORE_CLK_RATE;
542 	}
543 
544 	return 0;
545 }
546