1 #ifdef CONFIG_SMP
2 #include "sched-pelt.h"
3
4 int __update_load_avg_blocked_se(u64 now, struct sched_entity *se);
5 int __update_load_avg_se(u64 now, struct cfs_rq *cfs_rq, struct sched_entity *se);
6 int __update_load_avg_cfs_rq(u64 now, struct cfs_rq *cfs_rq);
7 int update_rt_rq_load_avg(u64 now, struct rq *rq, int running);
8 int update_dl_rq_load_avg(u64 now, struct rq *rq, int running);
9
10 #ifdef CONFIG_SCHED_THERMAL_PRESSURE
11 int update_thermal_load_avg(u64 now, struct rq *rq, u64 capacity);
12
thermal_load_avg(struct rq * rq)13 static inline u64 thermal_load_avg(struct rq *rq)
14 {
15 return READ_ONCE(rq->avg_thermal.load_avg);
16 }
17 #else
18 static inline int
update_thermal_load_avg(u64 now,struct rq * rq,u64 capacity)19 update_thermal_load_avg(u64 now, struct rq *rq, u64 capacity)
20 {
21 return 0;
22 }
23
thermal_load_avg(struct rq * rq)24 static inline u64 thermal_load_avg(struct rq *rq)
25 {
26 return 0;
27 }
28 #endif
29
30 #ifdef CONFIG_HAVE_SCHED_AVG_IRQ
31 int update_irq_load_avg(struct rq *rq, u64 running);
32 #else
33 static inline int
update_irq_load_avg(struct rq * rq,u64 running)34 update_irq_load_avg(struct rq *rq, u64 running)
35 {
36 return 0;
37 }
38 #endif
39
get_pelt_divider(struct sched_avg * avg)40 static inline u32 get_pelt_divider(struct sched_avg *avg)
41 {
42 return LOAD_AVG_MAX - 1024 + avg->period_contrib;
43 }
44
cfs_se_util_change(struct sched_avg * avg)45 static inline void cfs_se_util_change(struct sched_avg *avg)
46 {
47 unsigned int enqueued;
48
49 if (!sched_feat(UTIL_EST))
50 return;
51
52 /* Avoid store if the flag has been already reset */
53 enqueued = avg->util_est.enqueued;
54 if (!(enqueued & UTIL_AVG_UNCHANGED))
55 return;
56
57 /* Reset flag to report util_avg has been updated */
58 enqueued &= ~UTIL_AVG_UNCHANGED;
59 WRITE_ONCE(avg->util_est.enqueued, enqueued);
60 }
61
62 /*
63 * The clock_pelt scales the time to reflect the effective amount of
64 * computation done during the running delta time but then sync back to
65 * clock_task when rq is idle.
66 *
67 *
68 * absolute time | 1| 2| 3| 4| 5| 6| 7| 8| 9|10|11|12|13|14|15|16
69 * @ max capacity ------******---------------******---------------
70 * @ half capacity ------************---------************---------
71 * clock pelt | 1| 2| 3| 4| 7| 8| 9| 10| 11|14|15|16
72 *
73 */
update_rq_clock_pelt(struct rq * rq,s64 delta)74 static inline void update_rq_clock_pelt(struct rq *rq, s64 delta)
75 {
76 if (unlikely(is_idle_task(rq->curr))) {
77 /* The rq is idle, we can sync to clock_task */
78 rq->clock_pelt = rq_clock_task(rq);
79 return;
80 }
81
82 /*
83 * When a rq runs at a lower compute capacity, it will need
84 * more time to do the same amount of work than at max
85 * capacity. In order to be invariant, we scale the delta to
86 * reflect how much work has been really done.
87 * Running longer results in stealing idle time that will
88 * disturb the load signal compared to max capacity. This
89 * stolen idle time will be automatically reflected when the
90 * rq will be idle and the clock will be synced with
91 * rq_clock_task.
92 */
93
94 /*
95 * Scale the elapsed time to reflect the real amount of
96 * computation
97 */
98 delta = cap_scale(delta, arch_scale_cpu_capacity(cpu_of(rq)));
99 delta = cap_scale(delta, arch_scale_freq_capacity(cpu_of(rq)));
100
101 rq->clock_pelt += delta;
102 }
103
104 /*
105 * When rq becomes idle, we have to check if it has lost idle time
106 * because it was fully busy. A rq is fully used when the /Sum util_sum
107 * is greater or equal to:
108 * (LOAD_AVG_MAX - 1024 + rq->cfs.avg.period_contrib) << SCHED_CAPACITY_SHIFT;
109 * For optimization and computing rounding purpose, we don't take into account
110 * the position in the current window (period_contrib) and we use the higher
111 * bound of util_sum to decide.
112 */
update_idle_rq_clock_pelt(struct rq * rq)113 static inline void update_idle_rq_clock_pelt(struct rq *rq)
114 {
115 u32 divider = ((LOAD_AVG_MAX - 1024) << SCHED_CAPACITY_SHIFT) - LOAD_AVG_MAX;
116 u32 util_sum = rq->cfs.avg.util_sum;
117 util_sum += rq->avg_rt.util_sum;
118 util_sum += rq->avg_dl.util_sum;
119
120 /*
121 * Reflecting stolen time makes sense only if the idle
122 * phase would be present at max capacity. As soon as the
123 * utilization of a rq has reached the maximum value, it is
124 * considered as an always running rq without idle time to
125 * steal. This potential idle time is considered as lost in
126 * this case. We keep track of this lost idle time compare to
127 * rq's clock_task.
128 */
129 if (util_sum >= divider)
130 rq->lost_idle_time += rq_clock_task(rq) - rq->clock_pelt;
131 }
132
rq_clock_pelt(struct rq * rq)133 static inline u64 rq_clock_pelt(struct rq *rq)
134 {
135 lockdep_assert_rq_held(rq);
136 assert_clock_updated(rq);
137
138 return rq->clock_pelt - rq->lost_idle_time;
139 }
140
141 #ifdef CONFIG_CFS_BANDWIDTH
142 /* rq->task_clock normalized against any time this cfs_rq has spent throttled */
cfs_rq_clock_pelt(struct cfs_rq * cfs_rq)143 static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)
144 {
145 if (unlikely(cfs_rq->throttle_count))
146 return cfs_rq->throttled_clock_task - cfs_rq->throttled_clock_task_time;
147
148 return rq_clock_pelt(rq_of(cfs_rq)) - cfs_rq->throttled_clock_task_time;
149 }
150 #else
cfs_rq_clock_pelt(struct cfs_rq * cfs_rq)151 static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)
152 {
153 return rq_clock_pelt(rq_of(cfs_rq));
154 }
155 #endif
156
157 #else
158
159 static inline int
update_cfs_rq_load_avg(u64 now,struct cfs_rq * cfs_rq)160 update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq)
161 {
162 return 0;
163 }
164
165 static inline int
update_rt_rq_load_avg(u64 now,struct rq * rq,int running)166 update_rt_rq_load_avg(u64 now, struct rq *rq, int running)
167 {
168 return 0;
169 }
170
171 static inline int
update_dl_rq_load_avg(u64 now,struct rq * rq,int running)172 update_dl_rq_load_avg(u64 now, struct rq *rq, int running)
173 {
174 return 0;
175 }
176
177 static inline int
update_thermal_load_avg(u64 now,struct rq * rq,u64 capacity)178 update_thermal_load_avg(u64 now, struct rq *rq, u64 capacity)
179 {
180 return 0;
181 }
182
thermal_load_avg(struct rq * rq)183 static inline u64 thermal_load_avg(struct rq *rq)
184 {
185 return 0;
186 }
187
188 static inline int
update_irq_load_avg(struct rq * rq,u64 running)189 update_irq_load_avg(struct rq *rq, u64 running)
190 {
191 return 0;
192 }
193
rq_clock_pelt(struct rq * rq)194 static inline u64 rq_clock_pelt(struct rq *rq)
195 {
196 return rq_clock_task(rq);
197 }
198
199 static inline void
update_rq_clock_pelt(struct rq * rq,s64 delta)200 update_rq_clock_pelt(struct rq *rq, s64 delta) { }
201
202 static inline void
update_idle_rq_clock_pelt(struct rq * rq)203 update_idle_rq_clock_pelt(struct rq *rq) { }
204
205 #endif
206
207
208