Lines Matching refs:dl_se
23 static inline struct task_struct *dl_task_of(struct sched_dl_entity *dl_se) in dl_task_of() argument
25 return container_of(dl_se, struct task_struct, dl); in dl_task_of()
33 static inline struct dl_rq *dl_rq_of_se(struct sched_dl_entity *dl_se) in dl_rq_of_se() argument
35 struct task_struct *p = dl_task_of(dl_se); in dl_rq_of_se()
41 static inline int on_dl_rq(struct sched_dl_entity *dl_se) in on_dl_rq() argument
43 return !RB_EMPTY_NODE(&dl_se->rb_node); in on_dl_rq()
47 static inline struct sched_dl_entity *pi_of(struct sched_dl_entity *dl_se) in pi_of() argument
49 return dl_se->pi_se; in pi_of()
52 static inline bool is_dl_boosted(struct sched_dl_entity *dl_se) in is_dl_boosted() argument
54 return pi_of(dl_se) != dl_se; in is_dl_boosted()
57 static inline struct sched_dl_entity *pi_of(struct sched_dl_entity *dl_se) in pi_of() argument
59 return dl_se; in pi_of()
62 static inline bool is_dl_boosted(struct sched_dl_entity *dl_se) in is_dl_boosted() argument
206 void add_rq_bw(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) in add_rq_bw() argument
208 if (!dl_entity_is_special(dl_se)) in add_rq_bw()
209 __add_rq_bw(dl_se->dl_bw, dl_rq); in add_rq_bw()
213 void sub_rq_bw(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) in sub_rq_bw() argument
215 if (!dl_entity_is_special(dl_se)) in sub_rq_bw()
216 __sub_rq_bw(dl_se->dl_bw, dl_rq); in sub_rq_bw()
220 void add_running_bw(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) in add_running_bw() argument
222 if (!dl_entity_is_special(dl_se)) in add_running_bw()
223 __add_running_bw(dl_se->dl_bw, dl_rq); in add_running_bw()
227 void sub_running_bw(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) in sub_running_bw() argument
229 if (!dl_entity_is_special(dl_se)) in sub_running_bw()
230 __sub_running_bw(dl_se->dl_bw, dl_rq); in sub_running_bw()
316 struct sched_dl_entity *dl_se = &p->dl; in task_non_contending() local
317 struct hrtimer *timer = &dl_se->inactive_timer; in task_non_contending()
318 struct dl_rq *dl_rq = dl_rq_of_se(dl_se); in task_non_contending()
326 if (dl_se->dl_runtime == 0) in task_non_contending()
329 if (dl_entity_is_special(dl_se)) in task_non_contending()
332 WARN_ON(dl_se->dl_non_contending); in task_non_contending()
334 zerolag_time = dl_se->deadline - in task_non_contending()
335 div64_long((dl_se->runtime * dl_se->dl_period), in task_non_contending()
336 dl_se->dl_runtime); in task_non_contending()
348 if ((zerolag_time < 0) || hrtimer_active(&dl_se->inactive_timer)) { in task_non_contending()
350 sub_running_bw(dl_se, dl_rq); in task_non_contending()
365 dl_se->dl_non_contending = 1; in task_non_contending()
370 static void task_contending(struct sched_dl_entity *dl_se, int flags) in task_contending() argument
372 struct dl_rq *dl_rq = dl_rq_of_se(dl_se); in task_contending()
378 if (dl_se->dl_runtime == 0) in task_contending()
382 add_rq_bw(dl_se, dl_rq); in task_contending()
384 if (dl_se->dl_non_contending) { in task_contending()
385 dl_se->dl_non_contending = 0; in task_contending()
393 if (hrtimer_try_to_cancel(&dl_se->inactive_timer) == 1) in task_contending()
394 put_task_struct(dl_task_of(dl_se)); in task_contending()
403 add_running_bw(dl_se, dl_rq); in task_contending()
409 struct sched_dl_entity *dl_se = &p->dl; in is_leftmost() local
411 return dl_rq->root.rb_leftmost == &dl_se->rb_node; in is_leftmost()
500 static void inc_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) in inc_dl_migration() argument
502 struct task_struct *p = dl_task_of(dl_se); in inc_dl_migration()
510 static void dec_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) in dec_dl_migration() argument
512 struct task_struct *p = dl_task_of(dl_se); in dec_dl_migration()
677 void inc_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) in inc_dl_migration() argument
682 void dec_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) in dec_dl_migration() argument
720 static inline void setup_new_dl_entity(struct sched_dl_entity *dl_se) in setup_new_dl_entity() argument
722 struct dl_rq *dl_rq = dl_rq_of_se(dl_se); in setup_new_dl_entity()
725 WARN_ON(is_dl_boosted(dl_se)); in setup_new_dl_entity()
726 WARN_ON(dl_time_before(rq_clock(rq), dl_se->deadline)); in setup_new_dl_entity()
733 if (dl_se->dl_throttled) in setup_new_dl_entity()
741 dl_se->deadline = rq_clock(rq) + dl_se->dl_deadline; in setup_new_dl_entity()
742 dl_se->runtime = dl_se->dl_runtime; in setup_new_dl_entity()
763 static void replenish_dl_entity(struct sched_dl_entity *dl_se) in replenish_dl_entity() argument
765 struct dl_rq *dl_rq = dl_rq_of_se(dl_se); in replenish_dl_entity()
768 BUG_ON(pi_of(dl_se)->dl_runtime <= 0); in replenish_dl_entity()
774 if (dl_se->dl_deadline == 0) { in replenish_dl_entity()
775 dl_se->deadline = rq_clock(rq) + pi_of(dl_se)->dl_deadline; in replenish_dl_entity()
776 dl_se->runtime = pi_of(dl_se)->dl_runtime; in replenish_dl_entity()
779 if (dl_se->dl_yielded && dl_se->runtime > 0) in replenish_dl_entity()
780 dl_se->runtime = 0; in replenish_dl_entity()
788 while (dl_se->runtime <= 0) { in replenish_dl_entity()
789 dl_se->deadline += pi_of(dl_se)->dl_period; in replenish_dl_entity()
790 dl_se->runtime += pi_of(dl_se)->dl_runtime; in replenish_dl_entity()
802 if (dl_time_before(dl_se->deadline, rq_clock(rq))) { in replenish_dl_entity()
804 dl_se->deadline = rq_clock(rq) + pi_of(dl_se)->dl_deadline; in replenish_dl_entity()
805 dl_se->runtime = pi_of(dl_se)->dl_runtime; in replenish_dl_entity()
808 if (dl_se->dl_yielded) in replenish_dl_entity()
809 dl_se->dl_yielded = 0; in replenish_dl_entity()
810 if (dl_se->dl_throttled) in replenish_dl_entity()
811 dl_se->dl_throttled = 0; in replenish_dl_entity()
838 static bool dl_entity_overflow(struct sched_dl_entity *dl_se, u64 t) in dl_entity_overflow() argument
860 left = (pi_of(dl_se)->dl_deadline >> DL_SCALE) * (dl_se->runtime >> DL_SCALE); in dl_entity_overflow()
861 right = ((dl_se->deadline - t) >> DL_SCALE) * in dl_entity_overflow()
862 (pi_of(dl_se)->dl_runtime >> DL_SCALE); in dl_entity_overflow()
886 update_dl_revised_wakeup(struct sched_dl_entity *dl_se, struct rq *rq) in update_dl_revised_wakeup() argument
888 u64 laxity = dl_se->deadline - rq_clock(rq); in update_dl_revised_wakeup()
896 WARN_ON(dl_time_before(dl_se->deadline, rq_clock(rq))); in update_dl_revised_wakeup()
898 dl_se->runtime = (dl_se->dl_density * laxity) >> BW_SHIFT; in update_dl_revised_wakeup()
912 static inline bool dl_is_implicit(struct sched_dl_entity *dl_se) in dl_is_implicit() argument
914 return dl_se->dl_deadline == dl_se->dl_period; in dl_is_implicit()
947 static void update_dl_entity(struct sched_dl_entity *dl_se) in update_dl_entity() argument
949 struct dl_rq *dl_rq = dl_rq_of_se(dl_se); in update_dl_entity()
952 if (dl_time_before(dl_se->deadline, rq_clock(rq)) || in update_dl_entity()
953 dl_entity_overflow(dl_se, rq_clock(rq))) { in update_dl_entity()
955 if (unlikely(!dl_is_implicit(dl_se) && in update_dl_entity()
956 !dl_time_before(dl_se->deadline, rq_clock(rq)) && in update_dl_entity()
957 !is_dl_boosted(dl_se))) { in update_dl_entity()
958 update_dl_revised_wakeup(dl_se, rq); in update_dl_entity()
962 dl_se->deadline = rq_clock(rq) + pi_of(dl_se)->dl_deadline; in update_dl_entity()
963 dl_se->runtime = pi_of(dl_se)->dl_runtime; in update_dl_entity()
967 static inline u64 dl_next_period(struct sched_dl_entity *dl_se) in dl_next_period() argument
969 return dl_se->deadline - dl_se->dl_deadline + dl_se->dl_period; in dl_next_period()
984 struct sched_dl_entity *dl_se = &p->dl; in start_dl_timer() local
985 struct hrtimer *timer = &dl_se->dl_timer; in start_dl_timer()
997 act = ns_to_ktime(dl_next_period(dl_se)); in start_dl_timer()
1042 struct sched_dl_entity *dl_se = container_of(timer, in dl_task_timer() local
1045 struct task_struct *p = dl_task_of(dl_se); in dl_task_timer()
1062 if (is_dl_boosted(dl_se)) in dl_task_timer()
1069 if (!dl_se->dl_throttled) in dl_task_timer()
1090 replenish_dl_entity(dl_se); in dl_task_timer()
1147 void init_dl_task_timer(struct sched_dl_entity *dl_se) in init_dl_task_timer() argument
1149 struct hrtimer *timer = &dl_se->dl_timer; in init_dl_task_timer()
1173 static inline void dl_check_constrained_dl(struct sched_dl_entity *dl_se) in dl_check_constrained_dl() argument
1175 struct task_struct *p = dl_task_of(dl_se); in dl_check_constrained_dl()
1176 struct rq *rq = rq_of_dl_rq(dl_rq_of_se(dl_se)); in dl_check_constrained_dl()
1178 if (dl_time_before(dl_se->deadline, rq_clock(rq)) && in dl_check_constrained_dl()
1179 dl_time_before(rq_clock(rq), dl_next_period(dl_se))) { in dl_check_constrained_dl()
1180 if (unlikely(is_dl_boosted(dl_se) || !start_dl_timer(p))) in dl_check_constrained_dl()
1182 dl_se->dl_throttled = 1; in dl_check_constrained_dl()
1183 if (dl_se->runtime > 0) in dl_check_constrained_dl()
1184 dl_se->runtime = 0; in dl_check_constrained_dl()
1189 int dl_runtime_exceeded(struct sched_dl_entity *dl_se) in dl_runtime_exceeded() argument
1191 return (dl_se->runtime <= 0); in dl_runtime_exceeded()
1215 static u64 grub_reclaim(u64 delta, struct rq *rq, struct sched_dl_entity *dl_se) in grub_reclaim() argument
1219 u64 u_act_min = (dl_se->dl_bw * rq->dl.bw_ratio) >> RATIO_SHIFT; in grub_reclaim()
1244 struct sched_dl_entity *dl_se = &curr->dl; in update_curr_dl() local
1249 if (!dl_task(curr) || !on_dl_rq(dl_se)) in update_curr_dl()
1263 if (unlikely(dl_se->dl_yielded)) in update_curr_dl()
1279 if (dl_entity_is_special(dl_se)) in update_curr_dl()
1289 if (unlikely(dl_se->flags & SCHED_FLAG_RECLAIM)) { in update_curr_dl()
1301 dl_se->runtime -= scaled_delta_exec; in update_curr_dl()
1304 if (dl_runtime_exceeded(dl_se) || dl_se->dl_yielded) { in update_curr_dl()
1305 dl_se->dl_throttled = 1; in update_curr_dl()
1308 if (dl_runtime_exceeded(dl_se) && in update_curr_dl()
1309 (dl_se->flags & SCHED_FLAG_DL_OVERRUN)) in update_curr_dl()
1310 dl_se->dl_overrun = 1; in update_curr_dl()
1313 if (unlikely(is_dl_boosted(dl_se) || !start_dl_timer(curr))) in update_curr_dl()
1348 struct sched_dl_entity *dl_se = container_of(timer, in inactive_task_timer() local
1351 struct task_struct *p = dl_task_of(dl_se); in inactive_task_timer()
1363 if (READ_ONCE(p->__state) == TASK_DEAD && dl_se->dl_non_contending) { in inactive_task_timer()
1366 dl_se->dl_non_contending = 0; in inactive_task_timer()
1376 if (dl_se->dl_non_contending == 0) in inactive_task_timer()
1379 sub_running_bw(dl_se, &rq->dl); in inactive_task_timer()
1380 dl_se->dl_non_contending = 0; in inactive_task_timer()
1388 void init_dl_inactive_task_timer(struct sched_dl_entity *dl_se) in init_dl_inactive_task_timer() argument
1390 struct hrtimer *timer = &dl_se->inactive_timer; in init_dl_inactive_task_timer()
1442 void inc_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) in inc_dl_tasks() argument
1444 int prio = dl_task_of(dl_se)->prio; in inc_dl_tasks()
1445 u64 deadline = dl_se->deadline; in inc_dl_tasks()
1452 inc_dl_migration(dl_se, dl_rq); in inc_dl_tasks()
1456 void dec_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) in dec_dl_tasks() argument
1458 int prio = dl_task_of(dl_se)->prio; in dec_dl_tasks()
1465 dec_dl_deadline(dl_rq, dl_se->deadline); in dec_dl_tasks()
1466 dec_dl_migration(dl_se, dl_rq); in dec_dl_tasks()
1478 __schedstats_from_dl_se(struct sched_dl_entity *dl_se) in __schedstats_from_dl_se() argument
1480 return &dl_task_of(dl_se)->stats; in __schedstats_from_dl_se()
1484 update_stats_wait_start_dl(struct dl_rq *dl_rq, struct sched_dl_entity *dl_se) in update_stats_wait_start_dl() argument
1491 stats = __schedstats_from_dl_se(dl_se); in update_stats_wait_start_dl()
1492 __update_stats_wait_start(rq_of_dl_rq(dl_rq), dl_task_of(dl_se), stats); in update_stats_wait_start_dl()
1496 update_stats_wait_end_dl(struct dl_rq *dl_rq, struct sched_dl_entity *dl_se) in update_stats_wait_end_dl() argument
1503 stats = __schedstats_from_dl_se(dl_se); in update_stats_wait_end_dl()
1504 __update_stats_wait_end(rq_of_dl_rq(dl_rq), dl_task_of(dl_se), stats); in update_stats_wait_end_dl()
1508 update_stats_enqueue_sleeper_dl(struct dl_rq *dl_rq, struct sched_dl_entity *dl_se) in update_stats_enqueue_sleeper_dl() argument
1515 stats = __schedstats_from_dl_se(dl_se); in update_stats_enqueue_sleeper_dl()
1516 __update_stats_enqueue_sleeper(rq_of_dl_rq(dl_rq), dl_task_of(dl_se), stats); in update_stats_enqueue_sleeper_dl()
1520 update_stats_enqueue_dl(struct dl_rq *dl_rq, struct sched_dl_entity *dl_se, in update_stats_enqueue_dl() argument
1527 update_stats_enqueue_sleeper_dl(dl_rq, dl_se); in update_stats_enqueue_dl()
1531 update_stats_dequeue_dl(struct dl_rq *dl_rq, struct sched_dl_entity *dl_se, in update_stats_dequeue_dl() argument
1534 struct task_struct *p = dl_task_of(dl_se); in update_stats_dequeue_dl()
1553 static void __enqueue_dl_entity(struct sched_dl_entity *dl_se) in __enqueue_dl_entity() argument
1555 struct dl_rq *dl_rq = dl_rq_of_se(dl_se); in __enqueue_dl_entity()
1557 BUG_ON(!RB_EMPTY_NODE(&dl_se->rb_node)); in __enqueue_dl_entity()
1559 rb_add_cached(&dl_se->rb_node, &dl_rq->root, __dl_less); in __enqueue_dl_entity()
1561 inc_dl_tasks(dl_se, dl_rq); in __enqueue_dl_entity()
1564 static void __dequeue_dl_entity(struct sched_dl_entity *dl_se) in __dequeue_dl_entity() argument
1566 struct dl_rq *dl_rq = dl_rq_of_se(dl_se); in __dequeue_dl_entity()
1568 if (RB_EMPTY_NODE(&dl_se->rb_node)) in __dequeue_dl_entity()
1571 rb_erase_cached(&dl_se->rb_node, &dl_rq->root); in __dequeue_dl_entity()
1573 RB_CLEAR_NODE(&dl_se->rb_node); in __dequeue_dl_entity()
1575 dec_dl_tasks(dl_se, dl_rq); in __dequeue_dl_entity()
1579 enqueue_dl_entity(struct sched_dl_entity *dl_se, int flags) in enqueue_dl_entity() argument
1581 BUG_ON(on_dl_rq(dl_se)); in enqueue_dl_entity()
1583 update_stats_enqueue_dl(dl_rq_of_se(dl_se), dl_se, flags); in enqueue_dl_entity()
1591 task_contending(dl_se, flags); in enqueue_dl_entity()
1592 update_dl_entity(dl_se); in enqueue_dl_entity()
1594 replenish_dl_entity(dl_se); in enqueue_dl_entity()
1596 dl_time_before(dl_se->deadline, in enqueue_dl_entity()
1597 rq_clock(rq_of_dl_rq(dl_rq_of_se(dl_se))))) { in enqueue_dl_entity()
1598 setup_new_dl_entity(dl_se); in enqueue_dl_entity()
1601 __enqueue_dl_entity(dl_se); in enqueue_dl_entity()
1604 static void dequeue_dl_entity(struct sched_dl_entity *dl_se) in dequeue_dl_entity() argument
1606 __dequeue_dl_entity(dl_se); in dequeue_dl_entity()
1912 struct sched_dl_entity *dl_se = &p->dl; in set_next_task_dl() local
1917 update_stats_wait_end_dl(dl_rq, dl_se); in set_next_task_dl()
1947 struct sched_dl_entity *dl_se; in pick_task_dl() local
1954 dl_se = pick_next_dl_entity(rq, dl_rq); in pick_task_dl()
1955 BUG_ON(!dl_se); in pick_task_dl()
1956 p = dl_task_of(dl_se); in pick_task_dl()
1974 struct sched_dl_entity *dl_se = &p->dl; in put_prev_task_dl() local
1978 update_stats_wait_start_dl(dl_rq, dl_se); in put_prev_task_dl()
2835 struct sched_dl_entity *dl_se = &p->dl; in __setparam_dl() local
2837 dl_se->dl_runtime = attr->sched_runtime; in __setparam_dl()
2838 dl_se->dl_deadline = attr->sched_deadline; in __setparam_dl()
2839 dl_se->dl_period = attr->sched_period ?: dl_se->dl_deadline; in __setparam_dl()
2840 dl_se->flags = attr->sched_flags & SCHED_DL_FLAGS; in __setparam_dl()
2841 dl_se->dl_bw = to_ratio(dl_se->dl_period, dl_se->dl_runtime); in __setparam_dl()
2842 dl_se->dl_density = to_ratio(dl_se->dl_deadline, dl_se->dl_runtime); in __setparam_dl()
2847 struct sched_dl_entity *dl_se = &p->dl; in __getparam_dl() local
2850 attr->sched_runtime = dl_se->dl_runtime; in __getparam_dl()
2851 attr->sched_deadline = dl_se->dl_deadline; in __getparam_dl()
2852 attr->sched_period = dl_se->dl_period; in __getparam_dl()
2854 attr->sched_flags |= dl_se->flags; in __getparam_dl()
2925 struct sched_dl_entity *dl_se = &p->dl; in __dl_clear_params() local
2927 dl_se->dl_runtime = 0; in __dl_clear_params()
2928 dl_se->dl_deadline = 0; in __dl_clear_params()
2929 dl_se->dl_period = 0; in __dl_clear_params()
2930 dl_se->flags = 0; in __dl_clear_params()
2931 dl_se->dl_bw = 0; in __dl_clear_params()
2932 dl_se->dl_density = 0; in __dl_clear_params()
2934 dl_se->dl_throttled = 0; in __dl_clear_params()
2935 dl_se->dl_yielded = 0; in __dl_clear_params()
2936 dl_se->dl_non_contending = 0; in __dl_clear_params()
2937 dl_se->dl_overrun = 0; in __dl_clear_params()
2940 dl_se->pi_se = dl_se; in __dl_clear_params()
2946 struct sched_dl_entity *dl_se = &p->dl; in dl_param_changed() local
2948 if (dl_se->dl_runtime != attr->sched_runtime || in dl_param_changed()
2949 dl_se->dl_deadline != attr->sched_deadline || in dl_param_changed()
2950 dl_se->dl_period != attr->sched_period || in dl_param_changed()
2951 dl_se->flags != (attr->sched_flags & SCHED_DL_FLAGS)) in dl_param_changed()