Lines Matching refs:rt_se
113 #define rt_entity_is_task(rt_se) (!(rt_se)->my_q) argument
115 static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se) in rt_task_of() argument
118 WARN_ON_ONCE(!rt_entity_is_task(rt_se)); in rt_task_of()
120 return container_of(rt_se, struct task_struct, rt); in rt_task_of()
128 static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se) in rt_rq_of_se() argument
130 return rt_se->rt_rq; in rt_rq_of_se()
133 static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se) in rq_of_rt_se() argument
135 struct rt_rq *rt_rq = rt_se->rt_rq; in rq_of_rt_se()
142 if (tg->rt_se) in unregister_rt_sched_group()
154 if (tg->rt_se) in free_rt_sched_group()
155 kfree(tg->rt_se[i]); in free_rt_sched_group()
159 kfree(tg->rt_se); in free_rt_sched_group()
163 struct sched_rt_entity *rt_se, int cpu, in init_tg_rt_entry() argument
174 tg->rt_se[cpu] = rt_se; in init_tg_rt_entry()
176 if (!rt_se) in init_tg_rt_entry()
180 rt_se->rt_rq = &rq->rt; in init_tg_rt_entry()
182 rt_se->rt_rq = parent->my_q; in init_tg_rt_entry()
184 rt_se->my_q = rt_rq; in init_tg_rt_entry()
185 rt_se->parent = parent; in init_tg_rt_entry()
186 INIT_LIST_HEAD(&rt_se->run_list); in init_tg_rt_entry()
192 struct sched_rt_entity *rt_se; in alloc_rt_sched_group() local
198 tg->rt_se = kcalloc(nr_cpu_ids, sizeof(rt_se), GFP_KERNEL); in alloc_rt_sched_group()
199 if (!tg->rt_se) in alloc_rt_sched_group()
211 rt_se = kzalloc_node(sizeof(struct sched_rt_entity), in alloc_rt_sched_group()
213 if (!rt_se) in alloc_rt_sched_group()
218 init_tg_rt_entry(tg, rt_rq, rt_se, i, parent->rt_se[i]); in alloc_rt_sched_group()
231 #define rt_entity_is_task(rt_se) (1) argument
233 static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se) in rt_task_of() argument
235 return container_of(rt_se, struct task_struct, rt); in rt_task_of()
243 static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se) in rq_of_rt_se() argument
245 struct task_struct *p = rt_task_of(rt_se); in rq_of_rt_se()
250 static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se) in rt_rq_of_se() argument
252 struct rq *rq = rq_of_rt_se(rt_se); in rt_rq_of_se()
324 static void inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) in inc_rt_migration() argument
328 if (!rt_entity_is_task(rt_se)) in inc_rt_migration()
331 p = rt_task_of(rt_se); in inc_rt_migration()
341 static void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) in dec_rt_migration() argument
345 if (!rt_entity_is_task(rt_se)) in dec_rt_migration()
348 p = rt_task_of(rt_se); in dec_rt_migration()
418 void inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) in inc_rt_migration() argument
423 void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) in dec_rt_migration() argument
444 static inline int on_rt_rq(struct sched_rt_entity *rt_se) in on_rt_rq() argument
446 return rt_se->on_rq; in on_rt_rq()
523 #define for_each_sched_rt_entity(rt_se) \ argument
524 for (; rt_se; rt_se = rt_se->parent)
526 static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se) in group_rt_rq() argument
528 return rt_se->my_q; in group_rt_rq()
531 static void enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags);
532 static void dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags);
538 struct sched_rt_entity *rt_se; in sched_rt_rq_enqueue() local
542 rt_se = rt_rq->tg->rt_se[cpu]; in sched_rt_rq_enqueue()
545 if (!rt_se) in sched_rt_rq_enqueue()
547 else if (!on_rt_rq(rt_se)) in sched_rt_rq_enqueue()
548 enqueue_rt_entity(rt_se, 0); in sched_rt_rq_enqueue()
557 struct sched_rt_entity *rt_se; in sched_rt_rq_dequeue() local
560 rt_se = rt_rq->tg->rt_se[cpu]; in sched_rt_rq_dequeue()
562 if (!rt_se) { in sched_rt_rq_dequeue()
567 else if (on_rt_rq(rt_se)) in sched_rt_rq_dequeue()
568 dequeue_rt_entity(rt_se, 0); in sched_rt_rq_dequeue()
576 static int rt_se_boosted(struct sched_rt_entity *rt_se) in rt_se_boosted() argument
578 struct rt_rq *rt_rq = group_rt_rq(rt_se); in rt_se_boosted()
584 p = rt_task_of(rt_se); in rt_se_boosted()
628 #define for_each_sched_rt_entity(rt_se) \ argument
629 for (; rt_se; rt_se = NULL)
631 static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se) in group_rt_rq() argument
944 static inline int rt_se_prio(struct sched_rt_entity *rt_se) in rt_se_prio() argument
947 struct rt_rq *rt_rq = group_rt_rq(rt_se); in rt_se_prio()
953 return rt_task_of(rt_se)->prio; in rt_se_prio()
1006 struct sched_rt_entity *rt_se = &curr->rt; in update_curr_rt() local
1032 for_each_sched_rt_entity(rt_se) { in update_curr_rt()
1033 struct rt_rq *rt_rq = rt_rq_of_se(rt_se); in update_curr_rt()
1176 inc_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) in inc_rt_group() argument
1178 if (rt_se_boosted(rt_se)) in inc_rt_group()
1186 dec_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) in dec_rt_group() argument
1188 if (rt_se_boosted(rt_se)) in dec_rt_group()
1197 inc_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) in inc_rt_group() argument
1203 void dec_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) {} in dec_rt_group() argument
1208 unsigned int rt_se_nr_running(struct sched_rt_entity *rt_se) in rt_se_nr_running() argument
1210 struct rt_rq *group_rq = group_rt_rq(rt_se); in rt_se_nr_running()
1219 unsigned int rt_se_rr_nr_running(struct sched_rt_entity *rt_se) in rt_se_rr_nr_running() argument
1221 struct rt_rq *group_rq = group_rt_rq(rt_se); in rt_se_rr_nr_running()
1227 tsk = rt_task_of(rt_se); in rt_se_rr_nr_running()
1233 void inc_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) in inc_rt_tasks() argument
1235 int prio = rt_se_prio(rt_se); in inc_rt_tasks()
1238 rt_rq->rt_nr_running += rt_se_nr_running(rt_se); in inc_rt_tasks()
1239 rt_rq->rr_nr_running += rt_se_rr_nr_running(rt_se); in inc_rt_tasks()
1242 inc_rt_migration(rt_se, rt_rq); in inc_rt_tasks()
1243 inc_rt_group(rt_se, rt_rq); in inc_rt_tasks()
1247 void dec_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) in dec_rt_tasks() argument
1249 WARN_ON(!rt_prio(rt_se_prio(rt_se))); in dec_rt_tasks()
1251 rt_rq->rt_nr_running -= rt_se_nr_running(rt_se); in dec_rt_tasks()
1252 rt_rq->rr_nr_running -= rt_se_rr_nr_running(rt_se); in dec_rt_tasks()
1254 dec_rt_prio(rt_rq, rt_se_prio(rt_se)); in dec_rt_tasks()
1255 dec_rt_migration(rt_se, rt_rq); in dec_rt_tasks()
1256 dec_rt_group(rt_se, rt_rq); in dec_rt_tasks()
1272 static void __delist_rt_entity(struct sched_rt_entity *rt_se, struct rt_prio_array *array) in __delist_rt_entity() argument
1274 list_del_init(&rt_se->run_list); in __delist_rt_entity()
1276 if (list_empty(array->queue + rt_se_prio(rt_se))) in __delist_rt_entity()
1277 __clear_bit(rt_se_prio(rt_se), array->bitmap); in __delist_rt_entity()
1279 rt_se->on_list = 0; in __delist_rt_entity()
1283 __schedstats_from_rt_se(struct sched_rt_entity *rt_se) in __schedstats_from_rt_se() argument
1287 if (!rt_entity_is_task(rt_se)) in __schedstats_from_rt_se()
1291 return &rt_task_of(rt_se)->stats; in __schedstats_from_rt_se()
1295 update_stats_wait_start_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se) in update_stats_wait_start_rt() argument
1303 if (rt_entity_is_task(rt_se)) in update_stats_wait_start_rt()
1304 p = rt_task_of(rt_se); in update_stats_wait_start_rt()
1306 stats = __schedstats_from_rt_se(rt_se); in update_stats_wait_start_rt()
1314 update_stats_enqueue_sleeper_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se) in update_stats_enqueue_sleeper_rt() argument
1322 if (rt_entity_is_task(rt_se)) in update_stats_enqueue_sleeper_rt()
1323 p = rt_task_of(rt_se); in update_stats_enqueue_sleeper_rt()
1325 stats = __schedstats_from_rt_se(rt_se); in update_stats_enqueue_sleeper_rt()
1333 update_stats_enqueue_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se, in update_stats_enqueue_rt() argument
1340 update_stats_enqueue_sleeper_rt(rt_rq, rt_se); in update_stats_enqueue_rt()
1344 update_stats_wait_end_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se) in update_stats_wait_end_rt() argument
1352 if (rt_entity_is_task(rt_se)) in update_stats_wait_end_rt()
1353 p = rt_task_of(rt_se); in update_stats_wait_end_rt()
1355 stats = __schedstats_from_rt_se(rt_se); in update_stats_wait_end_rt()
1363 update_stats_dequeue_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se, in update_stats_dequeue_rt() argument
1371 if (rt_entity_is_task(rt_se)) in update_stats_dequeue_rt()
1372 p = rt_task_of(rt_se); in update_stats_dequeue_rt()
1388 static void __enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags) in __enqueue_rt_entity() argument
1390 struct rt_rq *rt_rq = rt_rq_of_se(rt_se); in __enqueue_rt_entity()
1392 struct rt_rq *group_rq = group_rt_rq(rt_se); in __enqueue_rt_entity()
1393 struct list_head *queue = array->queue + rt_se_prio(rt_se); in __enqueue_rt_entity()
1402 if (rt_se->on_list) in __enqueue_rt_entity()
1403 __delist_rt_entity(rt_se, array); in __enqueue_rt_entity()
1408 WARN_ON_ONCE(rt_se->on_list); in __enqueue_rt_entity()
1410 list_add(&rt_se->run_list, queue); in __enqueue_rt_entity()
1412 list_add_tail(&rt_se->run_list, queue); in __enqueue_rt_entity()
1414 __set_bit(rt_se_prio(rt_se), array->bitmap); in __enqueue_rt_entity()
1415 rt_se->on_list = 1; in __enqueue_rt_entity()
1417 rt_se->on_rq = 1; in __enqueue_rt_entity()
1419 inc_rt_tasks(rt_se, rt_rq); in __enqueue_rt_entity()
1422 static void __dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags) in __dequeue_rt_entity() argument
1424 struct rt_rq *rt_rq = rt_rq_of_se(rt_se); in __dequeue_rt_entity()
1428 WARN_ON_ONCE(!rt_se->on_list); in __dequeue_rt_entity()
1429 __delist_rt_entity(rt_se, array); in __dequeue_rt_entity()
1431 rt_se->on_rq = 0; in __dequeue_rt_entity()
1433 dec_rt_tasks(rt_se, rt_rq); in __dequeue_rt_entity()
1440 static void dequeue_rt_stack(struct sched_rt_entity *rt_se, unsigned int flags) in dequeue_rt_stack() argument
1444 for_each_sched_rt_entity(rt_se) { in dequeue_rt_stack()
1445 rt_se->back = back; in dequeue_rt_stack()
1446 back = rt_se; in dequeue_rt_stack()
1451 for (rt_se = back; rt_se; rt_se = rt_se->back) { in dequeue_rt_stack()
1452 if (on_rt_rq(rt_se)) in dequeue_rt_stack()
1453 __dequeue_rt_entity(rt_se, flags); in dequeue_rt_stack()
1457 static void enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags) in enqueue_rt_entity() argument
1459 struct rq *rq = rq_of_rt_se(rt_se); in enqueue_rt_entity()
1461 update_stats_enqueue_rt(rt_rq_of_se(rt_se), rt_se, flags); in enqueue_rt_entity()
1463 dequeue_rt_stack(rt_se, flags); in enqueue_rt_entity()
1464 for_each_sched_rt_entity(rt_se) in enqueue_rt_entity()
1465 __enqueue_rt_entity(rt_se, flags); in enqueue_rt_entity()
1469 static void dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags) in dequeue_rt_entity() argument
1471 struct rq *rq = rq_of_rt_se(rt_se); in dequeue_rt_entity()
1473 update_stats_dequeue_rt(rt_rq_of_se(rt_se), rt_se, flags); in dequeue_rt_entity()
1475 dequeue_rt_stack(rt_se, flags); in dequeue_rt_entity()
1477 for_each_sched_rt_entity(rt_se) { in dequeue_rt_entity()
1478 struct rt_rq *rt_rq = group_rt_rq(rt_se); in dequeue_rt_entity()
1481 __enqueue_rt_entity(rt_se, flags); in dequeue_rt_entity()
1492 struct sched_rt_entity *rt_se = &p->rt; in enqueue_task_rt() local
1495 rt_se->timeout = 0; in enqueue_task_rt()
1498 update_stats_wait_start_rt(rt_rq_of_se(rt_se), rt_se); in enqueue_task_rt()
1500 enqueue_rt_entity(rt_se, flags); in enqueue_task_rt()
1508 struct sched_rt_entity *rt_se = &p->rt; in dequeue_task_rt() local
1511 dequeue_rt_entity(rt_se, flags); in dequeue_task_rt()
1521 requeue_rt_entity(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se, int head) in requeue_rt_entity() argument
1523 if (on_rt_rq(rt_se)) { in requeue_rt_entity()
1525 struct list_head *queue = array->queue + rt_se_prio(rt_se); in requeue_rt_entity()
1528 list_move(&rt_se->run_list, queue); in requeue_rt_entity()
1530 list_move_tail(&rt_se->run_list, queue); in requeue_rt_entity()
1536 struct sched_rt_entity *rt_se = &p->rt; in requeue_task_rt() local
1539 for_each_sched_rt_entity(rt_se) { in requeue_task_rt()
1540 rt_rq = rt_rq_of_se(rt_se); in requeue_task_rt()
1541 requeue_rt_entity(rt_rq, rt_se, head); in requeue_task_rt()
1700 struct sched_rt_entity *rt_se = &p->rt; in set_next_task_rt() local
1705 update_stats_wait_end_rt(rt_rq, rt_se); in set_next_task_rt()
1743 struct sched_rt_entity *rt_se; in _pick_next_task_rt() local
1747 rt_se = pick_next_rt_entity(rq, rt_rq); in _pick_next_task_rt()
1748 BUG_ON(!rt_se); in _pick_next_task_rt()
1749 rt_rq = group_rt_rq(rt_se); in _pick_next_task_rt()
1752 return rt_task_of(rt_se); in _pick_next_task_rt()
1779 struct sched_rt_entity *rt_se = &p->rt; in put_prev_task_rt() local
1783 update_stats_wait_start_rt(rt_rq, rt_se); in put_prev_task_rt()
2578 struct sched_rt_entity *rt_se = &p->rt; in task_tick_rt() local
2601 for_each_sched_rt_entity(rt_se) { in task_tick_rt()
2602 if (rt_se->run_list.prev != rt_se->run_list.next) { in task_tick_rt()