Lines Matching refs:t
17 typedef void (*pertask_func_t)(struct task_struct *t, struct list_head *hop);
246 struct task_struct *t; in rcu_spawn_tasks_kthread_generic() local
248 t = kthread_run(rcu_tasks_kthread, rtp, "%s_kthread", rtp->kname); in rcu_spawn_tasks_kthread_generic()
249 …if (WARN_ONCE(IS_ERR(t), "%s: Could not start %s grace-period kthread, OOM is now expected behavio… in rcu_spawn_tasks_kthread_generic()
294 static void exit_tasks_rcu_finish_trace(struct task_struct *t);
305 struct task_struct *g, *t; in rcu_tasks_wait_gp() local
321 for_each_process_thread(g, t) in rcu_tasks_wait_gp()
322 rtp->pertask_func(t, &holdouts); in rcu_tasks_wait_gp()
440 static void rcu_tasks_pertask(struct task_struct *t, struct list_head *hop) in rcu_tasks_pertask() argument
442 if (t != current && READ_ONCE(t->on_rq) && !is_idle_task(t)) { in rcu_tasks_pertask()
443 get_task_struct(t); in rcu_tasks_pertask()
444 t->rcu_tasks_nvcsw = READ_ONCE(t->nvcsw); in rcu_tasks_pertask()
445 WRITE_ONCE(t->rcu_tasks_holdout, true); in rcu_tasks_pertask()
446 list_add(&t->rcu_tasks_holdout_list, hop); in rcu_tasks_pertask()
464 static void check_holdout_task(struct task_struct *t, in check_holdout_task() argument
469 if (!READ_ONCE(t->rcu_tasks_holdout) || in check_holdout_task()
470 t->rcu_tasks_nvcsw != READ_ONCE(t->nvcsw) || in check_holdout_task()
471 !READ_ONCE(t->on_rq) || in check_holdout_task()
473 !is_idle_task(t) && t->rcu_tasks_idle_cpu >= 0)) { in check_holdout_task()
474 WRITE_ONCE(t->rcu_tasks_holdout, false); in check_holdout_task()
475 list_del_init(&t->rcu_tasks_holdout_list); in check_holdout_task()
476 put_task_struct(t); in check_holdout_task()
479 rcu_request_urgent_qs_task(t); in check_holdout_task()
486 cpu = task_cpu(t); in check_holdout_task()
488 t, ".I"[is_idle_task(t)], in check_holdout_task()
490 t->rcu_tasks_nvcsw, t->nvcsw, t->rcu_tasks_holdout, in check_holdout_task()
491 t->rcu_tasks_idle_cpu, cpu); in check_holdout_task()
492 sched_show_task(t); in check_holdout_task()
499 struct task_struct *t, *t1; in check_all_holdout_tasks() local
501 list_for_each_entry_safe(t, t1, hop, rcu_tasks_holdout_list) { in check_all_holdout_tasks()
502 check_holdout_task(t, needreport, firstreport); in check_all_holdout_tasks()
626 struct task_struct *t = current; in exit_tasks_rcu_finish() local
629 __srcu_read_unlock(&tasks_rcu_exit_srcu, t->rcu_tasks_idx); in exit_tasks_rcu_finish()
631 exit_tasks_rcu_finish_trace(t); in exit_tasks_rcu_finish()
850 void rcu_read_unlock_trace_special(struct task_struct *t) in rcu_read_unlock_trace_special() argument
852 int nq = READ_ONCE(t->trc_reader_special.b.need_qs); in rcu_read_unlock_trace_special()
855 t->trc_reader_special.b.need_mb) in rcu_read_unlock_trace_special()
859 WRITE_ONCE(t->trc_reader_special.b.need_qs, false); in rcu_read_unlock_trace_special()
860 WRITE_ONCE(t->trc_reader_nesting, 0); in rcu_read_unlock_trace_special()
867 static void trc_add_holdout(struct task_struct *t, struct list_head *bhp) in trc_add_holdout() argument
869 if (list_empty(&t->trc_holdout_list)) { in trc_add_holdout()
870 get_task_struct(t); in trc_add_holdout()
871 list_add(&t->trc_holdout_list, bhp); in trc_add_holdout()
876 static void trc_del_holdout(struct task_struct *t) in trc_del_holdout() argument
878 if (!list_empty(&t->trc_holdout_list)) { in trc_del_holdout()
879 list_del_init(&t->trc_holdout_list); in trc_del_holdout()
880 put_task_struct(t); in trc_del_holdout()
887 struct task_struct *t = current; in trc_read_check_handler() local
891 if (unlikely(texp != t)) { in trc_read_check_handler()
897 if (likely(!READ_ONCE(t->trc_reader_nesting))) { in trc_read_check_handler()
898 WRITE_ONCE(t->trc_reader_checked, true); in trc_read_check_handler()
902 if (unlikely(READ_ONCE(t->trc_reader_nesting) < 0)) in trc_read_check_handler()
904 WRITE_ONCE(t->trc_reader_checked, true); in trc_read_check_handler()
910 WARN_ON_ONCE(READ_ONCE(t->trc_reader_special.b.need_qs)); in trc_read_check_handler()
911 WRITE_ONCE(t->trc_reader_special.b.need_qs, true); in trc_read_check_handler()
922 static int trc_inspect_reader(struct task_struct *t, void *arg) in trc_inspect_reader() argument
924 int cpu = task_cpu(t); in trc_inspect_reader()
928 if (task_curr(t)) { in trc_inspect_reader()
929 WARN_ON_ONCE(ofl && !is_idle_task(t)); in trc_inspect_reader()
940 !rcu_dynticks_zero_in_eqs(cpu, &t->trc_reader_nesting)) in trc_inspect_reader()
948 nesting = t->trc_reader_nesting; in trc_inspect_reader()
954 t->trc_reader_checked = nesting >= 0; in trc_inspect_reader()
962 WARN_ON_ONCE(READ_ONCE(t->trc_reader_special.b.need_qs)); in trc_inspect_reader()
963 WRITE_ONCE(t->trc_reader_special.b.need_qs, true); in trc_inspect_reader()
968 static void trc_wait_for_one_reader(struct task_struct *t, in trc_wait_for_one_reader() argument
974 if (smp_load_acquire(&t->trc_ipi_to_cpu) != -1) // Order IPI in trc_wait_for_one_reader()
978 if (t == current) { in trc_wait_for_one_reader()
979 t->trc_reader_checked = true; in trc_wait_for_one_reader()
980 WARN_ON_ONCE(READ_ONCE(t->trc_reader_nesting)); in trc_wait_for_one_reader()
985 get_task_struct(t); in trc_wait_for_one_reader()
986 if (!task_call_func(t, trc_inspect_reader, NULL)) { in trc_wait_for_one_reader()
987 put_task_struct(t); in trc_wait_for_one_reader()
990 put_task_struct(t); in trc_wait_for_one_reader()
999 trc_add_holdout(t, bhp); in trc_wait_for_one_reader()
1000 if (task_curr(t) && in trc_wait_for_one_reader()
1003 cpu = task_cpu(t); in trc_wait_for_one_reader()
1006 if (per_cpu(trc_ipi_to_cpu, cpu) || t->trc_ipi_to_cpu >= 0) in trc_wait_for_one_reader()
1010 t->trc_ipi_to_cpu = cpu; in trc_wait_for_one_reader()
1012 if (smp_call_function_single(cpu, trc_read_check_handler, t, 0)) { in trc_wait_for_one_reader()
1019 t->trc_ipi_to_cpu = -1; in trc_wait_for_one_reader()
1042 static void rcu_tasks_trace_pertask(struct task_struct *t, in rcu_tasks_trace_pertask() argument
1047 if (unlikely(t == NULL)) in rcu_tasks_trace_pertask()
1050 WRITE_ONCE(t->trc_reader_special.b.need_qs, false); in rcu_tasks_trace_pertask()
1051 WRITE_ONCE(t->trc_reader_checked, false); in rcu_tasks_trace_pertask()
1052 t->trc_ipi_to_cpu = -1; in rcu_tasks_trace_pertask()
1053 trc_wait_for_one_reader(t, hop); in rcu_tasks_trace_pertask()
1077 static void show_stalled_task_trace(struct task_struct *t, bool *firstreport) in show_stalled_task_trace() argument
1086 cpu = task_cpu(t); in show_stalled_task_trace()
1088 t->pid, in show_stalled_task_trace()
1089 ".I"[READ_ONCE(t->trc_ipi_to_cpu) >= 0], in show_stalled_task_trace()
1090 ".i"[is_idle_task(t)], in show_stalled_task_trace()
1092 READ_ONCE(t->trc_reader_nesting), in show_stalled_task_trace()
1093 " N"[!!READ_ONCE(t->trc_reader_special.b.need_qs)], in show_stalled_task_trace()
1095 sched_show_task(t); in show_stalled_task_trace()
1112 struct task_struct *g, *t; in check_all_holdout_tasks_trace() local
1117 list_for_each_entry_safe(t, g, hop, trc_holdout_list) { in check_all_holdout_tasks_trace()
1119 if (READ_ONCE(t->trc_ipi_to_cpu) == -1 && in check_all_holdout_tasks_trace()
1120 !READ_ONCE(t->trc_reader_checked)) in check_all_holdout_tasks_trace()
1121 trc_wait_for_one_reader(t, hop); in check_all_holdout_tasks_trace()
1124 if (READ_ONCE(t->trc_reader_checked)) in check_all_holdout_tasks_trace()
1125 trc_del_holdout(t); in check_all_holdout_tasks_trace()
1127 show_stalled_task_trace(t, firstreport); in check_all_holdout_tasks_trace()
1149 struct task_struct *g, *t; in rcu_tasks_trace_postgp() local
1178 for_each_process_thread(g, t) in rcu_tasks_trace_postgp()
1179 if (READ_ONCE(t->trc_reader_special.b.need_qs)) in rcu_tasks_trace_postgp()
1180 trc_add_holdout(t, &holdouts); in rcu_tasks_trace_postgp()
1183 list_for_each_entry_safe(t, g, &holdouts, trc_holdout_list) { in rcu_tasks_trace_postgp()
1184 if (READ_ONCE(t->trc_reader_special.b.need_qs)) in rcu_tasks_trace_postgp()
1185 show_stalled_task_trace(t, &firstreport); in rcu_tasks_trace_postgp()
1186 trc_del_holdout(t); // Release task_struct reference. in rcu_tasks_trace_postgp()
1198 static void exit_tasks_rcu_finish_trace(struct task_struct *t) in exit_tasks_rcu_finish_trace() argument
1200 WRITE_ONCE(t->trc_reader_checked, true); in exit_tasks_rcu_finish_trace()
1201 WARN_ON_ONCE(READ_ONCE(t->trc_reader_nesting)); in exit_tasks_rcu_finish_trace()
1202 WRITE_ONCE(t->trc_reader_nesting, 0); in exit_tasks_rcu_finish_trace()
1203 if (WARN_ON_ONCE(READ_ONCE(t->trc_reader_special.b.need_qs))) in exit_tasks_rcu_finish_trace()
1204 rcu_read_unlock_trace_special(t); in exit_tasks_rcu_finish_trace()
1301 static void exit_tasks_rcu_finish_trace(struct task_struct *t) { } in exit_tasks_rcu_finish_trace() argument