Lines Matching refs:rcu_state
75 return !time_before(j, READ_ONCE(rcu_state.gp_start) + d); in rcu_gp_might_be_stalled()
128 WRITE_ONCE(rcu_state.jiffies_stall, in rcu_cpu_stall_reset()
142 WRITE_ONCE(rcu_state.gp_start, j); in record_gp_stall_check_time()
145 WRITE_ONCE(rcu_state.jiffies_stall, j + j1); in record_gp_stall_check_time()
146 rcu_state.jiffies_resched = j + j1 / 2; in record_gp_stall_check_time()
147 rcu_state.n_force_qs_gpstart = READ_ONCE(rcu_state.n_force_qs); in record_gp_stall_check_time()
168 j = READ_ONCE(rcu_state.jiffies_kick_kthreads); in rcu_stall_kick_kthreads()
169 if (time_after(jiffies, j) && rcu_state.gp_kthread && in rcu_stall_kick_kthreads()
170 (rcu_gp_in_progress() || READ_ONCE(rcu_state.gp_flags))) { in rcu_stall_kick_kthreads()
172 rcu_state.name); in rcu_stall_kick_kthreads()
174 wake_up_process(rcu_state.gp_kthread); in rcu_stall_kick_kthreads()
175 WRITE_ONCE(rcu_state.jiffies_kick_kthreads, j + HZ); in rcu_stall_kick_kthreads()
395 unsigned long j = jiffies - READ_ONCE(rcu_state.gp_activity); in rcu_is_gp_kthread_starving()
428 ticks_value = rcu_seq_ctr(rcu_state.gp_seq - rdp->gp_seq); in print_cpu_stall_info()
451 data_race(rcu_state.n_force_qs) - rcu_state.n_force_qs_gpstart, in print_cpu_stall_info()
460 struct task_struct *gpk = rcu_state.gp_kthread; in rcu_check_gp_kthread_starvation()
466 rcu_state.name, j, in rcu_check_gp_kthread_starvation()
467 (long)rcu_seq_current(&rcu_state.gp_seq), in rcu_check_gp_kthread_starvation()
468 data_race(READ_ONCE(rcu_state.gp_flags)), in rcu_check_gp_kthread_starvation()
469 gp_state_getname(rcu_state.gp_state), in rcu_check_gp_kthread_starvation()
470 data_race(READ_ONCE(rcu_state.gp_state)), in rcu_check_gp_kthread_starvation()
473 …r("\tUnless %s kthread gets sufficient CPU time, OOM is now expected behavior.\n", rcu_state.name); in rcu_check_gp_kthread_starvation()
493 struct task_struct *gpk = rcu_state.gp_kthread; in rcu_check_gp_kthread_expired_fqs_timer()
502 gp_state = smp_load_acquire(&rcu_state.gp_state); in rcu_check_gp_kthread_expired_fqs_timer()
503 jiffies_fqs = READ_ONCE(rcu_state.jiffies_force_qs); in rcu_check_gp_kthread_expired_fqs_timer()
510 rcu_state.name, (jiffies - jiffies_fqs), in rcu_check_gp_kthread_expired_fqs_timer()
511 (long)rcu_seq_current(&rcu_state.gp_seq), in rcu_check_gp_kthread_expired_fqs_timer()
512 data_race(rcu_state.gp_flags), in rcu_check_gp_kthread_expired_fqs_timer()
542 trace_rcu_stall_warning(rcu_state.name, TPS("StallDetected")); in print_other_cpu_stall()
543 pr_err("INFO: %s detected stalls on CPUs/tasks:\n", rcu_state.name); in print_other_cpu_stall()
561 (long)rcu_seq_current(&rcu_state.gp_seq), totqlen); in print_other_cpu_stall()
569 if (rcu_seq_current(&rcu_state.gp_seq) != gp_seq) { in print_other_cpu_stall()
573 gpa = data_race(READ_ONCE(rcu_state.gp_activity)); in print_other_cpu_stall()
575 rcu_state.name, j - gpa, j, gpa, in print_other_cpu_stall()
581 if (ULONG_CMP_GE(jiffies, READ_ONCE(rcu_state.jiffies_stall))) in print_other_cpu_stall()
582 WRITE_ONCE(rcu_state.jiffies_stall, in print_other_cpu_stall()
613 trace_rcu_stall_warning(rcu_state.name, TPS("SelfDetected")); in print_cpu_stall()
614 pr_err("INFO: %s self-detected stall on CPU\n", rcu_state.name); in print_cpu_stall()
622 (long)rcu_seq_current(&rcu_state.gp_seq), totqlen); in print_cpu_stall()
631 if (ULONG_CMP_GE(jiffies, READ_ONCE(rcu_state.jiffies_stall))) in print_cpu_stall()
632 WRITE_ONCE(rcu_state.jiffies_stall, in print_cpu_stall()
685 gs1 = READ_ONCE(rcu_state.gp_seq); in check_cpu_stall()
687 js = READ_ONCE(rcu_state.jiffies_stall); in check_cpu_stall()
689 gps = READ_ONCE(rcu_state.gp_start); in check_cpu_stall()
691 gs2 = READ_ONCE(rcu_state.gp_seq); in check_cpu_stall()
700 cmpxchg(&rcu_state.jiffies_stall, js, jn) == js) { in check_cpu_stall()
718 cmpxchg(&rcu_state.jiffies_stall, js, jn) == js) { in check_cpu_stall()
734 if (didstall && READ_ONCE(rcu_state.jiffies_stall) == jn) { in check_cpu_stall()
736 WRITE_ONCE(rcu_state.jiffies_stall, jn); in check_cpu_stall()
816 struct task_struct *t = READ_ONCE(rcu_state.gp_kthread); in show_rcu_gp_kthreads()
819 ja = j - data_race(READ_ONCE(rcu_state.gp_activity)); in show_rcu_gp_kthreads()
820 jr = j - data_race(READ_ONCE(rcu_state.gp_req_activity)); in show_rcu_gp_kthreads()
821 js = j - data_race(READ_ONCE(rcu_state.gp_start)); in show_rcu_gp_kthreads()
822 jw = j - data_race(READ_ONCE(rcu_state.gp_wake_time)); in show_rcu_gp_kthreads()
824 rcu_state.name, gp_state_getname(rcu_state.gp_state), in show_rcu_gp_kthreads()
825 data_race(READ_ONCE(rcu_state.gp_state)), in show_rcu_gp_kthreads()
827 js, ja, jr, jw, (long)data_race(READ_ONCE(rcu_state.gp_wake_seq)), in show_rcu_gp_kthreads()
828 (long)data_race(READ_ONCE(rcu_state.gp_seq)), in show_rcu_gp_kthreads()
830 data_race(READ_ONCE(rcu_state.gp_max)), in show_rcu_gp_kthreads()
831 data_race(READ_ONCE(rcu_state.gp_flags))); in show_rcu_gp_kthreads()
833 if (ULONG_CMP_GE(READ_ONCE(rcu_state.gp_seq), READ_ONCE(rnp->gp_seq_needed)) && in show_rcu_gp_kthreads()
852 ULONG_CMP_GE(READ_ONCE(rcu_state.gp_seq), in show_rcu_gp_kthreads()
885 !smp_load_acquire(&rcu_state.gp_kthread)) // Get stable kthread. in rcu_check_gp_start_stall()
888 if (time_before(j, READ_ONCE(rcu_state.gp_req_activity) + gpssdelay) || in rcu_check_gp_start_stall()
889 time_before(j, READ_ONCE(rcu_state.gp_activity) + gpssdelay) || in rcu_check_gp_start_stall()
898 time_before(j, READ_ONCE(rcu_state.gp_req_activity) + gpssdelay) || in rcu_check_gp_start_stall()
899 time_before(j, READ_ONCE(rcu_state.gp_activity) + gpssdelay) || in rcu_check_gp_start_stall()
912 time_before(j, READ_ONCE(rcu_state.gp_req_activity) + gpssdelay) || in rcu_check_gp_start_stall()
913 time_before(j, READ_ONCE(rcu_state.gp_activity) + gpssdelay) || in rcu_check_gp_start_stall()
943 __func__, jiffies - data_race(READ_ONCE(rcu_state.gp_start))); in rcu_fwd_progress_check()
947 __func__, jiffies - data_race(READ_ONCE(rcu_state.gp_end))); in rcu_fwd_progress_check()