Lines Matching refs:rcu_state
85 static struct rcu_state rcu_state = { variable
86 .level = { &rcu_state.node[0] },
89 .barrier_mutex = __MUTEX_INITIALIZER(rcu_state.barrier_mutex),
92 .exp_mutex = __MUTEX_INITIALIZER(rcu_state.exp_mutex),
93 .exp_wake_mutex = __MUTEX_INITIALIZER(rcu_state.exp_wake_mutex),
94 .ofl_lock = __RAW_SPIN_LOCK_UNLOCKED(rcu_state.ofl_lock),
231 return rcu_seq_state(rcu_seq_current(&rcu_state.gp_seq)); in rcu_gp_in_progress()
560 return READ_ONCE(rcu_state.gp_seq); in rcu_get_gp_seq()
572 return rcu_state.expedited_sequence; in rcu_exp_batches_completed()
581 return &rcu_state.node[0]; in rcu_get_root()
592 *flags = READ_ONCE(rcu_state.gp_flags); in rcutorture_get_gp_data()
593 *gp_seq = rcu_seq_current(&rcu_state.gp_seq); in rcutorture_get_gp_data()
1206 trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti")); in dyntick_save_progress_counter()
1233 trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti")); in rcu_implicit_dynticks_qs()
1287 (time_after(jiffies, rcu_state.gp_start + jtsq * 2) || in rcu_implicit_dynticks_qs()
1288 time_after(jiffies, rcu_state.jiffies_resched) || in rcu_implicit_dynticks_qs()
1289 rcu_state.cbovld)) { in rcu_implicit_dynticks_qs()
1293 } else if (time_after(jiffies, rcu_state.gp_start + jtsq)) { in rcu_implicit_dynticks_qs()
1307 rcu_state.cbovld)) { in rcu_implicit_dynticks_qs()
1320 if (time_after(jiffies, rcu_state.jiffies_resched)) { in rcu_implicit_dynticks_qs()
1342 trace_rcu_future_grace_period(rcu_state.name, READ_ONCE(rnp->gp_seq), in trace_rcu_this_gp()
1415 WRITE_ONCE(rcu_state.gp_flags, rcu_state.gp_flags | RCU_GP_FLAG_INIT); in rcu_start_this_gp()
1416 WRITE_ONCE(rcu_state.gp_req_activity, jiffies); in rcu_start_this_gp()
1417 if (!READ_ONCE(rcu_state.gp_kthread)) { in rcu_start_this_gp()
1421 trace_rcu_grace_period(rcu_state.name, data_race(rcu_state.gp_seq), TPS("newreq")); in rcu_start_this_gp()
1468 struct task_struct *t = READ_ONCE(rcu_state.gp_kthread); in rcu_gp_kthread_wake()
1471 !READ_ONCE(rcu_state.gp_flags) || !t) in rcu_gp_kthread_wake()
1473 WRITE_ONCE(rcu_state.gp_wake_time, jiffies); in rcu_gp_kthread_wake()
1474 WRITE_ONCE(rcu_state.gp_wake_seq, READ_ONCE(rcu_state.gp_seq)); in rcu_gp_kthread_wake()
1475 swake_up_one(&rcu_state.gp_wq); in rcu_gp_kthread_wake()
1514 gp_seq_req = rcu_seq_snap(&rcu_state.gp_seq); in rcu_accelerate_cbs()
1520 trace_rcu_grace_period(rcu_state.name, gp_seq_req, TPS("AccWaitCB")); in rcu_accelerate_cbs()
1522 trace_rcu_grace_period(rcu_state.name, gp_seq_req, TPS("AccReadyCB")); in rcu_accelerate_cbs()
1543 c = rcu_seq_snap(&rcu_state.gp_seq); in rcu_accelerate_cbs_unlocked()
1636 trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuend")); in __note_gp_changes()
1652 trace_rcu_grace_period(rcu_state.name, rnp->gp_seq, TPS("cpustart")); in __note_gp_changes()
1690 !(rcu_seq_ctr(rcu_state.gp_seq) % in rcu_gp_slow()
1741 WRITE_ONCE(rcu_state.gp_activity, jiffies); in rcu_gp_init()
1743 if (!READ_ONCE(rcu_state.gp_flags)) { in rcu_gp_init()
1748 WRITE_ONCE(rcu_state.gp_flags, 0); /* Clear all flags: New GP. */ in rcu_gp_init()
1762 rcu_seq_start(&rcu_state.gp_seq); in rcu_gp_init()
1763 ASSERT_EXCLUSIVE_WRITER(rcu_state.gp_seq); in rcu_gp_init()
1764 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("start")); in rcu_gp_init()
1776 WRITE_ONCE(rcu_state.gp_state, RCU_GP_ONOFF); in rcu_gp_init()
1786 raw_spin_lock(&rcu_state.ofl_lock); in rcu_gp_init()
1792 raw_spin_unlock(&rcu_state.ofl_lock); in rcu_gp_init()
1828 raw_spin_unlock(&rcu_state.ofl_lock); in rcu_gp_init()
1844 WRITE_ONCE(rcu_state.gp_state, RCU_GP_INIT); in rcu_gp_init()
1851 WRITE_ONCE(rnp->gp_seq, rcu_state.gp_seq); in rcu_gp_init()
1855 trace_rcu_grace_period_init(rcu_state.name, rnp->gp_seq, in rcu_gp_init()
1866 WRITE_ONCE(rcu_state.gp_activity, jiffies); in rcu_gp_init()
1889 *gfp = READ_ONCE(rcu_state.gp_flags); in rcu_gp_fqs_check_wake()
1907 WRITE_ONCE(rcu_state.gp_activity, jiffies); in rcu_gp_fqs()
1908 WRITE_ONCE(rcu_state.n_force_qs, rcu_state.n_force_qs + 1); in rcu_gp_fqs()
1917 if (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) { in rcu_gp_fqs()
1919 WRITE_ONCE(rcu_state.gp_flags, in rcu_gp_fqs()
1920 READ_ONCE(rcu_state.gp_flags) & ~RCU_GP_FLAG_FQS); in rcu_gp_fqs()
1938 if (rcu_state.cbovld) in rcu_gp_fqs_loop()
1943 WRITE_ONCE(rcu_state.jiffies_force_qs, jiffies + j); in rcu_gp_fqs_loop()
1949 WRITE_ONCE(rcu_state.jiffies_kick_kthreads, in rcu_gp_fqs_loop()
1952 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, in rcu_gp_fqs_loop()
1954 WRITE_ONCE(rcu_state.gp_state, RCU_GP_WAIT_FQS); in rcu_gp_fqs_loop()
1955 (void)swait_event_idle_timeout_exclusive(rcu_state.gp_wq, in rcu_gp_fqs_loop()
1958 WRITE_ONCE(rcu_state.gp_state, RCU_GP_DOING_FQS); in rcu_gp_fqs_loop()
1965 if (!time_after(rcu_state.jiffies_force_qs, jiffies) || in rcu_gp_fqs_loop()
1967 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, in rcu_gp_fqs_loop()
1973 gf = rcu_state.cbovld ? RCU_GP_FLAG_OVLD : 0; in rcu_gp_fqs_loop()
1975 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, in rcu_gp_fqs_loop()
1978 WRITE_ONCE(rcu_state.gp_activity, jiffies); in rcu_gp_fqs_loop()
1984 WRITE_ONCE(rcu_state.gp_activity, jiffies); in rcu_gp_fqs_loop()
1986 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, in rcu_gp_fqs_loop()
1990 if (time_after(jiffies, rcu_state.jiffies_force_qs)) in rcu_gp_fqs_loop()
1993 j = rcu_state.jiffies_force_qs - j; in rcu_gp_fqs_loop()
2013 WRITE_ONCE(rcu_state.gp_activity, jiffies); in rcu_gp_cleanup()
2015 rcu_state.gp_end = jiffies; in rcu_gp_cleanup()
2016 gp_duration = rcu_state.gp_end - rcu_state.gp_start; in rcu_gp_cleanup()
2017 if (gp_duration > rcu_state.gp_max) in rcu_gp_cleanup()
2018 rcu_state.gp_max = gp_duration; in rcu_gp_cleanup()
2039 new_gp_seq = rcu_state.gp_seq; in rcu_gp_cleanup()
2062 WRITE_ONCE(rcu_state.gp_activity, jiffies); in rcu_gp_cleanup()
2069 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("end")); in rcu_gp_cleanup()
2070 rcu_seq_end(&rcu_state.gp_seq); in rcu_gp_cleanup()
2071 ASSERT_EXCLUSIVE_WRITER(rcu_state.gp_seq); in rcu_gp_cleanup()
2072 WRITE_ONCE(rcu_state.gp_state, RCU_GP_IDLE); in rcu_gp_cleanup()
2083 WRITE_ONCE(rcu_state.gp_flags, RCU_GP_FLAG_INIT); in rcu_gp_cleanup()
2084 WRITE_ONCE(rcu_state.gp_req_activity, jiffies); in rcu_gp_cleanup()
2085 trace_rcu_grace_period(rcu_state.name, in rcu_gp_cleanup()
2086 rcu_state.gp_seq, in rcu_gp_cleanup()
2089 WRITE_ONCE(rcu_state.gp_flags, in rcu_gp_cleanup()
2090 rcu_state.gp_flags & RCU_GP_FLAG_INIT); in rcu_gp_cleanup()
2109 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, in rcu_gp_kthread()
2111 WRITE_ONCE(rcu_state.gp_state, RCU_GP_WAIT_GPS); in rcu_gp_kthread()
2112 swait_event_idle_exclusive(rcu_state.gp_wq, in rcu_gp_kthread()
2113 READ_ONCE(rcu_state.gp_flags) & in rcu_gp_kthread()
2116 WRITE_ONCE(rcu_state.gp_state, RCU_GP_DONE_GPS); in rcu_gp_kthread()
2121 WRITE_ONCE(rcu_state.gp_activity, jiffies); in rcu_gp_kthread()
2123 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, in rcu_gp_kthread()
2131 WRITE_ONCE(rcu_state.gp_state, RCU_GP_CLEANUP); in rcu_gp_kthread()
2133 WRITE_ONCE(rcu_state.gp_state, RCU_GP_CLEANED); in rcu_gp_kthread()
2151 WRITE_ONCE(rcu_state.gp_flags, in rcu_report_qs_rsp()
2152 READ_ONCE(rcu_state.gp_flags) | RCU_GP_FLAG_FQS); in rcu_report_qs_rsp()
2195 trace_rcu_quiescent_state_report(rcu_state.name, rnp->gp_seq, in rcu_report_qs_rnp()
2366 trace_rcu_grace_period(rcu_state.name, READ_ONCE(rnp->gp_seq), in rcutree_dying_cpu()
2430 WRITE_ONCE(rcu_state.n_online_cpus, rcu_state.n_online_cpus - 1); in rcutree_dead_cpu()
2455 trace_rcu_batch_start(rcu_state.name, in rcu_do_batch()
2457 trace_rcu_batch_end(rcu_state.name, 0, in rcu_do_batch()
2482 trace_rcu_batch_start(rcu_state.name, in rcu_do_batch()
2502 trace_rcu_invoke_callback(rcu_state.name, rhp); in rcu_do_batch()
2536 trace_rcu_batch_end(rcu_state.name, count, !!rcl.head, need_resched(), in rcu_do_batch()
2551 rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs); in rcu_do_batch()
2619 rcu_state.cbovld = rcu_state.cbovldnext; in force_qs_rnp()
2620 rcu_state.cbovldnext = false; in force_qs_rnp()
2625 rcu_state.cbovldnext |= !!rnp->cbovldmask; in force_qs_rnp()
2671 ret = (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) || in rcu_force_quiescent_state()
2684 if (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) { in rcu_force_quiescent_state()
2688 WRITE_ONCE(rcu_state.gp_flags, in rcu_force_quiescent_state()
2689 READ_ONCE(rcu_state.gp_flags) | RCU_GP_FLAG_FQS); in rcu_force_quiescent_state()
2899 if (READ_ONCE(rcu_state.n_force_qs) == rdp->n_force_qs_snap && in __call_rcu_core()
2902 rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs); in __call_rcu_core()
3005 trace_rcu_kvfree_callback(rcu_state.name, head, in __call_rcu()
3009 trace_rcu_callback(rcu_state.name, head, in __call_rcu()
3270 rcu_state.name, bkvhead[i]->nr_records, in kfree_rcu_work()
3278 rcu_state.name, in kfree_rcu_work()
3312 trace_rcu_invoke_kvfree_callback(rcu_state.name, head, offset); in kfree_rcu_work()
3692 ret = READ_ONCE(rcu_state.n_online_cpus) <= 1; in rcu_blocking_is_gp()
3764 return rcu_seq_snap(&rcu_state.gp_seq); in get_state_synchronize_rcu()
3828 if (rcu_seq_done(&rcu_state.gp_seq, oldstate)) { in poll_state_synchronize_rcu()
3919 trace_rcu_barrier(rcu_state.name, s, cpu, in rcu_barrier_trace()
3920 atomic_read(&rcu_state.barrier_cpu_count), done); in rcu_barrier_trace()
3935 unsigned long __maybe_unused s = rcu_state.barrier_sequence; in rcu_barrier_callback()
3937 if (atomic_dec_and_test(&rcu_state.barrier_cpu_count)) { in rcu_barrier_callback()
3939 complete(&rcu_state.barrier_completion); in rcu_barrier_callback()
3953 rcu_barrier_trace(TPS("IRQ"), -1, rcu_state.barrier_sequence); in rcu_barrier_func()
3959 atomic_inc(&rcu_state.barrier_cpu_count); in rcu_barrier_func()
3963 rcu_state.barrier_sequence); in rcu_barrier_func()
3980 unsigned long s = rcu_seq_snap(&rcu_state.barrier_sequence); in rcu_barrier()
3985 mutex_lock(&rcu_state.barrier_mutex); in rcu_barrier()
3988 if (rcu_seq_done(&rcu_state.barrier_sequence, s)) { in rcu_barrier()
3990 rcu_state.barrier_sequence); in rcu_barrier()
3992 mutex_unlock(&rcu_state.barrier_mutex); in rcu_barrier()
3997 rcu_seq_start(&rcu_state.barrier_sequence); in rcu_barrier()
3998 rcu_barrier_trace(TPS("Inc1"), -1, rcu_state.barrier_sequence); in rcu_barrier()
4007 init_completion(&rcu_state.barrier_completion); in rcu_barrier()
4008 atomic_set(&rcu_state.barrier_cpu_count, 2); in rcu_barrier()
4023 rcu_state.barrier_sequence); in rcu_barrier()
4028 rcu_state.barrier_sequence); in rcu_barrier()
4034 rcu_state.barrier_sequence); in rcu_barrier()
4037 rcu_state.barrier_sequence); in rcu_barrier()
4046 if (atomic_sub_and_test(2, &rcu_state.barrier_cpu_count)) in rcu_barrier()
4047 complete(&rcu_state.barrier_completion); in rcu_barrier()
4050 wait_for_completion(&rcu_state.barrier_completion); in rcu_barrier()
4053 rcu_barrier_trace(TPS("Inc2"), -1, rcu_state.barrier_sequence); in rcu_barrier()
4054 rcu_seq_end(&rcu_state.barrier_sequence); in rcu_barrier()
4057 mutex_unlock(&rcu_state.barrier_mutex); in rcu_barrier()
4102 rdp->rcu_ofl_gp_seq = rcu_state.gp_seq; in rcu_boot_init_percpu_data()
4104 rdp->rcu_onl_gp_seq = rcu_state.gp_seq; in rcu_boot_init_percpu_data()
4129 rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs); in rcutree_prepare_cpu()
4156 trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuonl")); in rcutree_prepare_cpu()
4160 WRITE_ONCE(rcu_state.n_online_cpus, rcu_state.n_online_cpus + 1); in rcutree_prepare_cpu()
4258 smp_store_release(&rcu_state.ncpus, rcu_state.ncpus + newcpu); /* ^^^ */ in rcu_cpu_starting()
4259 ASSERT_EXCLUSIVE_WRITER(rcu_state.ncpus); in rcu_cpu_starting()
4261 rdp->rcu_onl_gp_seq = READ_ONCE(rcu_state.gp_seq); in rcu_cpu_starting()
4262 rdp->rcu_onl_gp_flags = READ_ONCE(rcu_state.gp_flags); in rcu_cpu_starting()
4305 raw_spin_lock(&rcu_state.ofl_lock); in rcu_report_dead()
4307 rdp->rcu_ofl_gp_seq = READ_ONCE(rcu_state.gp_seq); in rcu_report_dead()
4308 rdp->rcu_ofl_gp_flags = READ_ONCE(rcu_state.gp_flags); in rcu_report_dead()
4316 raw_spin_unlock(&rcu_state.ofl_lock); in rcu_report_dead()
4423 t = kthread_create(rcu_gp_kthread, NULL, "%s", rcu_state.name); in rcu_spawn_gp_kthread()
4432 WRITE_ONCE(rcu_state.gp_activity, jiffies); in rcu_spawn_gp_kthread()
4433 WRITE_ONCE(rcu_state.gp_req_activity, jiffies); in rcu_spawn_gp_kthread()
4435 smp_store_release(&rcu_state.gp_kthread, t); /* ^^^ */ in rcu_spawn_gp_kthread()
4489 rcu_state.level[i] = in rcu_init_one()
4490 rcu_state.level[i - 1] + num_rcu_lvl[i - 1]; in rcu_init_one()
4497 rnp = rcu_state.level[i]; in rcu_init_one()
4505 rnp->gp_seq = rcu_state.gp_seq; in rcu_init_one()
4506 rnp->gp_seq_needed = rcu_state.gp_seq; in rcu_init_one()
4507 rnp->completedqs = rcu_state.gp_seq; in rcu_init_one()
4521 rnp->parent = rcu_state.level[i - 1] + in rcu_init_one()
4535 init_swait_queue_head(&rcu_state.gp_wq); in rcu_init_one()
4536 init_swait_queue_head(&rcu_state.expedited_wq); in rcu_init_one()