Lines Matching refs:rcu_data
77 static DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, rcu_data) = {
151 static void rcu_report_exp_rdp(struct rcu_data *rdp);
153 static void check_cb_ovld_locked(struct rcu_data *rdp, struct rcu_node *rnp);
154 static bool rcu_rdp_is_offloaded(struct rcu_data *rdp);
240 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_get_n_cbs_cpu()
260 return arch_atomic_add_return(incby, this_cpu_ptr(&rcu_data.dynticks)); in rcu_dynticks_inc()
316 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_dynticks_eqs_online()
330 return !(arch_atomic_read(this_cpu_ptr(&rcu_data.dynticks)) & 0x1); in rcu_dynticks_curr_cpu_in_eqs()
337 static int rcu_dynticks_snap(struct rcu_data *rdp) in rcu_dynticks_snap()
355 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_is_idle_cpu()
365 static bool rcu_dynticks_in_eqs_since(struct rcu_data *rdp, int snap) in rcu_dynticks_in_eqs_since()
376 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_dynticks_zero_in_eqs()
406 raw_cpu_write(rcu_data.rcu_need_heavy_qs, false); in rcu_momentary_dyntick_idle()
434 RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nesting) < 0, in rcu_is_cpu_rrupt_from_idle()
436 RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nmi_nesting) <= 0, in rcu_is_cpu_rrupt_from_idle()
440 nesting = __this_cpu_read(rcu_data.dynticks_nmi_nesting); in rcu_is_cpu_rrupt_from_idle()
450 return __this_cpu_read(rcu_data.dynticks_nesting) == 0; in rcu_is_cpu_rrupt_from_idle()
552 static void force_qs_rnp(int (*f)(struct rcu_data *rdp));
611 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_eqs_enter()
685 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_irq_work_resched()
743 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_nmi_exit()
819 RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nesting) <= 0, in rcu_irq_exit_check_preempt()
821 RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nmi_nesting) != in rcu_irq_exit_check_preempt()
854 struct rcu_data *rdp; in rcu_eqs_exit()
858 rdp = this_cpu_ptr(&rcu_data); in rcu_eqs_exit()
946 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in __rcu_irq_enter_check_tick()
995 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_nmi_enter()
1094 static void rcu_disable_urgency_upon_qs(struct rcu_data *rdp) in rcu_disable_urgency_upon_qs()
1142 smp_store_release(per_cpu_ptr(&rcu_data.rcu_urgent_qs, cpu), true); in rcu_request_urgent_qs_task()
1162 struct rcu_data *rdp; in rcu_lockdep_current_cpu_online()
1169 rdp = this_cpu_ptr(&rcu_data); in rcu_lockdep_current_cpu_online()
1187 static void rcu_gpnum_ovf(struct rcu_node *rnp, struct rcu_data *rdp) in rcu_gpnum_ovf()
1202 static int dyntick_save_progress_counter(struct rcu_data *rdp) in dyntick_save_progress_counter()
1219 static int rcu_implicit_dynticks_qs(struct rcu_data *rdp) in rcu_implicit_dynticks_qs()
1339 static void trace_rcu_this_gp(struct rcu_node *rnp, struct rcu_data *rdp, in trace_rcu_this_gp()
1363 static bool rcu_start_this_gp(struct rcu_node *rnp_start, struct rcu_data *rdp, in rcu_start_this_gp()
1441 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_future_gp_cleanup()
1490 static bool rcu_accelerate_cbs(struct rcu_node *rnp, struct rcu_data *rdp) in rcu_accelerate_cbs()
1537 struct rcu_data *rdp) in rcu_accelerate_cbs_unlocked()
1566 static bool rcu_advance_cbs(struct rcu_node *rnp, struct rcu_data *rdp) in rcu_advance_cbs()
1590 struct rcu_data *rdp) in rcu_advance_cbs_nowake()
1619 static bool __note_gp_changes(struct rcu_node *rnp, struct rcu_data *rdp) in __note_gp_changes()
1666 static void note_gp_changes(struct rcu_data *rdp) in note_gp_changes()
1738 struct rcu_data *rdp; in rcu_gp_init()
1848 rdp = this_cpu_ptr(&rcu_data); in rcu_gp_init()
2009 struct rcu_data *rdp; in rcu_gp_cleanup()
2047 rdp = this_cpu_ptr(&rcu_data); in rcu_gp_cleanup()
2055 rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_gp_cleanup()
2074 rdp = this_cpu_ptr(&rcu_data); in rcu_gp_cleanup()
2275 rcu_report_qs_rdp(struct rcu_data *rdp) in rcu_report_qs_rdp()
2326 rcu_check_quiescent_state(struct rcu_data *rdp) in rcu_check_quiescent_state()
2359 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcutree_dying_cpu()
2424 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcutree_dead_cpu()
2442 static void rcu_do_batch(struct rcu_data *rdp) in rcu_do_batch()
2586 raw_cpu_inc(rcu_data.ticks_this_gp); in rcu_sched_clock_irq()
2588 if (smp_load_acquire(this_cpu_ptr(&rcu_data.rcu_urgent_qs))) { in rcu_sched_clock_irq()
2594 __this_cpu_write(rcu_data.rcu_urgent_qs, false); in rcu_sched_clock_irq()
2611 static void force_qs_rnp(int (*f)(struct rcu_data *rdp)) in force_qs_rnp()
2616 struct rcu_data *rdp; in force_qs_rnp()
2641 rdp = per_cpu_ptr(&rcu_data, cpu); in force_qs_rnp()
2669 rnp = __this_cpu_read(rcu_data.mynode); in rcu_force_quiescent_state()
2707 struct rcu_data *rdp = raw_cpu_ptr(&rcu_data); in rcu_core()
2773 __this_cpu_write(rcu_data.rcu_cpu_has_work, 1); in invoke_rcu_core_kthread()
2774 t = __this_cpu_read(rcu_data.rcu_cpu_kthread_task); in invoke_rcu_core_kthread()
2776 rcu_wake_cond(t, __this_cpu_read(rcu_data.rcu_cpu_kthread_status)); in invoke_rcu_core_kthread()
2795 per_cpu(rcu_data.rcu_cpu_kthread_status, cpu) = RCU_KTHREAD_OFFCPU; in rcu_cpu_kthread_park()
2800 return __this_cpu_read(rcu_data.rcu_cpu_has_work); in rcu_cpu_kthread_should_run()
2810 unsigned int *statusp = this_cpu_ptr(&rcu_data.rcu_cpu_kthread_status); in rcu_cpu_kthread()
2811 char work, *workp = this_cpu_ptr(&rcu_data.rcu_cpu_has_work); in rcu_cpu_kthread()
2839 .store = &rcu_data.rcu_cpu_kthread_task,
2855 per_cpu(rcu_data.rcu_cpu_has_work, cpu) = 0; in rcu_spawn_core_kthreads()
2866 static void __call_rcu_core(struct rcu_data *rdp, struct rcu_head *head, in __call_rcu_core()
2921 static void check_cb_ovld_locked(struct rcu_data *rdp, struct rcu_node *rnp) in check_cb_ovld_locked()
2944 static void check_cb_ovld(struct rcu_data *rdp) in check_cb_ovld()
2963 struct rcu_data *rdp; in __call_rcu()
2986 rdp = this_cpu_ptr(&rcu_data); in __call_rcu()
3785 struct rcu_data *rdp; in start_poll_synchronize_rcu()
3790 rdp = this_cpu_ptr(&rcu_data); in start_poll_synchronize_rcu()
3872 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_pending()
3951 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_barrier_func()
3979 struct rcu_data *rdp; in rcu_barrier()
4017 rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_barrier()
4095 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_boot_init_percpu_data()
4123 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcutree_prepare_cpu()
4170 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcutree_affinity_setting()
4182 struct rcu_data *rdp; in rcutree_online_cpu()
4185 rdp = per_cpu_ptr(&rcu_data, cpu); in rcutree_online_cpu()
4207 struct rcu_data *rdp; in rcutree_offline_cpu()
4210 rdp = per_cpu_ptr(&rcu_data, cpu); in rcutree_offline_cpu()
4238 struct rcu_data *rdp; in rcu_cpu_starting()
4242 rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_cpu_starting()
4290 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_report_dead()
4333 struct rcu_data *my_rdp; in rcutree_migrate_callbacks()
4335 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcutree_migrate_callbacks()
4343 my_rdp = this_cpu_ptr(&rcu_data); in rcutree_migrate_callbacks()
4541 per_cpu_ptr(&rcu_data, i)->mynode = rnp; in rcu_init_one()