Lines Matching refs:rdp
235 struct rcu_data *rdp; in rcu_report_exp_cpu_mult() local
244 rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_report_exp_cpu_mult()
245 if (!IS_ENABLED(CONFIG_NO_HZ_FULL) || !rdp->rcu_forced_tick_exp) in rcu_report_exp_cpu_mult()
247 rdp->rcu_forced_tick_exp = false; in rcu_report_exp_cpu_mult()
256 static void rcu_report_exp_rdp(struct rcu_data *rdp) in rcu_report_exp_rdp() argument
258 WRITE_ONCE(rdp->exp_deferred_qs, false); in rcu_report_exp_rdp()
259 rcu_report_exp_cpu_mult(rdp->mynode, rdp->grpmask, true); in rcu_report_exp_rdp()
282 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, raw_smp_processor_id()); in exp_funnel_lock() local
283 struct rcu_node *rnp = rdp->mynode; in exp_funnel_lock()
353 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in sync_rcu_exp_select_node_cpus() local
354 unsigned long mask = rdp->grpmask; in sync_rcu_exp_select_node_cpus()
361 snap = rcu_dynticks_snap(rdp); in sync_rcu_exp_select_node_cpus()
365 rdp->exp_dynticks_snap = snap; in sync_rcu_exp_select_node_cpus()
381 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in sync_rcu_exp_select_node_cpus() local
382 unsigned long mask = rdp->grpmask; in sync_rcu_exp_select_node_cpus()
385 if (rcu_dynticks_in_eqs_since(rdp, rdp->exp_dynticks_snap)) { in sync_rcu_exp_select_node_cpus()
493 struct rcu_data *rdp; in synchronize_rcu_expedited_wait() local
505 rdp = per_cpu_ptr(&rcu_data, cpu); in synchronize_rcu_expedited_wait()
506 if (rdp->rcu_forced_tick_exp) in synchronize_rcu_expedited_wait()
508 rdp->rcu_forced_tick_exp = true; in synchronize_rcu_expedited_wait()
530 struct rcu_data *rdp; in synchronize_rcu_expedited_wait() local
536 rdp = per_cpu_ptr(&rcu_data, cpu); in synchronize_rcu_expedited_wait()
539 "o."[!!(rdp->grpmask & rnp->expmaskinit)], in synchronize_rcu_expedited_wait()
540 "N."[!!(rdp->grpmask & rnp->expmaskinitnext)]); in synchronize_rcu_expedited_wait()
644 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_exp_handler() local
645 struct rcu_node *rnp = rdp->mynode; in rcu_exp_handler()
656 rcu_report_exp_rdp(rdp); in rcu_exp_handler()
658 rdp->exp_deferred_qs = true; in rcu_exp_handler()
679 if (rnp->expmask & rdp->grpmask) { in rcu_exp_handler()
680 rdp->exp_deferred_qs = true; in rcu_exp_handler()
735 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_exp_handler() local
736 struct rcu_node *rnp = rdp->mynode; in rcu_exp_handler()
738 if (!(READ_ONCE(rnp->expmask) & rdp->grpmask) || in rcu_exp_handler()
753 struct rcu_data *rdp; in sync_sched_exp_online_cleanup() local
757 rdp = per_cpu_ptr(&rcu_data, cpu); in sync_sched_exp_online_cleanup()
758 rnp = rdp->mynode; in sync_sched_exp_online_cleanup()
761 if (!(READ_ONCE(rnp->expmask) & rdp->grpmask) || in sync_sched_exp_online_cleanup()
762 rdp->cpu_no_qs.b.exp) { in sync_sched_exp_online_cleanup()