Lines Matching refs:rtp
15 typedef void (*rcu_tasks_gp_func_t)(struct rcu_tasks *rtp);
20 typedef void (*postgp_func_t)(struct rcu_tasks *rtp);
132 static void set_tasks_gp_state(struct rcu_tasks *rtp, int newstate) in set_tasks_gp_state() argument
134 rtp->gp_state = newstate; in set_tasks_gp_state()
135 rtp->gp_jiffies = jiffies; in set_tasks_gp_state()
140 static const char *tasks_gp_state_getname(struct rcu_tasks *rtp) in tasks_gp_state_getname() argument
142 int i = data_race(rtp->gp_state); // Let KCSAN detect update races in tasks_gp_state_getname()
153 struct rcu_tasks *rtp) in call_rcu_tasks_generic() argument
160 raw_spin_lock_irqsave(&rtp->cbs_lock, flags); in call_rcu_tasks_generic()
161 needwake = !rtp->cbs_head; in call_rcu_tasks_generic()
162 WRITE_ONCE(*rtp->cbs_tail, rhp); in call_rcu_tasks_generic()
163 rtp->cbs_tail = &rhp->next; in call_rcu_tasks_generic()
164 raw_spin_unlock_irqrestore(&rtp->cbs_lock, flags); in call_rcu_tasks_generic()
166 if (needwake && READ_ONCE(rtp->kthread_ptr)) in call_rcu_tasks_generic()
167 wake_up(&rtp->cbs_wq); in call_rcu_tasks_generic()
171 static void synchronize_rcu_tasks_generic(struct rcu_tasks *rtp) in synchronize_rcu_tasks_generic() argument
178 wait_rcu_gp(rtp->call_func); in synchronize_rcu_tasks_generic()
187 struct rcu_tasks *rtp = arg; in rcu_tasks_kthread() local
191 WRITE_ONCE(rtp->kthread_ptr, current); // Let GPs start! in rcu_tasks_kthread()
200 set_tasks_gp_state(rtp, RTGS_WAIT_CBS); in rcu_tasks_kthread()
203 raw_spin_lock_irqsave(&rtp->cbs_lock, flags); in rcu_tasks_kthread()
205 list = rtp->cbs_head; in rcu_tasks_kthread()
206 rtp->cbs_head = NULL; in rcu_tasks_kthread()
207 rtp->cbs_tail = &rtp->cbs_head; in rcu_tasks_kthread()
208 raw_spin_unlock_irqrestore(&rtp->cbs_lock, flags); in rcu_tasks_kthread()
212 wait_event_interruptible(rtp->cbs_wq, in rcu_tasks_kthread()
213 READ_ONCE(rtp->cbs_head)); in rcu_tasks_kthread()
214 if (!rtp->cbs_head) { in rcu_tasks_kthread()
216 set_tasks_gp_state(rtp, RTGS_WAIT_WAIT_CBS); in rcu_tasks_kthread()
223 set_tasks_gp_state(rtp, RTGS_WAIT_GP); in rcu_tasks_kthread()
224 rtp->gp_start = jiffies; in rcu_tasks_kthread()
225 rtp->gp_func(rtp); in rcu_tasks_kthread()
226 rtp->n_gps++; in rcu_tasks_kthread()
229 set_tasks_gp_state(rtp, RTGS_INVOKE_CBS); in rcu_tasks_kthread()
239 schedule_timeout_idle(rtp->gp_sleep); in rcu_tasks_kthread()
244 static void __init rcu_spawn_tasks_kthread_generic(struct rcu_tasks *rtp) in rcu_spawn_tasks_kthread_generic() argument
248 t = kthread_run(rcu_tasks_kthread, rtp, "%s_kthread", rtp->kname); in rcu_spawn_tasks_kthread_generic()
249 …%s: Could not start %s grace-period kthread, OOM is now expected behavior\n", __func__, rtp->name)) in rcu_spawn_tasks_kthread_generic()
280 static void show_rcu_tasks_generic_gp_kthread(struct rcu_tasks *rtp, char *s) in show_rcu_tasks_generic_gp_kthread() argument
283 rtp->kname, in show_rcu_tasks_generic_gp_kthread()
284 tasks_gp_state_getname(rtp), data_race(rtp->gp_state), in show_rcu_tasks_generic_gp_kthread()
285 jiffies - data_race(rtp->gp_jiffies), in show_rcu_tasks_generic_gp_kthread()
286 data_race(rtp->n_gps), in show_rcu_tasks_generic_gp_kthread()
287 data_race(rtp->n_ipis_fails), data_race(rtp->n_ipis), in show_rcu_tasks_generic_gp_kthread()
288 ".k"[!!data_race(rtp->kthread_ptr)], in show_rcu_tasks_generic_gp_kthread()
289 ".C"[!!data_race(rtp->cbs_head)], in show_rcu_tasks_generic_gp_kthread()
303 static void rcu_tasks_wait_gp(struct rcu_tasks *rtp) in rcu_tasks_wait_gp() argument
310 set_tasks_gp_state(rtp, RTGS_PRE_WAIT_GP); in rcu_tasks_wait_gp()
311 rtp->pregp_func(); in rcu_tasks_wait_gp()
319 set_tasks_gp_state(rtp, RTGS_SCAN_TASKLIST); in rcu_tasks_wait_gp()
322 rtp->pertask_func(t, &holdouts); in rcu_tasks_wait_gp()
325 set_tasks_gp_state(rtp, RTGS_POST_SCAN_TASKLIST); in rcu_tasks_wait_gp()
326 rtp->postscan_func(&holdouts); in rcu_tasks_wait_gp()
336 fract = rtp->init_fract; in rcu_tasks_wait_gp()
344 set_tasks_gp_state(rtp, RTGS_WAIT_SCAN_HOLDOUTS); in rcu_tasks_wait_gp()
356 set_tasks_gp_state(rtp, RTGS_SCAN_HOLDOUTS); in rcu_tasks_wait_gp()
357 rtp->holdouts_func(&holdouts, needreport, &firstreport); in rcu_tasks_wait_gp()
360 set_tasks_gp_state(rtp, RTGS_POST_GP); in rcu_tasks_wait_gp()
361 rtp->postgp_func(rtp); in rcu_tasks_wait_gp()
508 static void rcu_tasks_postgp(struct rcu_tasks *rtp) in rcu_tasks_postgp() argument
661 static void rcu_tasks_rude_wait_gp(struct rcu_tasks *rtp) in rcu_tasks_rude_wait_gp() argument
663 rtp->n_ipis += cpumask_weight(cpu_online_mask); in rcu_tasks_rude_wait_gp()
1145 static void rcu_tasks_trace_postgp(struct rcu_tasks *rtp) in rcu_tasks_trace_postgp() argument
1168 set_tasks_gp_state(rtp, RTGS_WAIT_READERS); in rcu_tasks_trace_postgp()