Lines Matching refs:rfp
2132 static void rcu_torture_fwd_cb_hist(struct rcu_fwd *rfp) in rcu_torture_fwd_cb_hist() argument
2139 for (i = ARRAY_SIZE(rfp->n_launders_hist) - 1; i > 0; i--) in rcu_torture_fwd_cb_hist()
2140 if (rfp->n_launders_hist[i].n_launders > 0) in rcu_torture_fwd_cb_hist()
2143 __func__, jiffies - rfp->rcu_fwd_startat); in rcu_torture_fwd_cb_hist()
2144 gps_old = rfp->rcu_launder_gp_seq_start; in rcu_torture_fwd_cb_hist()
2146 gps = rfp->n_launders_hist[j].launder_gp_seq; in rcu_torture_fwd_cb_hist()
2149 rfp->n_launders_hist[j].n_launders, in rcu_torture_fwd_cb_hist()
2163 struct rcu_fwd *rfp = rfcp->rfc_rfp; in rcu_torture_fwd_cb_cr() local
2167 spin_lock_irqsave(&rfp->rcu_fwd_lock, flags); in rcu_torture_fwd_cb_cr()
2168 rfcpp = rfp->rcu_fwd_cb_tail; in rcu_torture_fwd_cb_cr()
2169 rfp->rcu_fwd_cb_tail = &rfcp->rfc_next; in rcu_torture_fwd_cb_cr()
2171 WRITE_ONCE(rfp->n_launders_cb, rfp->n_launders_cb + 1); in rcu_torture_fwd_cb_cr()
2172 i = ((jiffies - rfp->rcu_fwd_startat) / (HZ / FWD_CBS_HIST_DIV)); in rcu_torture_fwd_cb_cr()
2173 if (i >= ARRAY_SIZE(rfp->n_launders_hist)) in rcu_torture_fwd_cb_cr()
2174 i = ARRAY_SIZE(rfp->n_launders_hist) - 1; in rcu_torture_fwd_cb_cr()
2175 rfp->n_launders_hist[i].n_launders++; in rcu_torture_fwd_cb_cr()
2176 rfp->n_launders_hist[i].launder_gp_seq = cur_ops->get_gp_seq(); in rcu_torture_fwd_cb_cr()
2177 spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags); in rcu_torture_fwd_cb_cr()
2197 static unsigned long rcu_torture_fwd_prog_cbfree(struct rcu_fwd *rfp) in rcu_torture_fwd_prog_cbfree() argument
2204 spin_lock_irqsave(&rfp->rcu_fwd_lock, flags); in rcu_torture_fwd_prog_cbfree()
2205 rfcp = rfp->rcu_fwd_cb_head; in rcu_torture_fwd_prog_cbfree()
2207 spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags); in rcu_torture_fwd_prog_cbfree()
2210 rfp->rcu_fwd_cb_head = rfcp->rfc_next; in rcu_torture_fwd_prog_cbfree()
2211 if (!rfp->rcu_fwd_cb_head) in rcu_torture_fwd_prog_cbfree()
2212 rfp->rcu_fwd_cb_tail = &rfp->rcu_fwd_cb_head; in rcu_torture_fwd_prog_cbfree()
2213 spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags); in rcu_torture_fwd_prog_cbfree()
2227 static void rcu_torture_fwd_prog_nr(struct rcu_fwd *rfp, in rcu_torture_fwd_prog_nr() argument
2260 WRITE_ONCE(rfp->rcu_fwd_startat, jiffies); in rcu_torture_fwd_prog_nr()
2261 stopat = rfp->rcu_fwd_startat + dur; in rcu_torture_fwd_prog_nr()
2296 static void rcu_torture_fwd_prog_cr(struct rcu_fwd *rfp) in rcu_torture_fwd_prog_cr() argument
2320 WRITE_ONCE(rfp->rcu_fwd_startat, jiffies); in rcu_torture_fwd_prog_cr()
2321 stopat = rfp->rcu_fwd_startat + MAX_FWD_CB_JIFFIES; in rcu_torture_fwd_prog_cr()
2323 rfp->n_launders_cb = 0; // Hoist initialization for multi-kthread in rcu_torture_fwd_prog_cr()
2327 for (i = 0; i < ARRAY_SIZE(rfp->n_launders_hist); i++) in rcu_torture_fwd_prog_cr()
2328 rfp->n_launders_hist[i].n_launders = 0; in rcu_torture_fwd_prog_cr()
2331 rfp->rcu_launder_gp_seq_start = gps; in rcu_torture_fwd_prog_cr()
2336 rfcp = READ_ONCE(rfp->rcu_fwd_cb_head); in rcu_torture_fwd_prog_cr()
2344 rfp->rcu_fwd_cb_head = rfcpn; in rcu_torture_fwd_prog_cr()
2356 rfcp->rfc_rfp = rfp; in rcu_torture_fwd_prog_cr()
2367 n_launders_cb_snap = READ_ONCE(rfp->n_launders_cb); in rcu_torture_fwd_prog_cr()
2371 (void)rcu_torture_fwd_prog_cbfree(rfp); in rcu_torture_fwd_prog_cr()
2378 stoppedat - rfp->rcu_fwd_startat, jiffies - stoppedat, in rcu_torture_fwd_prog_cr()
2382 rcu_torture_fwd_cb_hist(rfp); in rcu_torture_fwd_prog_cr()
2397 struct rcu_fwd *rfp; in rcutorture_oom_notify() local
2400 rfp = rcu_fwds; in rcutorture_oom_notify()
2401 if (!rfp) { in rcutorture_oom_notify()
2407 rcu_torture_fwd_cb_hist(rfp); in rcutorture_oom_notify()
2408 rcu_fwd_progress_check(1 + (jiffies - READ_ONCE(rfp->rcu_fwd_startat)) / 2); in rcutorture_oom_notify()
2412 __func__, rcu_torture_fwd_prog_cbfree(rfp)); in rcutorture_oom_notify()
2415 __func__, rcu_torture_fwd_prog_cbfree(rfp)); in rcutorture_oom_notify()
2418 __func__, rcu_torture_fwd_prog_cbfree(rfp)); in rcutorture_oom_notify()
2434 struct rcu_fwd *rfp = args; in rcu_torture_fwd_prog() local
2447 rcu_torture_fwd_prog_nr(rfp, &tested, &tested_tries); in rcu_torture_fwd_prog()
2449 rcu_torture_fwd_prog_cr(rfp); in rcu_torture_fwd_prog()
2465 struct rcu_fwd *rfp; in rcu_torture_fwd_prog_init() local
2485 rfp = kzalloc(sizeof(*rfp), GFP_KERNEL); in rcu_torture_fwd_prog_init()
2486 if (!rfp) in rcu_torture_fwd_prog_init()
2488 spin_lock_init(&rfp->rcu_fwd_lock); in rcu_torture_fwd_prog_init()
2489 rfp->rcu_fwd_cb_tail = &rfp->rcu_fwd_cb_head; in rcu_torture_fwd_prog_init()
2491 rcu_fwds = rfp; in rcu_torture_fwd_prog_init()
2494 return torture_create_kthread(rcu_torture_fwd_prog, rfp, fwd_prog_task); in rcu_torture_fwd_prog_init()
2499 struct rcu_fwd *rfp; in rcu_torture_fwd_prog_cleanup() local
2502 rfp = rcu_fwds; in rcu_torture_fwd_prog_cleanup()
2507 kfree(rfp); in rcu_torture_fwd_prog_cleanup()