Lines Matching refs:sched_domain
214 unsigned int sched_domain, unsigned int type) in flush_latency_buckets() argument
216 unsigned int *buckets = kqd->latency_buckets[sched_domain][type]; in flush_latency_buckets()
217 atomic_t *cpu_buckets = cpu_latency->buckets[sched_domain][type]; in flush_latency_buckets()
229 unsigned int sched_domain, unsigned int type, in calculate_percentile() argument
232 unsigned int *buckets = kqd->latency_buckets[sched_domain][type]; in calculate_percentile()
245 if (!kqd->latency_timeout[sched_domain]) in calculate_percentile()
246 kqd->latency_timeout[sched_domain] = max(jiffies + HZ, 1UL); in calculate_percentile()
248 time_is_after_jiffies(kqd->latency_timeout[sched_domain])) { in calculate_percentile()
251 kqd->latency_timeout[sched_domain] = 0; in calculate_percentile()
259 memset(buckets, 0, sizeof(kqd->latency_buckets[sched_domain][type])); in calculate_percentile()
261 trace_kyber_latency(kqd->dev, kyber_domain_names[sched_domain], in calculate_percentile()
269 unsigned int sched_domain, unsigned int depth) in kyber_resize_domain() argument
271 depth = clamp(depth, 1U, kyber_depth[sched_domain]); in kyber_resize_domain()
272 if (depth != kqd->domain_tokens[sched_domain].sb.depth) { in kyber_resize_domain()
273 sbitmap_queue_resize(&kqd->domain_tokens[sched_domain], depth); in kyber_resize_domain()
274 trace_kyber_adjust(kqd->dev, kyber_domain_names[sched_domain], in kyber_resize_domain()
282 unsigned int sched_domain; in kyber_timer_fn() local
291 for (sched_domain = 0; sched_domain < KYBER_OTHER; sched_domain++) { in kyber_timer_fn()
292 flush_latency_buckets(kqd, cpu_latency, sched_domain, in kyber_timer_fn()
294 flush_latency_buckets(kqd, cpu_latency, sched_domain, in kyber_timer_fn()
304 for (sched_domain = 0; sched_domain < KYBER_OTHER; sched_domain++) { in kyber_timer_fn()
307 p90 = calculate_percentile(kqd, sched_domain, KYBER_IO_LATENCY, in kyber_timer_fn()
318 for (sched_domain = 0; sched_domain < KYBER_OTHER; sched_domain++) { in kyber_timer_fn()
322 p99 = calculate_percentile(kqd, sched_domain, in kyber_timer_fn()
334 p99 = kqd->domain_p99[sched_domain]; in kyber_timer_fn()
335 kqd->domain_p99[sched_domain] = -1; in kyber_timer_fn()
337 kqd->domain_p99[sched_domain] = p99; in kyber_timer_fn()
352 orig_depth = kqd->domain_tokens[sched_domain].sb.depth; in kyber_timer_fn()
354 kyber_resize_domain(kqd, sched_domain, depth); in kyber_timer_fn()
542 unsigned int sched_domain; in rq_clear_domain_token() local
547 sched_domain = kyber_sched_domain(rq->cmd_flags); in rq_clear_domain_token()
548 sbitmap_queue_clear(&kqd->domain_tokens[sched_domain], nr, in rq_clear_domain_token()
573 unsigned int sched_domain = kyber_sched_domain(bio->bi_opf); in kyber_bio_merge() local
574 struct list_head *rq_list = &kcq->rq_list[sched_domain]; in kyber_bio_merge()
596 unsigned int sched_domain = kyber_sched_domain(rq->cmd_flags); in kyber_insert_requests() local
598 struct list_head *head = &kcq->rq_list[sched_domain]; in kyber_insert_requests()
606 sbitmap_set_bit(&khd->kcq_map[sched_domain], in kyber_insert_requests()
620 unsigned int sched_domain, unsigned int type, in add_latency_sample() argument
634 atomic_inc(&cpu_latency->buckets[sched_domain][type][bucket]); in add_latency_sample()
641 unsigned int sched_domain; in kyber_completed_request() local
644 sched_domain = kyber_sched_domain(rq->cmd_flags); in kyber_completed_request()
645 if (sched_domain == KYBER_OTHER) in kyber_completed_request()
649 target = kqd->latency_targets[sched_domain]; in kyber_completed_request()
650 add_latency_sample(cpu_latency, sched_domain, KYBER_TOTAL_LATENCY, in kyber_completed_request()
652 add_latency_sample(cpu_latency, sched_domain, KYBER_IO_LATENCY, target, in kyber_completed_request()
661 unsigned int sched_domain; member
671 list_splice_tail_init(&kcq->rq_list[flush_data->sched_domain], in flush_busy_kcq()
680 unsigned int sched_domain, in kyber_flush_busy_kcqs() argument
685 .sched_domain = sched_domain, in kyber_flush_busy_kcqs()
689 sbitmap_for_each_set(&khd->kcq_map[sched_domain], in kyber_flush_busy_kcqs()
708 unsigned int sched_domain = khd->cur_domain; in kyber_get_domain_token() local
709 struct sbitmap_queue *domain_tokens = &kqd->domain_tokens[sched_domain]; in kyber_get_domain_token()
710 struct sbq_wait *wait = &khd->domain_wait[sched_domain]; in kyber_get_domain_token()
723 &khd->wait_index[sched_domain]); in kyber_get_domain_token()
724 khd->domain_ws[sched_domain] = ws; in kyber_get_domain_token()
742 ws = khd->domain_ws[sched_domain]; in kyber_get_domain_token()