Lines Matching refs:sds
6109 struct sched_domain_shared *sds; in set_idle_cores() local
6111 sds = rcu_dereference(per_cpu(sd_llc_shared, cpu)); in set_idle_cores()
6112 if (sds) in set_idle_cores()
6113 WRITE_ONCE(sds->has_idle_cores, val); in set_idle_cores()
6118 struct sched_domain_shared *sds; in test_idle_cores() local
6120 sds = rcu_dereference(per_cpu(sd_llc_shared, cpu)); in test_idle_cores()
6121 if (sds) in test_idle_cores()
6122 return READ_ONCE(sds->has_idle_cores); in test_idle_cores()
8309 static inline void init_sd_lb_stats(struct sd_lb_stats *sds) in init_sd_lb_stats() argument
8318 *sds = (struct sd_lb_stats){ in init_sd_lb_stats()
8590 static bool asym_smt_can_pull_tasks(int dst_cpu, struct sd_lb_stats *sds, in asym_smt_can_pull_tasks() argument
8598 local_is_smt = sds->local->flags & SD_SHARE_CPUCAPACITY; in asym_smt_can_pull_tasks()
8624 int local_busy_cpus = sds->local->group_weight - in asym_smt_can_pull_tasks()
8625 sds->local_stat.idle_cpus; in asym_smt_can_pull_tasks()
8639 if (!sds->local_stat.sum_nr_running) in asym_smt_can_pull_tasks()
8650 sched_asym(struct lb_env *env, struct sd_lb_stats *sds, struct sg_lb_stats *sgs, in sched_asym() argument
8654 if ((sds->local->flags & SD_SHARE_CPUCAPACITY) || in sched_asym()
8656 return asym_smt_can_pull_tasks(env->dst_cpu, sds, sgs, group); in sched_asym()
8669 struct sd_lb_stats *sds, in update_sg_lb_stats() argument
8678 local_group = group == sds->local; in update_sg_lb_stats()
8728 sched_asym(env, sds, sgs, group)) { in update_sg_lb_stats()
8754 struct sd_lb_stats *sds, in update_sd_pick_busiest() argument
8758 struct sg_lb_stats *busiest = &sds->busiest_stat; in update_sd_pick_busiest()
8772 sds->local_stat.group_type != group_has_spare)) in update_sd_pick_busiest()
8802 if (sched_asym_prefer(sg->asym_prefer_cpu, sds->busiest->asym_prefer_cpu)) in update_sd_pick_busiest()
9218 static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sds) in update_sd_lb_stats() argument
9222 struct sg_lb_stats *local = &sds->local_stat; in update_sd_lb_stats()
9232 sds->local = sg; in update_sd_lb_stats()
9240 update_sg_lb_stats(env, sds, sg, sgs, &sg_status); in update_sd_lb_stats()
9246 if (update_sd_pick_busiest(env, sds, sg, sgs)) { in update_sd_lb_stats()
9247 sds->busiest = sg; in update_sd_lb_stats()
9248 sds->busiest_stat = *sgs; in update_sd_lb_stats()
9253 sds->total_load += sgs->group_load; in update_sd_lb_stats()
9254 sds->total_capacity += sgs->group_capacity; in update_sd_lb_stats()
9260 sds->prefer_sibling = child && child->flags & SD_PREFER_SIBLING; in update_sd_lb_stats()
9264 env->fbq_type = fbq_classify_group(&sds->busiest_stat); in update_sd_lb_stats()
9307 static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *sds) in calculate_imbalance() argument
9311 local = &sds->local_stat; in calculate_imbalance()
9312 busiest = &sds->busiest_stat; in calculate_imbalance()
9377 if (busiest->group_weight == 1 || sds->prefer_sibling) { in calculate_imbalance()
9419 sds->avg_load = (sds->total_load * SCHED_CAPACITY_SCALE) / in calculate_imbalance()
9420 sds->total_capacity; in calculate_imbalance()
9441 (busiest->avg_load - sds->avg_load) * busiest->group_capacity, in calculate_imbalance()
9442 (sds->avg_load - local->avg_load) * local->group_capacity in calculate_imbalance()
9482 struct sd_lb_stats sds; in find_busiest_group() local
9484 init_sd_lb_stats(&sds); in find_busiest_group()
9490 update_sd_lb_stats(env, &sds); in find_busiest_group()
9499 local = &sds.local_stat; in find_busiest_group()
9500 busiest = &sds.busiest_stat; in find_busiest_group()
9503 if (!sds.busiest) in find_busiest_group()
9542 sds.avg_load = (sds.total_load * SCHED_CAPACITY_SCALE) / in find_busiest_group()
9543 sds.total_capacity; in find_busiest_group()
9549 if (local->avg_load >= sds.avg_load) in find_busiest_group()
9562 if (sds.prefer_sibling && local->group_type == group_has_spare && in find_busiest_group()
9597 calculate_imbalance(env, &sds); in find_busiest_group()
9598 return env->imbalance ? sds.busiest : NULL; in find_busiest_group()
10431 struct sched_domain_shared *sds; in nohz_balancer_kick() local
10515 sds = rcu_dereference(per_cpu(sd_llc_shared, cpu)); in nohz_balancer_kick()
10516 if (sds) { in nohz_balancer_kick()
10526 nr_busy = atomic_read(&sds->nr_busy_cpus); in nohz_balancer_kick()