Lines Matching refs:blkcg
47 struct blkcg blkcg_root;
97 css_put(&blkg->blkcg->css); in __blkg_release()
152 static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q, in blkg_alloc() argument
175 blkg->blkcg = blkcg; in blkg_alloc()
189 pd = pol->pd_alloc_fn(gfp_mask, q, blkcg); in blkg_alloc()
205 struct blkcg_gq *blkg_lookup_slowpath(struct blkcg *blkcg, in blkg_lookup_slowpath() argument
216 blkg = radix_tree_lookup(&blkcg->blkg_tree, q->id); in blkg_lookup_slowpath()
220 rcu_assign_pointer(blkcg->blkg_hint, blkg); in blkg_lookup_slowpath()
233 static struct blkcg_gq *blkg_create(struct blkcg *blkcg, in blkg_create() argument
250 if (!css_tryget_online(&blkcg->css)) { in blkg_create()
257 new_blkg = blkg_alloc(blkcg, q, GFP_NOWAIT | __GFP_NOWARN); in blkg_create()
266 if (blkcg_parent(blkcg)) { in blkg_create()
267 blkg->parent = __blkg_lookup(blkcg_parent(blkcg), q, false); in blkg_create()
284 spin_lock(&blkcg->lock); in blkg_create()
285 ret = radix_tree_insert(&blkcg->blkg_tree, q->id, blkg); in blkg_create()
287 hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list); in blkg_create()
298 spin_unlock(&blkcg->lock); in blkg_create()
308 css_put(&blkcg->css); in blkg_create()
327 static struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg, in blkg_lookup_create() argument
335 blkg = blkg_lookup(blkcg, q); in blkg_lookup_create()
340 blkg = __blkg_lookup(blkcg, q, true); in blkg_lookup_create()
350 struct blkcg *pos = blkcg; in blkg_lookup_create()
351 struct blkcg *parent = blkcg_parent(blkcg); in blkg_lookup_create()
370 if (pos == blkcg) in blkg_lookup_create()
381 struct blkcg *blkcg = blkg->blkcg; in blkg_destroy() local
385 lockdep_assert_held(&blkcg->lock); in blkg_destroy()
400 radix_tree_delete(&blkcg->blkg_tree, blkg->q->id); in blkg_destroy()
409 if (rcu_access_pointer(blkcg->blkg_hint) == blkg) in blkg_destroy()
410 rcu_assign_pointer(blkcg->blkg_hint, NULL); in blkg_destroy()
433 struct blkcg *blkcg = blkg->blkcg; in blkg_destroy_all() local
435 spin_lock(&blkcg->lock); in blkg_destroy_all()
437 spin_unlock(&blkcg->lock); in blkg_destroy_all()
458 struct blkcg *blkcg = css_to_blkcg(css); in blkcg_reset_stats() local
463 spin_lock_irq(&blkcg->lock); in blkcg_reset_stats()
470 hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) { in blkcg_reset_stats()
486 spin_unlock_irq(&blkcg->lock); in blkcg_reset_stats()
516 void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg, in blkcg_print_blkgs() argument
526 hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) { in blkcg_print_blkgs()
560 static struct blkcg_gq *blkg_lookup_check(struct blkcg *blkcg, in blkg_lookup_check() argument
569 return __blkg_lookup(blkcg, q, true /* update_hint */); in blkg_lookup_check()
622 int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol, in blkg_conf_prep() argument
648 blkg = blkg_lookup_check(blkcg, pol, q); in blkg_conf_prep()
662 struct blkcg *pos = blkcg; in blkg_conf_prep()
663 struct blkcg *parent; in blkg_conf_prep()
666 parent = blkcg_parent(blkcg); in blkg_conf_prep()
710 if (pos == blkcg) in blkg_conf_prep()
791 struct blkcg *blkcg = css_to_blkcg(css); in blkcg_rstat_flush() local
800 hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) { in blkcg_rstat_flush()
946 struct blkcg *blkcg = css_to_blkcg(seq_css(sf)); in blkcg_print_stat() local
952 cgroup_rstat_flush(blkcg->css.cgroup); in blkcg_print_stat()
955 hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) { in blkcg_print_stat()
1011 struct blkcg *blkcg = css_to_blkcg(css); in blkcg_css_offline() local
1014 wb_blkcg_offline(blkcg); in blkcg_css_offline()
1017 blkcg_unpin_online(blkcg); in blkcg_css_offline()
1031 void blkcg_destroy_blkgs(struct blkcg *blkcg) in blkcg_destroy_blkgs() argument
1035 spin_lock_irq(&blkcg->lock); in blkcg_destroy_blkgs()
1037 while (!hlist_empty(&blkcg->blkg_list)) { in blkcg_destroy_blkgs()
1038 struct blkcg_gq *blkg = hlist_entry(blkcg->blkg_list.first, in blkcg_destroy_blkgs()
1048 spin_unlock_irq(&blkcg->lock); in blkcg_destroy_blkgs()
1050 spin_lock_irq(&blkcg->lock); in blkcg_destroy_blkgs()
1058 spin_unlock_irq(&blkcg->lock); in blkcg_destroy_blkgs()
1063 struct blkcg *blkcg = css_to_blkcg(css); in blkcg_css_free() local
1068 list_del(&blkcg->all_blkcgs_node); in blkcg_css_free()
1071 if (blkcg->cpd[i]) in blkcg_css_free()
1072 blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]); in blkcg_css_free()
1076 kfree(blkcg); in blkcg_css_free()
1082 struct blkcg *blkcg; in blkcg_css_alloc() local
1089 blkcg = &blkcg_root; in blkcg_css_alloc()
1091 blkcg = kzalloc(sizeof(*blkcg), GFP_KERNEL); in blkcg_css_alloc()
1092 if (!blkcg) { in blkcg_css_alloc()
1116 blkcg->cpd[i] = cpd; in blkcg_css_alloc()
1117 cpd->blkcg = blkcg; in blkcg_css_alloc()
1123 spin_lock_init(&blkcg->lock); in blkcg_css_alloc()
1124 refcount_set(&blkcg->online_pin, 1); in blkcg_css_alloc()
1125 INIT_RADIX_TREE(&blkcg->blkg_tree, GFP_NOWAIT | __GFP_NOWARN); in blkcg_css_alloc()
1126 INIT_HLIST_HEAD(&blkcg->blkg_list); in blkcg_css_alloc()
1128 INIT_LIST_HEAD(&blkcg->cgwb_list); in blkcg_css_alloc()
1130 list_add_tail(&blkcg->all_blkcgs_node, &all_blkcgs); in blkcg_css_alloc()
1133 return &blkcg->css; in blkcg_css_alloc()
1137 if (blkcg->cpd[i]) in blkcg_css_alloc()
1138 blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]); in blkcg_css_alloc()
1140 if (blkcg != &blkcg_root) in blkcg_css_alloc()
1141 kfree(blkcg); in blkcg_css_alloc()
1149 struct blkcg *blkcg = css_to_blkcg(css); in blkcg_css_online() local
1150 struct blkcg *parent = blkcg_parent(blkcg); in blkcg_css_online()
1244 struct blkcg *blkcg; in blkcg_bind() local
1249 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) in blkcg_bind()
1250 if (blkcg->cpd[pol->plid]) in blkcg_bind()
1251 pol->cpd_bind_fn(blkcg->cpd[pol->plid]); in blkcg_bind()
1329 blkg->blkcg); in blkcg_activate_policy()
1347 blkg->blkcg); in blkcg_activate_policy()
1381 struct blkcg *blkcg = blkg->blkcg; in blkcg_activate_policy() local
1383 spin_lock(&blkcg->lock); in blkcg_activate_policy()
1388 spin_unlock(&blkcg->lock); in blkcg_activate_policy()
1420 struct blkcg *blkcg = blkg->blkcg; in blkcg_deactivate_policy() local
1422 spin_lock(&blkcg->lock); in blkcg_deactivate_policy()
1429 spin_unlock(&blkcg->lock); in blkcg_deactivate_policy()
1448 struct blkcg *blkcg; in blkcg_policy_register() local
1475 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) { in blkcg_policy_register()
1482 blkcg->cpd[pol->plid] = cpd; in blkcg_policy_register()
1483 cpd->blkcg = blkcg; in blkcg_policy_register()
1504 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) { in blkcg_policy_register()
1505 if (blkcg->cpd[pol->plid]) { in blkcg_policy_register()
1506 pol->cpd_free_fn(blkcg->cpd[pol->plid]); in blkcg_policy_register()
1507 blkcg->cpd[pol->plid] = NULL; in blkcg_policy_register()
1527 struct blkcg *blkcg; in blkcg_policy_unregister() local
1544 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) { in blkcg_policy_unregister()
1545 if (blkcg->cpd[pol->plid]) { in blkcg_policy_unregister()
1546 pol->cpd_free_fn(blkcg->cpd[pol->plid]); in blkcg_policy_unregister()
1547 blkcg->cpd[pol->plid] = NULL; in blkcg_policy_unregister()
1711 struct blkcg *blkcg; in blkcg_maybe_throttle_current() local
1724 blkcg = css_to_blkcg(css); in blkcg_maybe_throttle_current()
1726 blkcg = css_to_blkcg(task_css(current, io_cgrp_id)); in blkcg_maybe_throttle_current()
1728 if (!blkcg) in blkcg_maybe_throttle_current()
1730 blkg = blkg_lookup(blkcg, q); in blkcg_maybe_throttle_current()
1930 cgroup_rstat_updated(bio->bi_blkg->blkcg->css.cgroup, cpu); in blk_cgroup_bio_start()