Lines Matching refs:lru
21 static inline bool list_lru_memcg_aware(struct list_lru *lru) in list_lru_memcg_aware() argument
23 return lru->memcg_aware; in list_lru_memcg_aware()
26 static void list_lru_register(struct list_lru *lru) in list_lru_register() argument
28 if (!list_lru_memcg_aware(lru)) in list_lru_register()
32 list_add(&lru->list, &memcg_list_lrus); in list_lru_register()
36 static void list_lru_unregister(struct list_lru *lru) in list_lru_unregister() argument
38 if (!list_lru_memcg_aware(lru)) in list_lru_unregister()
42 list_del(&lru->list); in list_lru_unregister()
46 static int lru_shrinker_id(struct list_lru *lru) in lru_shrinker_id() argument
48 return lru->shrinker_id; in lru_shrinker_id()
62 return memcg_lrus->lru[idx]; in list_lru_from_memcg_idx()
63 return &nlru->lru; in list_lru_from_memcg_idx()
70 struct list_lru_one *l = &nlru->lru; in list_lru_from_kmem()
87 static void list_lru_register(struct list_lru *lru) in list_lru_register() argument
91 static void list_lru_unregister(struct list_lru *lru) in list_lru_unregister() argument
95 static int lru_shrinker_id(struct list_lru *lru) in lru_shrinker_id() argument
100 static inline bool list_lru_memcg_aware(struct list_lru *lru) in list_lru_memcg_aware() argument
108 return &nlru->lru; in list_lru_from_memcg_idx()
117 return &nlru->lru; in list_lru_from_kmem()
121 bool list_lru_add(struct list_lru *lru, struct list_head *item) in list_lru_add() argument
124 struct list_lru_node *nlru = &lru->node[nid]; in list_lru_add()
135 lru_shrinker_id(lru)); in list_lru_add()
145 bool list_lru_del(struct list_lru *lru, struct list_head *item) in list_lru_del() argument
148 struct list_lru_node *nlru = &lru->node[nid]; in list_lru_del()
180 unsigned long list_lru_count_one(struct list_lru *lru, in list_lru_count_one() argument
183 struct list_lru_node *nlru = &lru->node[nid]; in list_lru_count_one()
199 unsigned long list_lru_count_node(struct list_lru *lru, int nid) in list_lru_count_node() argument
203 nlru = &lru->node[nid]; in list_lru_count_node()
267 list_lru_walk_one(struct list_lru *lru, int nid, struct mem_cgroup *memcg, in list_lru_walk_one() argument
271 struct list_lru_node *nlru = &lru->node[nid]; in list_lru_walk_one()
283 list_lru_walk_one_irq(struct list_lru *lru, int nid, struct mem_cgroup *memcg, in list_lru_walk_one_irq() argument
287 struct list_lru_node *nlru = &lru->node[nid]; in list_lru_walk_one_irq()
297 unsigned long list_lru_walk_node(struct list_lru *lru, int nid, in list_lru_walk_node() argument
304 isolated += list_lru_walk_one(lru, nid, NULL, isolate, cb_arg, in list_lru_walk_node()
306 if (*nr_to_walk > 0 && list_lru_memcg_aware(lru)) { in list_lru_walk_node()
308 struct list_lru_node *nlru = &lru->node[nid]; in list_lru_walk_node()
337 kfree(memcg_lrus->lru[i]); in __memcg_destroy_list_lru_node()
353 memcg_lrus->lru[i] = l; in __memcg_init_list_lru_node()
366 memcg_lrus = kvmalloc(struct_size(memcg_lrus, lru, size), GFP_KERNEL); in memcg_init_list_lru_node()
400 new = kvmalloc(struct_size(new, lru, new_size), GFP_KERNEL); in memcg_update_list_lru_node()
409 memcpy(&new->lru, &old->lru, flex_array_size(new, lru, old_size)); in memcg_update_list_lru_node()
427 static int memcg_init_list_lru(struct list_lru *lru, bool memcg_aware) in memcg_init_list_lru() argument
431 lru->memcg_aware = memcg_aware; in memcg_init_list_lru()
437 if (memcg_init_list_lru_node(&lru->node[i])) in memcg_init_list_lru()
443 if (!lru->node[i].memcg_lrus) in memcg_init_list_lru()
445 memcg_destroy_list_lru_node(&lru->node[i]); in memcg_init_list_lru()
450 static void memcg_destroy_list_lru(struct list_lru *lru) in memcg_destroy_list_lru() argument
454 if (!list_lru_memcg_aware(lru)) in memcg_destroy_list_lru()
458 memcg_destroy_list_lru_node(&lru->node[i]); in memcg_destroy_list_lru()
461 static int memcg_update_list_lru(struct list_lru *lru, in memcg_update_list_lru() argument
467 if (memcg_update_list_lru_node(&lru->node[i], in memcg_update_list_lru()
474 if (!lru->node[i].memcg_lrus) in memcg_update_list_lru()
477 memcg_cancel_update_list_lru_node(&lru->node[i], in memcg_update_list_lru()
483 static void memcg_cancel_update_list_lru(struct list_lru *lru, in memcg_cancel_update_list_lru() argument
489 memcg_cancel_update_list_lru_node(&lru->node[i], in memcg_cancel_update_list_lru()
496 struct list_lru *lru; in memcg_update_all_list_lrus() local
500 list_for_each_entry(lru, &memcg_list_lrus, list) { in memcg_update_all_list_lrus()
501 ret = memcg_update_list_lru(lru, old_size, new_size); in memcg_update_all_list_lrus()
509 list_for_each_entry_continue_reverse(lru, &memcg_list_lrus, list) in memcg_update_all_list_lrus()
510 memcg_cancel_update_list_lru(lru, old_size, new_size); in memcg_update_all_list_lrus()
514 static void memcg_drain_list_lru_node(struct list_lru *lru, int nid, in memcg_drain_list_lru_node() argument
517 struct list_lru_node *nlru = &lru->node[nid]; in memcg_drain_list_lru_node()
534 set_shrinker_bit(dst_memcg, nid, lru_shrinker_id(lru)); in memcg_drain_list_lru_node()
541 static void memcg_drain_list_lru(struct list_lru *lru, in memcg_drain_list_lru() argument
547 memcg_drain_list_lru_node(lru, i, src_idx, dst_memcg); in memcg_drain_list_lru()
552 struct list_lru *lru; in memcg_drain_all_list_lrus() local
555 list_for_each_entry(lru, &memcg_list_lrus, list) in memcg_drain_all_list_lrus()
556 memcg_drain_list_lru(lru, src_idx, dst_memcg); in memcg_drain_all_list_lrus()
560 static int memcg_init_list_lru(struct list_lru *lru, bool memcg_aware) in memcg_init_list_lru() argument
565 static void memcg_destroy_list_lru(struct list_lru *lru) in memcg_destroy_list_lru() argument
570 int __list_lru_init(struct list_lru *lru, bool memcg_aware, in __list_lru_init() argument
578 lru->shrinker_id = shrinker->id; in __list_lru_init()
580 lru->shrinker_id = -1; in __list_lru_init()
584 lru->node = kcalloc(nr_node_ids, sizeof(*lru->node), GFP_KERNEL); in __list_lru_init()
585 if (!lru->node) in __list_lru_init()
589 spin_lock_init(&lru->node[i].lock); in __list_lru_init()
591 lockdep_set_class(&lru->node[i].lock, key); in __list_lru_init()
592 init_one_lru(&lru->node[i].lru); in __list_lru_init()
595 err = memcg_init_list_lru(lru, memcg_aware); in __list_lru_init()
597 kfree(lru->node); in __list_lru_init()
599 lru->node = NULL; in __list_lru_init()
603 list_lru_register(lru); in __list_lru_init()
610 void list_lru_destroy(struct list_lru *lru) in list_lru_destroy() argument
613 if (!lru->node) in list_lru_destroy()
618 list_lru_unregister(lru); in list_lru_destroy()
620 memcg_destroy_list_lru(lru); in list_lru_destroy()
621 kfree(lru->node); in list_lru_destroy()
622 lru->node = NULL; in list_lru_destroy()
625 lru->shrinker_id = -1; in list_lru_destroy()