Lines Matching refs:storage
77 struct bpf_cgroup_storage *storage; in cgroup_storage_lookup() local
79 storage = container_of(node, struct bpf_cgroup_storage, node); in cgroup_storage_lookup()
81 switch (bpf_cgroup_storage_key_cmp(map, key, &storage->key)) { in cgroup_storage_lookup()
91 return storage; in cgroup_storage_lookup()
102 struct bpf_cgroup_storage *storage) in cgroup_storage_insert() argument
113 switch (bpf_cgroup_storage_key_cmp(map, &storage->key, &this->key)) { in cgroup_storage_insert()
125 rb_link_node(&storage->node, parent, new); in cgroup_storage_insert()
126 rb_insert_color(&storage->node, root); in cgroup_storage_insert()
134 struct bpf_cgroup_storage *storage; in cgroup_storage_lookup_elem() local
136 storage = cgroup_storage_lookup(map, key, false); in cgroup_storage_lookup_elem()
137 if (!storage) in cgroup_storage_lookup_elem()
140 return &READ_ONCE(storage->buf)->data[0]; in cgroup_storage_lookup_elem()
146 struct bpf_cgroup_storage *storage; in cgroup_storage_update_elem() local
156 storage = cgroup_storage_lookup((struct bpf_cgroup_storage_map *)map, in cgroup_storage_update_elem()
158 if (!storage) in cgroup_storage_update_elem()
162 copy_map_value_locked(map, storage->buf->data, value, false); in cgroup_storage_update_elem()
176 new = xchg(&storage->buf, new); in cgroup_storage_update_elem()
186 struct bpf_cgroup_storage *storage; in bpf_percpu_cgroup_storage_copy() local
191 storage = cgroup_storage_lookup(map, key, false); in bpf_percpu_cgroup_storage_copy()
192 if (!storage) { in bpf_percpu_cgroup_storage_copy()
204 per_cpu_ptr(storage->percpu_buf, cpu), size); in bpf_percpu_cgroup_storage_copy()
215 struct bpf_cgroup_storage *storage; in bpf_percpu_cgroup_storage_update() local
223 storage = cgroup_storage_lookup(map, key, false); in bpf_percpu_cgroup_storage_update()
224 if (!storage) { in bpf_percpu_cgroup_storage_update()
237 bpf_long_memcpy(per_cpu_ptr(storage->percpu_buf, cpu), in bpf_percpu_cgroup_storage_update()
249 struct bpf_cgroup_storage *storage; in cgroup_storage_get_next_key() local
257 storage = cgroup_storage_lookup(map, key, true); in cgroup_storage_get_next_key()
258 if (!storage) in cgroup_storage_get_next_key()
261 storage = list_next_entry(storage, list_map); in cgroup_storage_get_next_key()
262 if (!storage) in cgroup_storage_get_next_key()
265 storage = list_first_entry(&map->list, in cgroup_storage_get_next_key()
273 *next = storage->key; in cgroup_storage_get_next_key()
276 *next = storage->key.cgroup_inode_id; in cgroup_storage_get_next_key()
335 struct bpf_cgroup_storage *storage, *stmp; in cgroup_storage_map_free() local
339 list_for_each_entry_safe(storage, stmp, storages, list_map) { in cgroup_storage_map_free()
340 bpf_cgroup_storage_unlink(storage); in cgroup_storage_map_free()
341 bpf_cgroup_storage_free(storage); in cgroup_storage_map_free()
419 struct bpf_cgroup_storage *storage; in cgroup_storage_seq_show_elem() local
423 storage = cgroup_storage_lookup(map_to_storage(map), key, false); in cgroup_storage_seq_show_elem()
424 if (!storage) { in cgroup_storage_seq_show_elem()
434 &READ_ONCE(storage->buf)->data[0], m); in cgroup_storage_seq_show_elem()
441 per_cpu_ptr(storage->percpu_buf, cpu), in cgroup_storage_seq_show_elem()
497 struct bpf_cgroup_storage *storage; in bpf_cgroup_storage_alloc() local
508 storage = bpf_map_kmalloc_node(map, sizeof(struct bpf_cgroup_storage), in bpf_cgroup_storage_alloc()
510 if (!storage) in bpf_cgroup_storage_alloc()
514 storage->buf = bpf_map_kmalloc_node(map, size, gfp, in bpf_cgroup_storage_alloc()
516 if (!storage->buf) in bpf_cgroup_storage_alloc()
518 check_and_init_map_value(map, storage->buf->data); in bpf_cgroup_storage_alloc()
520 storage->percpu_buf = bpf_map_alloc_percpu(map, size, 8, gfp); in bpf_cgroup_storage_alloc()
521 if (!storage->percpu_buf) in bpf_cgroup_storage_alloc()
525 storage->map = (struct bpf_cgroup_storage_map *)map; in bpf_cgroup_storage_alloc()
527 return storage; in bpf_cgroup_storage_alloc()
530 kfree(storage); in bpf_cgroup_storage_alloc()
536 struct bpf_cgroup_storage *storage = in free_shared_cgroup_storage_rcu() local
539 kfree(storage->buf); in free_shared_cgroup_storage_rcu()
540 kfree(storage); in free_shared_cgroup_storage_rcu()
545 struct bpf_cgroup_storage *storage = in free_percpu_cgroup_storage_rcu() local
548 free_percpu(storage->percpu_buf); in free_percpu_cgroup_storage_rcu()
549 kfree(storage); in free_percpu_cgroup_storage_rcu()
552 void bpf_cgroup_storage_free(struct bpf_cgroup_storage *storage) in bpf_cgroup_storage_free() argument
557 if (!storage) in bpf_cgroup_storage_free()
560 map = &storage->map->map; in bpf_cgroup_storage_free()
563 call_rcu(&storage->rcu, free_shared_cgroup_storage_rcu); in bpf_cgroup_storage_free()
565 call_rcu(&storage->rcu, free_percpu_cgroup_storage_rcu); in bpf_cgroup_storage_free()
568 void bpf_cgroup_storage_link(struct bpf_cgroup_storage *storage, in bpf_cgroup_storage_link() argument
574 if (!storage) in bpf_cgroup_storage_link()
577 storage->key.attach_type = type; in bpf_cgroup_storage_link()
578 storage->key.cgroup_inode_id = cgroup_id(cgroup); in bpf_cgroup_storage_link()
580 map = storage->map; in bpf_cgroup_storage_link()
583 WARN_ON(cgroup_storage_insert(map, storage)); in bpf_cgroup_storage_link()
584 list_add(&storage->list_map, &map->list); in bpf_cgroup_storage_link()
585 list_add(&storage->list_cg, &cgroup->bpf.storages); in bpf_cgroup_storage_link()
589 void bpf_cgroup_storage_unlink(struct bpf_cgroup_storage *storage) in bpf_cgroup_storage_unlink() argument
594 if (!storage) in bpf_cgroup_storage_unlink()
597 map = storage->map; in bpf_cgroup_storage_unlink()
601 rb_erase(&storage->node, root); in bpf_cgroup_storage_unlink()
603 list_del(&storage->list_map); in bpf_cgroup_storage_unlink()
604 list_del(&storage->list_cg); in bpf_cgroup_storage_unlink()