Lines Matching refs:selem
19 struct bpf_local_storage_elem *selem) in select_bucket() argument
21 return &smap->buckets[hash_ptr(selem, smap->bucket_log)]; in select_bucket()
51 static bool selem_linked_to_storage(const struct bpf_local_storage_elem *selem) in selem_linked_to_storage() argument
53 return !hlist_unhashed(&selem->snode); in selem_linked_to_storage()
56 static bool selem_linked_to_map(const struct bpf_local_storage_elem *selem) in selem_linked_to_map() argument
58 return !hlist_unhashed(&selem->map_node); in selem_linked_to_map()
65 struct bpf_local_storage_elem *selem; in bpf_selem_alloc() local
70 selem = bpf_map_kzalloc(&smap->map, smap->elem_size, in bpf_selem_alloc()
72 if (selem) { in bpf_selem_alloc()
74 memcpy(SDATA(selem)->data, value, smap->map.value_size); in bpf_selem_alloc()
75 return selem; in bpf_selem_alloc()
89 struct bpf_local_storage_elem *selem, in bpf_selem_unlink_storage_nolock() argument
96 smap = rcu_dereference(SDATA(selem)->smap); in bpf_selem_unlink_storage_nolock()
106 free_local_storage = hlist_is_singular_node(&selem->snode, in bpf_selem_unlink_storage_nolock()
129 hlist_del_init_rcu(&selem->snode); in bpf_selem_unlink_storage_nolock()
131 SDATA(selem)) in bpf_selem_unlink_storage_nolock()
134 kfree_rcu(selem, rcu); in bpf_selem_unlink_storage_nolock()
139 static void __bpf_selem_unlink_storage(struct bpf_local_storage_elem *selem) in __bpf_selem_unlink_storage() argument
145 if (unlikely(!selem_linked_to_storage(selem))) in __bpf_selem_unlink_storage()
149 local_storage = rcu_dereference(selem->local_storage); in __bpf_selem_unlink_storage()
151 if (likely(selem_linked_to_storage(selem))) in __bpf_selem_unlink_storage()
153 local_storage, selem, true); in __bpf_selem_unlink_storage()
161 struct bpf_local_storage_elem *selem) in bpf_selem_link_storage_nolock() argument
163 RCU_INIT_POINTER(selem->local_storage, local_storage); in bpf_selem_link_storage_nolock()
164 hlist_add_head_rcu(&selem->snode, &local_storage->list); in bpf_selem_link_storage_nolock()
167 void bpf_selem_unlink_map(struct bpf_local_storage_elem *selem) in bpf_selem_unlink_map() argument
173 if (unlikely(!selem_linked_to_map(selem))) in bpf_selem_unlink_map()
177 smap = rcu_dereference(SDATA(selem)->smap); in bpf_selem_unlink_map()
178 b = select_bucket(smap, selem); in bpf_selem_unlink_map()
180 if (likely(selem_linked_to_map(selem))) in bpf_selem_unlink_map()
181 hlist_del_init_rcu(&selem->map_node); in bpf_selem_unlink_map()
186 struct bpf_local_storage_elem *selem) in bpf_selem_link_map() argument
188 struct bpf_local_storage_map_bucket *b = select_bucket(smap, selem); in bpf_selem_link_map()
192 RCU_INIT_POINTER(SDATA(selem)->smap, smap); in bpf_selem_link_map()
193 hlist_add_head_rcu(&selem->map_node, &b->list); in bpf_selem_link_map()
197 void bpf_selem_unlink(struct bpf_local_storage_elem *selem) in bpf_selem_unlink() argument
203 bpf_selem_unlink_map(selem); in bpf_selem_unlink()
204 __bpf_selem_unlink_storage(selem); in bpf_selem_unlink()
213 struct bpf_local_storage_elem *selem; in bpf_local_storage_lookup() local
221 hlist_for_each_entry_rcu(selem, &local_storage->list, snode) in bpf_local_storage_lookup()
222 if (rcu_access_pointer(SDATA(selem)->smap) == smap) in bpf_local_storage_lookup()
225 if (!selem) in bpf_local_storage_lookup()
228 sdata = SDATA(selem); in bpf_local_storage_lookup()
238 if (selem_linked_to_storage(selem)) in bpf_local_storage_lookup()
333 struct bpf_local_storage_elem *selem; in bpf_local_storage_update() local
352 selem = bpf_selem_alloc(smap, owner, value, true); in bpf_local_storage_update()
353 if (!selem) in bpf_local_storage_update()
356 err = bpf_local_storage_alloc(owner, smap, selem); in bpf_local_storage_update()
358 kfree(selem); in bpf_local_storage_update()
363 return SDATA(selem); in bpf_local_storage_update()
404 selem = SELEM(old_sdata); in bpf_local_storage_update()
417 selem = bpf_selem_alloc(smap, owner, value, !old_sdata); in bpf_local_storage_update()
418 if (!selem) { in bpf_local_storage_update()
424 bpf_selem_link_map(smap, selem); in bpf_local_storage_update()
427 bpf_selem_link_storage_nolock(local_storage, selem); in bpf_local_storage_update()
438 return SDATA(selem); in bpf_local_storage_update()
480 struct bpf_local_storage_elem *selem; in bpf_local_storage_map_free() local
504 while ((selem = hlist_entry_safe( in bpf_local_storage_map_free()
511 bpf_selem_unlink(selem); in bpf_local_storage_map_free()