1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Copyright (c) 2019 Facebook 4 * Copyright 2020 Google LLC. 5 */ 6 7 #ifndef _BPF_LOCAL_STORAGE_H 8 #define _BPF_LOCAL_STORAGE_H 9 10 #include <linux/bpf.h> 11 #include <linux/rculist.h> 12 #include <linux/list.h> 13 #include <linux/hash.h> 14 #include <linux/types.h> 15 #include <uapi/linux/btf.h> 16 17 #define BPF_LOCAL_STORAGE_CACHE_SIZE 16 18 19 struct bpf_local_storage_map_bucket { 20 struct hlist_head list; 21 raw_spinlock_t lock; 22 }; 23 24 /* Thp map is not the primary owner of a bpf_local_storage_elem. 25 * Instead, the container object (eg. sk->sk_bpf_storage) is. 26 * 27 * The map (bpf_local_storage_map) is for two purposes 28 * 1. Define the size of the "local storage". It is 29 * the map's value_size. 30 * 31 * 2. Maintain a list to keep track of all elems such 32 * that they can be cleaned up during the map destruction. 33 * 34 * When a bpf local storage is being looked up for a 35 * particular object, the "bpf_map" pointer is actually used 36 * as the "key" to search in the list of elem in 37 * the respective bpf_local_storage owned by the object. 38 * 39 * e.g. sk->sk_bpf_storage is the mini-map with the "bpf_map" pointer 40 * as the searching key. 41 */ 42 struct bpf_local_storage_map { 43 struct bpf_map map; 44 /* Lookup elem does not require accessing the map. 45 * 46 * Updating/Deleting requires a bucket lock to 47 * link/unlink the elem from the map. Having 48 * multiple buckets to improve contention. 49 */ 50 struct bpf_local_storage_map_bucket *buckets; 51 u32 bucket_log; 52 u16 elem_size; 53 u16 cache_idx; 54 }; 55 56 struct bpf_local_storage_data { 57 /* smap is used as the searching key when looking up 58 * from the object's bpf_local_storage. 59 * 60 * Put it in the same cacheline as the data to minimize 61 * the number of cachelines accessed during the cache hit case. 62 */ 63 struct bpf_local_storage_map __rcu *smap; 64 u8 data[] __aligned(8); 65 }; 66 67 /* Linked to bpf_local_storage and bpf_local_storage_map */ 68 struct bpf_local_storage_elem { 69 struct hlist_node map_node; /* Linked to bpf_local_storage_map */ 70 struct hlist_node snode; /* Linked to bpf_local_storage */ 71 struct bpf_local_storage __rcu *local_storage; 72 struct rcu_head rcu; 73 /* 8 bytes hole */ 74 /* The data is stored in another cacheline to minimize 75 * the number of cachelines access during a cache hit. 76 */ 77 struct bpf_local_storage_data sdata ____cacheline_aligned; 78 }; 79 80 struct bpf_local_storage { 81 struct bpf_local_storage_data __rcu *cache[BPF_LOCAL_STORAGE_CACHE_SIZE]; 82 struct hlist_head list; /* List of bpf_local_storage_elem */ 83 void *owner; /* The object that owns the above "list" of 84 * bpf_local_storage_elem. 85 */ 86 struct rcu_head rcu; 87 raw_spinlock_t lock; /* Protect adding/removing from the "list" */ 88 }; 89 90 /* U16_MAX is much more than enough for sk local storage 91 * considering a tcp_sock is ~2k. 92 */ 93 #define BPF_LOCAL_STORAGE_MAX_VALUE_SIZE \ 94 min_t(u32, \ 95 (KMALLOC_MAX_SIZE - MAX_BPF_STACK - \ 96 sizeof(struct bpf_local_storage_elem)), \ 97 (U16_MAX - sizeof(struct bpf_local_storage_elem))) 98 99 #define SELEM(_SDATA) \ 100 container_of((_SDATA), struct bpf_local_storage_elem, sdata) 101 #define SDATA(_SELEM) (&(_SELEM)->sdata) 102 103 #define BPF_LOCAL_STORAGE_CACHE_SIZE 16 104 105 struct bpf_local_storage_cache { 106 spinlock_t idx_lock; 107 u64 idx_usage_counts[BPF_LOCAL_STORAGE_CACHE_SIZE]; 108 }; 109 110 #define DEFINE_BPF_STORAGE_CACHE(name) \ 111 static struct bpf_local_storage_cache name = { \ 112 .idx_lock = __SPIN_LOCK_UNLOCKED(name.idx_lock), \ 113 } 114 115 u16 bpf_local_storage_cache_idx_get(struct bpf_local_storage_cache *cache); 116 void bpf_local_storage_cache_idx_free(struct bpf_local_storage_cache *cache, 117 u16 idx); 118 119 /* Helper functions for bpf_local_storage */ 120 int bpf_local_storage_map_alloc_check(union bpf_attr *attr); 121 122 struct bpf_local_storage_map *bpf_local_storage_map_alloc(union bpf_attr *attr); 123 124 struct bpf_local_storage_data * 125 bpf_local_storage_lookup(struct bpf_local_storage *local_storage, 126 struct bpf_local_storage_map *smap, 127 bool cacheit_lockit); 128 129 void bpf_local_storage_map_free(struct bpf_local_storage_map *smap, 130 int __percpu *busy_counter); 131 132 int bpf_local_storage_map_check_btf(const struct bpf_map *map, 133 const struct btf *btf, 134 const struct btf_type *key_type, 135 const struct btf_type *value_type); 136 137 void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage, 138 struct bpf_local_storage_elem *selem); 139 140 bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage, 141 struct bpf_local_storage_elem *selem, 142 bool uncharge_omem); 143 144 void bpf_selem_unlink(struct bpf_local_storage_elem *selem); 145 146 void bpf_selem_link_map(struct bpf_local_storage_map *smap, 147 struct bpf_local_storage_elem *selem); 148 149 void bpf_selem_unlink_map(struct bpf_local_storage_elem *selem); 150 151 struct bpf_local_storage_elem * 152 bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner, void *value, 153 bool charge_mem); 154 155 int 156 bpf_local_storage_alloc(void *owner, 157 struct bpf_local_storage_map *smap, 158 struct bpf_local_storage_elem *first_selem); 159 160 struct bpf_local_storage_data * 161 bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap, 162 void *value, u64 map_flags); 163 164 #endif /* _BPF_LOCAL_STORAGE_H */ 165