Lines Matching refs:aux
84 struct bpf_prog_aux *aux; in bpf_prog_alloc_no_stats() local
92 aux = kzalloc(sizeof(*aux), GFP_KERNEL_ACCOUNT | gfp_extra_flags); in bpf_prog_alloc_no_stats()
93 if (aux == NULL) { in bpf_prog_alloc_no_stats()
100 kfree(aux); in bpf_prog_alloc_no_stats()
105 fp->aux = aux; in bpf_prog_alloc_no_stats()
106 fp->aux->prog = fp; in bpf_prog_alloc_no_stats()
109 INIT_LIST_HEAD_RCU(&fp->aux->ksym.lnode); in bpf_prog_alloc_no_stats()
110 mutex_init(&fp->aux->used_maps_mutex); in bpf_prog_alloc_no_stats()
111 mutex_init(&fp->aux->dst_mutex); in bpf_prog_alloc_no_stats()
129 kfree(prog->aux); in bpf_prog_alloc()
146 if (!prog->aux->nr_linfo || !prog->jit_requested) in bpf_prog_alloc_jited_linfo()
149 prog->aux->jited_linfo = kvcalloc(prog->aux->nr_linfo, in bpf_prog_alloc_jited_linfo()
150 sizeof(*prog->aux->jited_linfo), in bpf_prog_alloc_jited_linfo()
152 if (!prog->aux->jited_linfo) in bpf_prog_alloc_jited_linfo()
160 if (prog->aux->jited_linfo && in bpf_prog_jit_attempt_done()
161 (!prog->jited || !prog->aux->jited_linfo[0])) { in bpf_prog_jit_attempt_done()
162 kvfree(prog->aux->jited_linfo); in bpf_prog_jit_attempt_done()
163 prog->aux->jited_linfo = NULL; in bpf_prog_jit_attempt_done()
166 kfree(prog->aux->kfunc_tab); in bpf_prog_jit_attempt_done()
167 prog->aux->kfunc_tab = NULL; in bpf_prog_jit_attempt_done()
201 if (!prog->aux->jited_linfo) in bpf_prog_fill_jited_linfo()
205 linfo_idx = prog->aux->linfo_idx; in bpf_prog_fill_jited_linfo()
206 linfo = &prog->aux->linfo[linfo_idx]; in bpf_prog_fill_jited_linfo()
210 jited_linfo = &prog->aux->jited_linfo[linfo_idx]; in bpf_prog_fill_jited_linfo()
213 nr_linfo = prog->aux->nr_linfo - linfo_idx; in bpf_prog_fill_jited_linfo()
239 fp->aux->prog = fp; in bpf_prog_realloc()
244 fp_old->aux = NULL; in bpf_prog_realloc()
255 if (fp->aux) { in __bpf_prog_free()
256 mutex_destroy(&fp->aux->used_maps_mutex); in __bpf_prog_free()
257 mutex_destroy(&fp->aux->dst_mutex); in __bpf_prog_free()
258 kfree(fp->aux->poke_tab); in __bpf_prog_free()
259 kfree(fp->aux); in __bpf_prog_free()
427 nr_linfo = prog->aux->nr_linfo; in bpf_adj_linfo()
431 linfo = prog->aux->linfo; in bpf_adj_linfo()
519 for (i = 0; i < fp->aux->func_cnt; i++) in bpf_prog_kallsyms_del_subprogs()
520 bpf_prog_kallsyms_del(fp->aux->func[i]); in bpf_prog_kallsyms_del_subprogs()
545 prog->aux->ksym.start = (unsigned long) prog->bpf_func; in bpf_prog_ksym_set_addr()
546 prog->aux->ksym.end = addr + hdr->pages * PAGE_SIZE; in bpf_prog_ksym_set_addr()
552 char *sym = prog->aux->ksym.name; in bpf_prog_ksym_set_name()
566 sizeof(prog->aux->name) > KSYM_NAME_LEN); in bpf_prog_ksym_set_name()
572 if (prog->aux->func_info_cnt) { in bpf_prog_ksym_set_name()
573 type = btf_type_by_id(prog->aux->btf, in bpf_prog_ksym_set_name()
574 prog->aux->func_info[prog->aux->func_idx].type_id); in bpf_prog_ksym_set_name()
575 func_name = btf_name_by_offset(prog->aux->btf, type->name_off); in bpf_prog_ksym_set_name()
580 if (prog->aux->name[0]) in bpf_prog_ksym_set_name()
581 snprintf(sym, (size_t)(end - sym), "_%s", prog->aux->name); in bpf_prog_ksym_set_name()
653 return list_empty(&fp->aux->ksym.lnode) || in bpf_prog_kallsyms_verify_off()
654 fp->aux->ksym.lnode.prev == LIST_POISON2; in bpf_prog_kallsyms_verify_off()
665 fp->aux->ksym.prog = true; in bpf_prog_kallsyms_add()
667 bpf_ksym_add(&fp->aux->ksym); in bpf_prog_kallsyms_add()
675 bpf_ksym_del(&fp->aux->ksym); in bpf_prog_kallsyms_del()
740 if (!prog->aux->num_exentries) in search_bpf_extables()
743 e = search_extable(prog->aux->extable, prog->aux->num_exentries, addr); in search_bpf_extables()
780 struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab; in bpf_jit_add_poke_descriptor()
782 u32 slot = prog->aux->size_poke_tab; in bpf_jit_add_poke_descriptor()
805 prog->aux->size_poke_tab = size; in bpf_jit_add_poke_descriptor()
806 prog->aux->poke_tab = tab; in bpf_jit_add_poke_descriptor()
945 else if (prog->aux->func && in bpf_jit_get_func_addr()
946 off >= 0 && off < prog->aux->func_cnt) in bpf_jit_get_func_addr()
947 addr = (u8 *)prog->aux->func[off]->bpf_func; in bpf_jit_get_func_addr()
964 const struct bpf_insn *aux, in bpf_jit_blind_insn() argument
1073 *to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ aux[1].imm); in bpf_jit_blind_insn()
1076 *to++ = BPF_ALU64_REG(BPF_MOV, aux[0].dst_reg, BPF_REG_AX); in bpf_jit_blind_insn()
1079 *to++ = BPF_ALU32_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ aux[0].imm); in bpf_jit_blind_insn()
1083 *to++ = BPF_ALU64_REG(BPF_OR, aux[0].dst_reg, BPF_REG_AX); in bpf_jit_blind_insn()
1126 fp->aux = NULL; in bpf_prog_clone_free()
1137 fp->aux->prog = fp; in bpf_jit_prog_release_other()
1143 struct bpf_insn insn_buff[16], aux[2]; in bpf_jit_blind_constants() local
1166 memcpy(aux, insn, sizeof(aux)); in bpf_jit_blind_constants()
1168 rewritten = bpf_jit_blind_insn(insn, aux, insn_buff, in bpf_jit_blind_constants()
1169 clone->aux->verifier_zext); in bpf_jit_blind_constants()
1839 spin_lock(&array->aux->owner.lock); in bpf_prog_array_compatible()
1841 if (!array->aux->owner.type) { in bpf_prog_array_compatible()
1845 array->aux->owner.type = fp->type; in bpf_prog_array_compatible()
1846 array->aux->owner.jited = fp->jited; in bpf_prog_array_compatible()
1849 ret = array->aux->owner.type == fp->type && in bpf_prog_array_compatible()
1850 array->aux->owner.jited == fp->jited; in bpf_prog_array_compatible()
1852 spin_unlock(&array->aux->owner.lock); in bpf_prog_array_compatible()
1858 struct bpf_prog_aux *aux = fp->aux; in bpf_check_tail_call() local
1861 mutex_lock(&aux->used_maps_mutex); in bpf_check_tail_call()
1862 for (i = 0; i < aux->used_map_cnt; i++) { in bpf_check_tail_call()
1863 struct bpf_map *map = aux->used_maps[i]; in bpf_check_tail_call()
1877 mutex_unlock(&aux->used_maps_mutex); in bpf_check_tail_call()
1884 u32 stack_depth = max_t(u32, fp->aux->stack_depth, 1); in bpf_prog_select_func()
1925 if (!bpf_prog_is_dev_bound(fp->aux)) { in bpf_prog_select_runtime()
2032 prog_ids[i] = item->prog->aux->id; in bpf_prog_array_copy_core()
2226 void __bpf_free_used_maps(struct bpf_prog_aux *aux, in __bpf_free_used_maps() argument
2235 map->ops->map_poke_untrack(map, aux); in __bpf_free_used_maps()
2240 static void bpf_free_used_maps(struct bpf_prog_aux *aux) in bpf_free_used_maps() argument
2242 __bpf_free_used_maps(aux, aux->used_maps, aux->used_map_cnt); in bpf_free_used_maps()
2243 kfree(aux->used_maps); in bpf_free_used_maps()
2246 void __bpf_free_used_btfs(struct bpf_prog_aux *aux, in __bpf_free_used_btfs() argument
2262 static void bpf_free_used_btfs(struct bpf_prog_aux *aux) in bpf_free_used_btfs() argument
2264 __bpf_free_used_btfs(aux, aux->used_btfs, aux->used_btf_cnt); in bpf_free_used_btfs()
2265 kfree(aux->used_btfs); in bpf_free_used_btfs()
2270 struct bpf_prog_aux *aux; in bpf_prog_free_deferred() local
2273 aux = container_of(work, struct bpf_prog_aux, work); in bpf_prog_free_deferred()
2275 bpf_free_kfunc_btf_tab(aux->kfunc_btf_tab); in bpf_prog_free_deferred()
2277 bpf_free_used_maps(aux); in bpf_prog_free_deferred()
2278 bpf_free_used_btfs(aux); in bpf_prog_free_deferred()
2279 if (bpf_prog_is_dev_bound(aux)) in bpf_prog_free_deferred()
2280 bpf_prog_offload_destroy(aux->prog); in bpf_prog_free_deferred()
2282 if (aux->prog->has_callchain_buf) in bpf_prog_free_deferred()
2285 if (aux->dst_trampoline) in bpf_prog_free_deferred()
2286 bpf_trampoline_put(aux->dst_trampoline); in bpf_prog_free_deferred()
2287 for (i = 0; i < aux->func_cnt; i++) { in bpf_prog_free_deferred()
2292 aux->func[i]->aux->poke_tab = NULL; in bpf_prog_free_deferred()
2293 bpf_jit_free(aux->func[i]); in bpf_prog_free_deferred()
2295 if (aux->func_cnt) { in bpf_prog_free_deferred()
2296 kfree(aux->func); in bpf_prog_free_deferred()
2297 bpf_prog_unlock_free(aux->prog); in bpf_prog_free_deferred()
2299 bpf_jit_free(aux->prog); in bpf_prog_free_deferred()
2306 struct bpf_prog_aux *aux = fp->aux; in bpf_prog_free() local
2308 if (aux->dst_prog) in bpf_prog_free()
2309 bpf_prog_put(aux->dst_prog); in bpf_prog_free()
2310 INIT_WORK(&aux->work, bpf_prog_free_deferred); in bpf_prog_free()
2311 schedule_work(&aux->work); in bpf_prog_free()