Lines Matching refs:gen
45 static int blob_fd_array_off(struct bpf_gen *gen, int index) in blob_fd_array_off() argument
47 return gen->fd_array + index * sizeof(int); in blob_fd_array_off()
50 static int realloc_insn_buf(struct bpf_gen *gen, __u32 size) in realloc_insn_buf() argument
52 size_t off = gen->insn_cur - gen->insn_start; in realloc_insn_buf()
55 if (gen->error) in realloc_insn_buf()
56 return gen->error; in realloc_insn_buf()
58 gen->error = -ERANGE; in realloc_insn_buf()
61 insn_start = realloc(gen->insn_start, off + size); in realloc_insn_buf()
63 gen->error = -ENOMEM; in realloc_insn_buf()
64 free(gen->insn_start); in realloc_insn_buf()
65 gen->insn_start = NULL; in realloc_insn_buf()
68 gen->insn_start = insn_start; in realloc_insn_buf()
69 gen->insn_cur = insn_start + off; in realloc_insn_buf()
73 static int realloc_data_buf(struct bpf_gen *gen, __u32 size) in realloc_data_buf() argument
75 size_t off = gen->data_cur - gen->data_start; in realloc_data_buf()
78 if (gen->error) in realloc_data_buf()
79 return gen->error; in realloc_data_buf()
81 gen->error = -ERANGE; in realloc_data_buf()
84 data_start = realloc(gen->data_start, off + size); in realloc_data_buf()
86 gen->error = -ENOMEM; in realloc_data_buf()
87 free(gen->data_start); in realloc_data_buf()
88 gen->data_start = NULL; in realloc_data_buf()
91 gen->data_start = data_start; in realloc_data_buf()
92 gen->data_cur = data_start + off; in realloc_data_buf()
96 static void emit(struct bpf_gen *gen, struct bpf_insn insn) in emit() argument
98 if (realloc_insn_buf(gen, sizeof(insn))) in emit()
100 memcpy(gen->insn_cur, &insn, sizeof(insn)); in emit()
101 gen->insn_cur += sizeof(insn); in emit()
104 static void emit2(struct bpf_gen *gen, struct bpf_insn insn1, struct bpf_insn insn2) in emit2() argument
106 emit(gen, insn1); in emit2()
107 emit(gen, insn2); in emit2()
110 static int add_data(struct bpf_gen *gen, const void *data, __u32 size);
111 static void emit_sys_close_blob(struct bpf_gen *gen, int blob_off);
113 void bpf_gen__init(struct bpf_gen *gen, int log_level, int nr_progs, int nr_maps) in bpf_gen__init() argument
118 gen->fd_array = add_data(gen, NULL, MAX_FD_ARRAY_SZ * sizeof(int)); in bpf_gen__init()
119 gen->log_level = log_level; in bpf_gen__init()
121 emit(gen, BPF_MOV64_REG(BPF_REG_6, BPF_REG_1)); in bpf_gen__init()
124 emit(gen, BPF_MOV64_REG(BPF_REG_1, BPF_REG_10)); in bpf_gen__init()
125 emit(gen, BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -stack_sz)); in bpf_gen__init()
126 emit(gen, BPF_MOV64_IMM(BPF_REG_2, stack_sz)); in bpf_gen__init()
127 emit(gen, BPF_MOV64_IMM(BPF_REG_3, 0)); in bpf_gen__init()
128 emit(gen, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel)); in bpf_gen__init()
133 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, in bpf_gen__init()
139 nr_maps * (6 + (gen->log_level ? 6 : 0)))); in bpf_gen__init()
142 gen->cleanup_label = gen->insn_cur - gen->insn_start; in bpf_gen__init()
145 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_10, -stack_sz + i)); in bpf_gen__init()
146 emit(gen, BPF_JMP_IMM(BPF_JSLE, BPF_REG_1, 0, 1)); in bpf_gen__init()
147 emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_close)); in bpf_gen__init()
150 emit_sys_close_blob(gen, blob_fd_array_off(gen, i)); in bpf_gen__init()
152 emit(gen, BPF_MOV64_REG(BPF_REG_0, BPF_REG_7)); in bpf_gen__init()
153 emit(gen, BPF_EXIT_INSN()); in bpf_gen__init()
156 static int add_data(struct bpf_gen *gen, const void *data, __u32 size) in add_data() argument
162 if (realloc_data_buf(gen, size8)) in add_data()
164 prev = gen->data_cur; in add_data()
166 memcpy(gen->data_cur, data, size); in add_data()
167 memcpy(gen->data_cur + size, &zero, size8 - size); in add_data()
169 memset(gen->data_cur, 0, size8); in add_data()
171 gen->data_cur += size8; in add_data()
172 return prev - gen->data_start; in add_data()
178 static int add_map_fd(struct bpf_gen *gen) in add_map_fd() argument
180 if (gen->nr_maps == MAX_USED_MAPS) { in add_map_fd()
182 gen->error = -E2BIG; in add_map_fd()
185 return gen->nr_maps++; in add_map_fd()
188 static int add_kfunc_btf_fd(struct bpf_gen *gen) in add_kfunc_btf_fd() argument
192 if (gen->nr_fd_array == MAX_KFUNC_DESCS) { in add_kfunc_btf_fd()
193 cur = add_data(gen, NULL, sizeof(int)); in add_kfunc_btf_fd()
194 return (cur - gen->fd_array) / sizeof(int); in add_kfunc_btf_fd()
196 return MAX_USED_MAPS + gen->nr_fd_array++; in add_kfunc_btf_fd()
211 static void emit_rel_store(struct bpf_gen *gen, int off, int data) in emit_rel_store() argument
213 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE, in emit_rel_store()
215 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, in emit_rel_store()
217 emit(gen, BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, 0)); in emit_rel_store()
220 static void move_blob2blob(struct bpf_gen *gen, int off, int size, int blob_off) in move_blob2blob() argument
222 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_2, BPF_PSEUDO_MAP_IDX_VALUE, in move_blob2blob()
224 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_2, 0)); in move_blob2blob()
225 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, in move_blob2blob()
227 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0)); in move_blob2blob()
230 static void move_blob2ctx(struct bpf_gen *gen, int ctx_off, int size, int blob_off) in move_blob2ctx() argument
232 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, in move_blob2ctx()
234 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_1, 0)); in move_blob2ctx()
235 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_6, BPF_REG_0, ctx_off)); in move_blob2ctx()
238 static void move_ctx2blob(struct bpf_gen *gen, int off, int size, int ctx_off, in move_ctx2blob() argument
241 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_6, ctx_off)); in move_ctx2blob()
246 emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 3)); in move_ctx2blob()
247 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, in move_ctx2blob()
249 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0)); in move_ctx2blob()
252 static void move_stack2blob(struct bpf_gen *gen, int off, int size, int stack_off) in move_stack2blob() argument
254 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_10, stack_off)); in move_stack2blob()
255 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, in move_stack2blob()
257 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0)); in move_stack2blob()
260 static void move_stack2ctx(struct bpf_gen *gen, int ctx_off, int size, int stack_off) in move_stack2ctx() argument
262 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_10, stack_off)); in move_stack2ctx()
263 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_6, BPF_REG_0, ctx_off)); in move_stack2ctx()
266 static void emit_sys_bpf(struct bpf_gen *gen, int cmd, int attr, int attr_size) in emit_sys_bpf() argument
268 emit(gen, BPF_MOV64_IMM(BPF_REG_1, cmd)); in emit_sys_bpf()
269 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_2, BPF_PSEUDO_MAP_IDX_VALUE, in emit_sys_bpf()
271 emit(gen, BPF_MOV64_IMM(BPF_REG_3, attr_size)); in emit_sys_bpf()
272 emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_bpf)); in emit_sys_bpf()
274 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0)); in emit_sys_bpf()
282 static void emit_check_err(struct bpf_gen *gen) in emit_check_err() argument
284 __s64 off = -(gen->insn_cur - gen->insn_start - gen->cleanup_label) / 8 - 1; in emit_check_err()
290 emit(gen, BPF_JMP_IMM(BPF_JSLT, BPF_REG_7, 0, off)); in emit_check_err()
292 gen->error = -ERANGE; in emit_check_err()
293 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, -1)); in emit_check_err()
298 static void emit_debug(struct bpf_gen *gen, int reg1, int reg2, in emit_debug() argument
304 if (!gen->log_level) in emit_debug()
314 addr = add_data(gen, buf, len); in emit_debug()
316 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, in emit_debug()
318 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len)); in emit_debug()
320 emit(gen, BPF_MOV64_REG(BPF_REG_3, reg1)); in emit_debug()
322 emit(gen, BPF_MOV64_REG(BPF_REG_4, reg2)); in emit_debug()
323 emit(gen, BPF_EMIT_CALL(BPF_FUNC_trace_printk)); in emit_debug()
326 static void debug_regs(struct bpf_gen *gen, int reg1, int reg2, const char *fmt, ...) in debug_regs() argument
331 emit_debug(gen, reg1, reg2, fmt, args); in debug_regs()
335 static void debug_ret(struct bpf_gen *gen, const char *fmt, ...) in debug_ret() argument
340 emit_debug(gen, BPF_REG_7, -1, fmt, args); in debug_ret()
344 static void __emit_sys_close(struct bpf_gen *gen) in __emit_sys_close() argument
346 emit(gen, BPF_JMP_IMM(BPF_JSLE, BPF_REG_1, 0, in __emit_sys_close()
350 2 + (gen->log_level ? 6 : 0))); in __emit_sys_close()
351 emit(gen, BPF_MOV64_REG(BPF_REG_9, BPF_REG_1)); in __emit_sys_close()
352 emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_close)); in __emit_sys_close()
353 debug_regs(gen, BPF_REG_9, BPF_REG_0, "close(%%d) = %%d"); in __emit_sys_close()
356 static void emit_sys_close_stack(struct bpf_gen *gen, int stack_off) in emit_sys_close_stack() argument
358 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_10, stack_off)); in emit_sys_close_stack()
359 __emit_sys_close(gen); in emit_sys_close_stack()
362 static void emit_sys_close_blob(struct bpf_gen *gen, int blob_off) in emit_sys_close_blob() argument
364 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE, in emit_sys_close_blob()
366 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_0, 0)); in emit_sys_close_blob()
367 __emit_sys_close(gen); in emit_sys_close_blob()
370 int bpf_gen__finish(struct bpf_gen *gen, int nr_progs, int nr_maps) in bpf_gen__finish() argument
374 if (nr_progs != gen->nr_progs || nr_maps != gen->nr_maps) { in bpf_gen__finish()
376 gen->error = -EFAULT; in bpf_gen__finish()
377 return gen->error; in bpf_gen__finish()
379 emit_sys_close_stack(gen, stack_off(btf_fd)); in bpf_gen__finish()
380 for (i = 0; i < gen->nr_progs; i++) in bpf_gen__finish()
381 move_stack2ctx(gen, in bpf_gen__finish()
383 sizeof(struct bpf_map_desc) * gen->nr_maps + in bpf_gen__finish()
387 for (i = 0; i < gen->nr_maps; i++) in bpf_gen__finish()
388 move_blob2ctx(gen, in bpf_gen__finish()
392 blob_fd_array_off(gen, i)); in bpf_gen__finish()
393 emit(gen, BPF_MOV64_IMM(BPF_REG_0, 0)); in bpf_gen__finish()
394 emit(gen, BPF_EXIT_INSN()); in bpf_gen__finish()
395 pr_debug("gen: finish %d\n", gen->error); in bpf_gen__finish()
396 if (!gen->error) { in bpf_gen__finish()
397 struct gen_loader_opts *opts = gen->opts; in bpf_gen__finish()
399 opts->insns = gen->insn_start; in bpf_gen__finish()
400 opts->insns_sz = gen->insn_cur - gen->insn_start; in bpf_gen__finish()
401 opts->data = gen->data_start; in bpf_gen__finish()
402 opts->data_sz = gen->data_cur - gen->data_start; in bpf_gen__finish()
404 return gen->error; in bpf_gen__finish()
407 void bpf_gen__free(struct bpf_gen *gen) in bpf_gen__free() argument
409 if (!gen) in bpf_gen__free()
411 free(gen->data_start); in bpf_gen__free()
412 free(gen->insn_start); in bpf_gen__free()
413 free(gen); in bpf_gen__free()
416 void bpf_gen__load_btf(struct bpf_gen *gen, const void *btf_raw_data, in bpf_gen__load_btf() argument
425 btf_data = add_data(gen, btf_raw_data, btf_raw_size); in bpf_gen__load_btf()
428 btf_load_attr = add_data(gen, &attr, attr_size); in bpf_gen__load_btf()
431 move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_level), 4, in bpf_gen__load_btf()
433 move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_size), 4, in bpf_gen__load_btf()
435 move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_buf), 8, in bpf_gen__load_btf()
438 emit_rel_store(gen, attr_field(btf_load_attr, btf), btf_data); in bpf_gen__load_btf()
440 emit_sys_bpf(gen, BPF_BTF_LOAD, btf_load_attr, attr_size); in bpf_gen__load_btf()
441 debug_ret(gen, "btf_load size %d", btf_raw_size); in bpf_gen__load_btf()
442 emit_check_err(gen); in bpf_gen__load_btf()
444 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7, stack_off(btf_fd))); in bpf_gen__load_btf()
447 void bpf_gen__map_create(struct bpf_gen *gen, in bpf_gen__map_create() argument
490 map_create_attr = add_data(gen, &attr, attr_size); in bpf_gen__map_create()
493 move_stack2blob(gen, attr_field(map_create_attr, btf_fd), 4, in bpf_gen__map_create()
498 move_stack2blob(gen, attr_field(map_create_attr, inner_map_fd), 4, in bpf_gen__map_create()
507 move_ctx2blob(gen, attr_field(map_create_attr, max_entries), 4, in bpf_gen__map_create()
513 emit_sys_bpf(gen, BPF_MAP_CREATE, map_create_attr, attr_size); in bpf_gen__map_create()
514 debug_ret(gen, "map_create %s idx %d type %d value_size %d value_btf_id %d", in bpf_gen__map_create()
517 emit_check_err(gen); in bpf_gen__map_create()
524 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7, in bpf_gen__map_create()
526 } else if (map_idx != gen->nr_maps) { in bpf_gen__map_create()
527 gen->error = -EDOM; /* internal bug */ in bpf_gen__map_create()
531 idx = add_map_fd(gen); in bpf_gen__map_create()
532 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, in bpf_gen__map_create()
533 0, 0, 0, blob_fd_array_off(gen, idx))); in bpf_gen__map_create()
534 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_7, 0)); in bpf_gen__map_create()
537 emit_sys_close_stack(gen, stack_off(inner_map_fd)); in bpf_gen__map_create()
540 void bpf_gen__record_attach_target(struct bpf_gen *gen, const char *attach_name, in bpf_gen__record_attach_target() argument
547 gen->attach_kind = kind; in bpf_gen__record_attach_target()
548 ret = snprintf(gen->attach_target, sizeof(gen->attach_target), "%s%s", in bpf_gen__record_attach_target()
550 if (ret == sizeof(gen->attach_target)) in bpf_gen__record_attach_target()
551 gen->error = -ENOSPC; in bpf_gen__record_attach_target()
554 static void emit_find_attach_target(struct bpf_gen *gen) in emit_find_attach_target() argument
556 int name, len = strlen(gen->attach_target) + 1; in emit_find_attach_target()
558 pr_debug("gen: find_attach_tgt %s %d\n", gen->attach_target, gen->attach_kind); in emit_find_attach_target()
559 name = add_data(gen, gen->attach_target, len); in emit_find_attach_target()
561 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, in emit_find_attach_target()
563 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len)); in emit_find_attach_target()
564 emit(gen, BPF_MOV64_IMM(BPF_REG_3, gen->attach_kind)); in emit_find_attach_target()
565 emit(gen, BPF_MOV64_IMM(BPF_REG_4, 0)); in emit_find_attach_target()
566 emit(gen, BPF_EMIT_CALL(BPF_FUNC_btf_find_by_name_kind)); in emit_find_attach_target()
567 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0)); in emit_find_attach_target()
568 debug_ret(gen, "find_by_name_kind(%s,%d)", in emit_find_attach_target()
569 gen->attach_target, gen->attach_kind); in emit_find_attach_target()
570 emit_check_err(gen); in emit_find_attach_target()
576 void bpf_gen__record_extern(struct bpf_gen *gen, const char *name, bool is_weak, in bpf_gen__record_extern() argument
581 relo = libbpf_reallocarray(gen->relos, gen->relo_cnt + 1, sizeof(*relo)); in bpf_gen__record_extern()
583 gen->error = -ENOMEM; in bpf_gen__record_extern()
586 gen->relos = relo; in bpf_gen__record_extern()
587 relo += gen->relo_cnt; in bpf_gen__record_extern()
593 gen->relo_cnt++; in bpf_gen__record_extern()
597 static struct ksym_desc *get_ksym_desc(struct bpf_gen *gen, struct ksym_relo_desc *relo) in get_ksym_desc() argument
601 for (int i = 0; i < gen->nr_ksyms; i++) { in get_ksym_desc()
602 if (!strcmp(gen->ksyms[i].name, relo->name)) { in get_ksym_desc()
603 gen->ksyms[i].ref++; in get_ksym_desc()
604 return &gen->ksyms[i]; in get_ksym_desc()
607 kdesc = libbpf_reallocarray(gen->ksyms, gen->nr_ksyms + 1, sizeof(*kdesc)); in get_ksym_desc()
609 gen->error = -ENOMEM; in get_ksym_desc()
612 gen->ksyms = kdesc; in get_ksym_desc()
613 kdesc = &gen->ksyms[gen->nr_ksyms++]; in get_ksym_desc()
625 static void emit_bpf_find_by_name_kind(struct bpf_gen *gen, struct ksym_relo_desc *relo) in emit_bpf_find_by_name_kind() argument
629 name_off = add_data(gen, relo->name, len); in emit_bpf_find_by_name_kind()
630 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, in emit_bpf_find_by_name_kind()
632 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len)); in emit_bpf_find_by_name_kind()
633 emit(gen, BPF_MOV64_IMM(BPF_REG_3, relo->kind)); in emit_bpf_find_by_name_kind()
634 emit(gen, BPF_MOV64_IMM(BPF_REG_4, 0)); in emit_bpf_find_by_name_kind()
635 emit(gen, BPF_EMIT_CALL(BPF_FUNC_btf_find_by_name_kind)); in emit_bpf_find_by_name_kind()
636 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0)); in emit_bpf_find_by_name_kind()
637 debug_ret(gen, "find_by_name_kind(%s,%d)", relo->name, relo->kind); in emit_bpf_find_by_name_kind()
644 static void emit_bpf_kallsyms_lookup_name(struct bpf_gen *gen, struct ksym_relo_desc *relo) in emit_bpf_kallsyms_lookup_name() argument
648 name_off = add_data(gen, relo->name, len); in emit_bpf_kallsyms_lookup_name()
649 res_off = add_data(gen, NULL, 8); /* res is u64 */ in emit_bpf_kallsyms_lookup_name()
650 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, in emit_bpf_kallsyms_lookup_name()
652 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len)); in emit_bpf_kallsyms_lookup_name()
653 emit(gen, BPF_MOV64_IMM(BPF_REG_3, 0)); in emit_bpf_kallsyms_lookup_name()
654 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_4, BPF_PSEUDO_MAP_IDX_VALUE, in emit_bpf_kallsyms_lookup_name()
656 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_4)); in emit_bpf_kallsyms_lookup_name()
657 emit(gen, BPF_EMIT_CALL(BPF_FUNC_kallsyms_lookup_name)); in emit_bpf_kallsyms_lookup_name()
658 emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_9, BPF_REG_7, 0)); in emit_bpf_kallsyms_lookup_name()
659 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0)); in emit_bpf_kallsyms_lookup_name()
660 debug_ret(gen, "kallsyms_lookup_name(%s,%d)", relo->name, relo->kind); in emit_bpf_kallsyms_lookup_name()
673 static void emit_relo_kfunc_btf(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insn) in emit_relo_kfunc_btf() argument
678 kdesc = get_ksym_desc(gen, relo); in emit_relo_kfunc_btf()
683 move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4, in emit_relo_kfunc_btf()
685 move_blob2blob(gen, insn + offsetof(struct bpf_insn, off), 2, in emit_relo_kfunc_btf()
691 emit_bpf_find_by_name_kind(gen, relo); in emit_relo_kfunc_btf()
693 emit_check_err(gen); in emit_relo_kfunc_btf()
695 btf_fd_idx = add_kfunc_btf_fd(gen); in emit_relo_kfunc_btf()
699 gen->error = -E2BIG; in emit_relo_kfunc_btf()
704 emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, offsetof(struct bpf_insn, imm), 0)); in emit_relo_kfunc_btf()
706 emit(gen, BPF_JMP_IMM(BPF_JSLT, BPF_REG_7, 0, 1)); in emit_relo_kfunc_btf()
708 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7, offsetof(struct bpf_insn, imm))); in emit_relo_kfunc_btf()
710 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE, in emit_relo_kfunc_btf()
711 0, 0, 0, blob_fd_array_off(gen, btf_fd_idx))); in emit_relo_kfunc_btf()
713 emit(gen, BPF_JMP_IMM(BPF_JSLT, BPF_REG_7, 0, 3)); in emit_relo_kfunc_btf()
715 emit(gen, BPF_MOV64_REG(BPF_REG_9, BPF_REG_7)); in emit_relo_kfunc_btf()
716 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_9, 32)); in emit_relo_kfunc_btf()
717 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_9, 0)); in emit_relo_kfunc_btf()
719 emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), 0)); in emit_relo_kfunc_btf()
721 emit(gen, BPF_JMP_IMM(BPF_JSLT, BPF_REG_7, 0, 2)); in emit_relo_kfunc_btf()
723 emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_9, 0, 1)); in emit_relo_kfunc_btf()
725 emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), btf_fd_idx)); in emit_relo_kfunc_btf()
727 if (!gen->log_level) in emit_relo_kfunc_btf()
729 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_8, in emit_relo_kfunc_btf()
731 emit(gen, BPF_LDX_MEM(BPF_H, BPF_REG_9, BPF_REG_8, in emit_relo_kfunc_btf()
733 debug_regs(gen, BPF_REG_7, BPF_REG_9, " func (%s:count=%d): imm: %%d, off: %%d", in emit_relo_kfunc_btf()
735 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE, in emit_relo_kfunc_btf()
736 0, 0, 0, blob_fd_array_off(gen, kdesc->off))); in emit_relo_kfunc_btf()
737 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_9, BPF_REG_0, 0)); in emit_relo_kfunc_btf()
738 debug_regs(gen, BPF_REG_9, -1, " func (%s:count=%d): btf_fd", in emit_relo_kfunc_btf()
742 static void emit_ksym_relo_log(struct bpf_gen *gen, struct ksym_relo_desc *relo, in emit_ksym_relo_log() argument
745 if (!gen->log_level) in emit_ksym_relo_log()
747 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_8, in emit_ksym_relo_log()
749 emit(gen, BPF_LDX_MEM(BPF_H, BPF_REG_9, BPF_REG_8, sizeof(struct bpf_insn) + in emit_ksym_relo_log()
751 debug_regs(gen, BPF_REG_7, BPF_REG_9, " var t=%d w=%d (%s:count=%d): imm[0]: %%d, imm[1]: %%d", in emit_ksym_relo_log()
753 emit(gen, BPF_LDX_MEM(BPF_B, BPF_REG_9, BPF_REG_8, offsetofend(struct bpf_insn, code))); in emit_ksym_relo_log()
754 debug_regs(gen, BPF_REG_9, -1, " var t=%d w=%d (%s:count=%d): insn.reg", in emit_ksym_relo_log()
761 static void emit_relo_ksym_typeless(struct bpf_gen *gen, in emit_relo_ksym_typeless() argument
766 kdesc = get_ksym_desc(gen, relo); in emit_relo_ksym_typeless()
771 move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4, in emit_relo_ksym_typeless()
773 move_blob2blob(gen, insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 4, in emit_relo_ksym_typeless()
781 emit_bpf_kallsyms_lookup_name(gen, relo); in emit_relo_ksym_typeless()
782 emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_7, -ENOENT, 1)); in emit_relo_ksym_typeless()
783 emit_check_err(gen); in emit_relo_ksym_typeless()
785 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_9, offsetof(struct bpf_insn, imm))); in emit_relo_ksym_typeless()
787 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_9, 32)); in emit_relo_ksym_typeless()
788 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_9, in emit_relo_ksym_typeless()
791 emit_ksym_relo_log(gen, relo, kdesc->ref); in emit_relo_ksym_typeless()
808 static void emit_relo_ksym_btf(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insn) in emit_relo_ksym_btf() argument
813 kdesc = get_ksym_desc(gen, relo); in emit_relo_ksym_btf()
818 move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4, in emit_relo_ksym_btf()
820 move_blob2blob(gen, insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 4, in emit_relo_ksym_btf()
822 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_9, BPF_REG_8, offsetof(struct bpf_insn, imm))); in emit_relo_ksym_btf()
824 emit(gen, BPF_JMP_IMM(BPF_JNE, BPF_REG_9, 0, 3)); in emit_relo_ksym_btf()
829 emit_bpf_find_by_name_kind(gen, relo); in emit_relo_ksym_btf()
831 emit_check_err(gen); in emit_relo_ksym_btf()
833 emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, offsetof(struct bpf_insn, imm), 0)); in emit_relo_ksym_btf()
834 …emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 0… in emit_relo_ksym_btf()
836 emit(gen, BPF_JMP_IMM(BPF_JSLT, BPF_REG_7, 0, 4)); in emit_relo_ksym_btf()
838 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7, offsetof(struct bpf_insn, imm))); in emit_relo_ksym_btf()
840 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_7, 32)); in emit_relo_ksym_btf()
841 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7, in emit_relo_ksym_btf()
843 emit(gen, BPF_JMP_IMM(BPF_JSGE, BPF_REG_7, 0, 3)); in emit_relo_ksym_btf()
847 emit(gen, BPF_LDX_MEM(BPF_B, BPF_REG_9, BPF_REG_8, offsetofend(struct bpf_insn, code))); in emit_relo_ksym_btf()
848 emit(gen, BPF_ALU32_IMM(BPF_AND, BPF_REG_9, reg_mask)); in emit_relo_ksym_btf()
849 emit(gen, BPF_STX_MEM(BPF_B, BPF_REG_8, BPF_REG_9, offsetofend(struct bpf_insn, code))); in emit_relo_ksym_btf()
851 emit_ksym_relo_log(gen, relo, kdesc->ref); in emit_relo_ksym_btf()
854 static void emit_relo(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insns) in emit_relo() argument
860 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_8, BPF_PSEUDO_MAP_IDX_VALUE, 0, 0, 0, insn)); in emit_relo()
864 emit_relo_ksym_typeless(gen, relo, insn); in emit_relo()
866 emit_relo_ksym_btf(gen, relo, insn); in emit_relo()
869 emit_relo_kfunc_btf(gen, relo, insn); in emit_relo()
873 gen->error = -EDOM; in emit_relo()
878 static void emit_relos(struct bpf_gen *gen, int insns) in emit_relos() argument
882 for (i = 0; i < gen->relo_cnt; i++) in emit_relos()
883 emit_relo(gen, gen->relos + i, insns); in emit_relos()
886 static void cleanup_relos(struct bpf_gen *gen, int insns) in cleanup_relos() argument
890 for (i = 0; i < gen->nr_ksyms; i++) { in cleanup_relos()
892 if (gen->ksyms[i].kind == BTF_KIND_VAR && !gen->ksyms[i].typeless) { in cleanup_relos()
894 insn = gen->ksyms[i].insn; in cleanup_relos()
896 emit_sys_close_blob(gen, insn); in cleanup_relos()
897 } else if (gen->ksyms[i].kind == BTF_KIND_FUNC) { in cleanup_relos()
898 emit_sys_close_blob(gen, blob_fd_array_off(gen, gen->ksyms[i].off)); in cleanup_relos()
899 if (gen->ksyms[i].off < MAX_FD_ARRAY_SZ) in cleanup_relos()
900 gen->nr_fd_array--; in cleanup_relos()
903 if (gen->nr_ksyms) { in cleanup_relos()
904 free(gen->ksyms); in cleanup_relos()
905 gen->nr_ksyms = 0; in cleanup_relos()
906 gen->ksyms = NULL; in cleanup_relos()
908 if (gen->relo_cnt) { in cleanup_relos()
909 free(gen->relos); in cleanup_relos()
910 gen->relo_cnt = 0; in cleanup_relos()
911 gen->relos = NULL; in cleanup_relos()
915 void bpf_gen__prog_load(struct bpf_gen *gen, in bpf_gen__prog_load() argument
926 license = add_data(gen, load_attr->license, strlen(load_attr->license) + 1); in bpf_gen__prog_load()
928 insns = add_data(gen, load_attr->insns, in bpf_gen__prog_load()
941 func_info = add_data(gen, load_attr->func_info, in bpf_gen__prog_load()
946 line_info = add_data(gen, load_attr->line_info, in bpf_gen__prog_load()
951 prog_load_attr = add_data(gen, &attr, attr_size); in bpf_gen__prog_load()
954 emit_rel_store(gen, attr_field(prog_load_attr, license), license); in bpf_gen__prog_load()
957 emit_rel_store(gen, attr_field(prog_load_attr, insns), insns); in bpf_gen__prog_load()
960 emit_rel_store(gen, attr_field(prog_load_attr, func_info), func_info); in bpf_gen__prog_load()
963 emit_rel_store(gen, attr_field(prog_load_attr, line_info), line_info); in bpf_gen__prog_load()
966 emit_rel_store(gen, attr_field(prog_load_attr, fd_array), gen->fd_array); in bpf_gen__prog_load()
969 move_ctx2blob(gen, attr_field(prog_load_attr, log_level), 4, in bpf_gen__prog_load()
971 move_ctx2blob(gen, attr_field(prog_load_attr, log_size), 4, in bpf_gen__prog_load()
973 move_ctx2blob(gen, attr_field(prog_load_attr, log_buf), 8, in bpf_gen__prog_load()
976 move_stack2blob(gen, attr_field(prog_load_attr, prog_btf_fd), 4, in bpf_gen__prog_load()
978 if (gen->attach_kind) { in bpf_gen__prog_load()
979 emit_find_attach_target(gen); in bpf_gen__prog_load()
981 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE, in bpf_gen__prog_load()
983 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_7, in bpf_gen__prog_load()
985 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_7, 32)); in bpf_gen__prog_load()
986 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_7, in bpf_gen__prog_load()
989 emit_relos(gen, insns); in bpf_gen__prog_load()
991 emit_sys_bpf(gen, BPF_PROG_LOAD, prog_load_attr, attr_size); in bpf_gen__prog_load()
992 debug_ret(gen, "prog_load %s insn_cnt %d", attr.prog_name, attr.insn_cnt); in bpf_gen__prog_load()
994 cleanup_relos(gen, insns); in bpf_gen__prog_load()
995 if (gen->attach_kind) in bpf_gen__prog_load()
996 emit_sys_close_blob(gen, in bpf_gen__prog_load()
998 emit_check_err(gen); in bpf_gen__prog_load()
1000 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7, in bpf_gen__prog_load()
1001 stack_off(prog_fd[gen->nr_progs]))); in bpf_gen__prog_load()
1002 gen->nr_progs++; in bpf_gen__prog_load()
1005 void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue, in bpf_gen__map_update_elem() argument
1016 value = add_data(gen, pvalue, value_size); in bpf_gen__map_update_elem()
1017 key = add_data(gen, &zero, sizeof(zero)); in bpf_gen__map_update_elem()
1022 emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_6, in bpf_gen__map_update_elem()
1026 emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_3, 0, 4)); in bpf_gen__map_update_elem()
1027 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, in bpf_gen__map_update_elem()
1029 emit(gen, BPF_MOV64_IMM(BPF_REG_2, value_size)); in bpf_gen__map_update_elem()
1030 emit(gen, BPF_EMIT_CALL(BPF_FUNC_copy_from_user)); in bpf_gen__map_update_elem()
1032 map_update_attr = add_data(gen, &attr, attr_size); in bpf_gen__map_update_elem()
1033 move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4, in bpf_gen__map_update_elem()
1034 blob_fd_array_off(gen, map_idx)); in bpf_gen__map_update_elem()
1035 emit_rel_store(gen, attr_field(map_update_attr, key), key); in bpf_gen__map_update_elem()
1036 emit_rel_store(gen, attr_field(map_update_attr, value), value); in bpf_gen__map_update_elem()
1038 emit_sys_bpf(gen, BPF_MAP_UPDATE_ELEM, map_update_attr, attr_size); in bpf_gen__map_update_elem()
1039 debug_ret(gen, "update_elem idx %d value_size %d", map_idx, value_size); in bpf_gen__map_update_elem()
1040 emit_check_err(gen); in bpf_gen__map_update_elem()
1043 void bpf_gen__map_freeze(struct bpf_gen *gen, int map_idx) in bpf_gen__map_freeze() argument
1051 map_freeze_attr = add_data(gen, &attr, attr_size); in bpf_gen__map_freeze()
1052 move_blob2blob(gen, attr_field(map_freeze_attr, map_fd), 4, in bpf_gen__map_freeze()
1053 blob_fd_array_off(gen, map_idx)); in bpf_gen__map_freeze()
1055 emit_sys_bpf(gen, BPF_MAP_FREEZE, map_freeze_attr, attr_size); in bpf_gen__map_freeze()
1056 debug_ret(gen, "map_freeze"); in bpf_gen__map_freeze()
1057 emit_check_err(gen); in bpf_gen__map_freeze()