Lines Matching refs:mod
177 static void mod_tree_insert(struct module *mod) in mod_tree_insert() argument
179 mod->core_layout.mtn.mod = mod; in mod_tree_insert()
180 mod->init_layout.mtn.mod = mod; in mod_tree_insert()
182 __mod_tree_insert(&mod->core_layout.mtn); in mod_tree_insert()
183 if (mod->init_layout.size) in mod_tree_insert()
184 __mod_tree_insert(&mod->init_layout.mtn); in mod_tree_insert()
187 static void mod_tree_remove_init(struct module *mod) in mod_tree_remove_init() argument
189 if (mod->init_layout.size) in mod_tree_remove_init()
190 __mod_tree_remove(&mod->init_layout.mtn); in mod_tree_remove_init()
193 static void mod_tree_remove(struct module *mod) in mod_tree_remove() argument
195 __mod_tree_remove(&mod->core_layout.mtn); in mod_tree_remove()
196 mod_tree_remove_init(mod); in mod_tree_remove()
207 return container_of(ltn, struct mod_tree_node, node)->mod; in mod_find()
214 static void mod_tree_insert(struct module *mod) { } in mod_tree_insert() argument
215 static void mod_tree_remove_init(struct module *mod) { } in mod_tree_remove_init() argument
216 static void mod_tree_remove(struct module *mod) { } in mod_tree_remove() argument
220 struct module *mod; in mod_find() local
222 list_for_each_entry_rcu(mod, &modules, list, in mod_find()
224 if (within_module(addr, mod)) in mod_find()
225 return mod; in mod_find()
248 static void mod_update_bounds(struct module *mod) in mod_update_bounds() argument
250 __mod_update_bounds(mod->core_layout.base, mod->core_layout.size); in mod_update_bounds()
251 if (mod->init_layout.size) in mod_update_bounds()
252 __mod_update_bounds(mod->init_layout.base, mod->init_layout.size); in mod_update_bounds()
318 static inline int strong_try_module_get(struct module *mod) in strong_try_module_get() argument
320 BUG_ON(mod && mod->state == MODULE_STATE_UNFORMED); in strong_try_module_get()
321 if (mod && mod->state == MODULE_STATE_COMING) in strong_try_module_get()
323 if (try_module_get(mod)) in strong_try_module_get()
329 static inline void add_taint_module(struct module *mod, unsigned flag, in add_taint_module() argument
333 set_bit(flag, &mod->taints); in add_taint_module()
340 void __noreturn __module_put_and_exit(struct module *mod, long code) in __module_put_and_exit() argument
342 module_put(mod); in __module_put_and_exit()
526 struct module *mod; in find_symbol() local
535 list_for_each_entry_rcu(mod, &modules, list, in find_symbol()
538 { mod->syms, mod->syms + mod->num_syms, mod->crcs, in find_symbol()
540 { mod->gpl_syms, mod->gpl_syms + mod->num_gpl_syms, in find_symbol()
541 mod->gpl_crcs, in find_symbol()
545 if (mod->state == MODULE_STATE_UNFORMED) in find_symbol()
549 if (find_exported_symbol_in_section(&arr[i], mod, fsa)) in find_symbol()
564 struct module *mod; in find_module_all() local
568 list_for_each_entry_rcu(mod, &modules, list, in find_module_all()
570 if (!even_unformed && mod->state == MODULE_STATE_UNFORMED) in find_module_all()
572 if (strlen(mod->name) == len && !memcmp(mod->name, name, len)) in find_module_all()
573 return mod; in find_module_all()
585 static inline void __percpu *mod_percpu(struct module *mod) in mod_percpu() argument
587 return mod->percpu; in mod_percpu()
590 static int percpu_modalloc(struct module *mod, struct load_info *info) in percpu_modalloc() argument
600 mod->name, align, PAGE_SIZE); in percpu_modalloc()
604 mod->percpu = __alloc_reserved_percpu(pcpusec->sh_size, align); in percpu_modalloc()
605 if (!mod->percpu) { in percpu_modalloc()
607 mod->name, (unsigned long)pcpusec->sh_size); in percpu_modalloc()
610 mod->percpu_size = pcpusec->sh_size; in percpu_modalloc()
614 static void percpu_modfree(struct module *mod) in percpu_modfree() argument
616 free_percpu(mod->percpu); in percpu_modfree()
624 static void percpu_modcopy(struct module *mod, in percpu_modcopy() argument
630 memcpy(per_cpu_ptr(mod->percpu, cpu), from, size); in percpu_modcopy()
635 struct module *mod; in __is_module_percpu_address() local
640 list_for_each_entry_rcu(mod, &modules, list) { in __is_module_percpu_address()
641 if (mod->state == MODULE_STATE_UNFORMED) in __is_module_percpu_address()
643 if (!mod->percpu_size) in __is_module_percpu_address()
646 void *start = per_cpu_ptr(mod->percpu, cpu); in __is_module_percpu_address()
649 if (va >= start && va < start + mod->percpu_size) { in __is_module_percpu_address()
653 per_cpu_ptr(mod->percpu, in __is_module_percpu_address()
681 static inline void __percpu *mod_percpu(struct module *mod) in mod_percpu() argument
685 static int percpu_modalloc(struct module *mod, struct load_info *info) in percpu_modalloc() argument
692 static inline void percpu_modfree(struct module *mod) in percpu_modfree() argument
699 static inline void percpu_modcopy(struct module *mod, in percpu_modcopy() argument
718 static void setup_modinfo_##field(struct module *mod, const char *s) \
720 mod->field = kstrdup(s, GFP_KERNEL); \
725 return scnprintf(buffer, PAGE_SIZE, "%s\n", mk->mod->field); \
727 static int modinfo_##field##_exists(struct module *mod) \
729 return mod->field != NULL; \
731 static void free_modinfo_##field(struct module *mod) \
733 kfree(mod->field); \
734 mod->field = NULL; \
757 static int module_unload_init(struct module *mod) in module_unload_init() argument
763 atomic_set(&mod->refcnt, MODULE_REF_BASE); in module_unload_init()
765 INIT_LIST_HEAD(&mod->source_list); in module_unload_init()
766 INIT_LIST_HEAD(&mod->target_list); in module_unload_init()
769 atomic_inc(&mod->refcnt); in module_unload_init()
834 static void module_unload_free(struct module *mod) in module_unload_free() argument
839 list_for_each_entry_safe(use, tmp, &mod->target_list, target_list) { in module_unload_free()
841 pr_debug("%s unusing %s\n", mod->name, i->name); in module_unload_free()
866 static int try_release_module_ref(struct module *mod) in try_release_module_ref() argument
871 ret = atomic_sub_return(MODULE_REF_BASE, &mod->refcnt); in try_release_module_ref()
875 ret = atomic_add_unless(&mod->refcnt, MODULE_REF_BASE, 0); in try_release_module_ref()
880 static int try_stop_module(struct module *mod, int flags, int *forced) in try_stop_module() argument
883 if (try_release_module_ref(mod) != 0) { in try_stop_module()
890 mod->state = MODULE_STATE_GOING; in try_stop_module()
903 int module_refcount(struct module *mod) in module_refcount() argument
905 return atomic_read(&mod->refcnt) - MODULE_REF_BASE; in module_refcount()
910 static void free_module(struct module *mod);
915 struct module *mod; in SYSCALL_DEFINE2() local
931 mod = find_module(name); in SYSCALL_DEFINE2()
932 if (!mod) { in SYSCALL_DEFINE2()
937 if (!list_empty(&mod->source_list)) { in SYSCALL_DEFINE2()
944 if (mod->state != MODULE_STATE_LIVE) { in SYSCALL_DEFINE2()
946 pr_debug("%s already dying\n", mod->name); in SYSCALL_DEFINE2()
952 if (mod->init && !mod->exit) { in SYSCALL_DEFINE2()
962 ret = try_stop_module(mod, flags, &forced); in SYSCALL_DEFINE2()
968 if (mod->exit != NULL) in SYSCALL_DEFINE2()
969 mod->exit(); in SYSCALL_DEFINE2()
971 MODULE_STATE_GOING, mod); in SYSCALL_DEFINE2()
972 klp_module_going(mod); in SYSCALL_DEFINE2()
973 ftrace_release_mod(mod); in SYSCALL_DEFINE2()
978 strlcpy(last_unloaded_module, mod->name, sizeof(last_unloaded_module)); in SYSCALL_DEFINE2()
980 free_module(mod); in SYSCALL_DEFINE2()
989 static inline void print_unload_info(struct seq_file *m, struct module *mod) in print_unload_info() argument
994 seq_printf(m, " %i ", module_refcount(mod)); in print_unload_info()
1000 list_for_each_entry(use, &mod->source_list, source_list) { in print_unload_info()
1005 if (mod->init != NULL && mod->exit == NULL) { in print_unload_info()
1052 return sprintf(buffer, "%i\n", module_refcount(mk->mod)); in show_refcnt()
1103 static inline void print_unload_info(struct seq_file *m, struct module *mod) in print_unload_info() argument
1109 static inline void module_unload_free(struct module *mod) in module_unload_free() argument
1118 static inline int module_unload_init(struct module *mod) in module_unload_init() argument
1124 static size_t module_flags_taint(struct module *mod, char *buf) in module_flags_taint() argument
1130 if (taint_flags[i].module && test_bit(i, &mod->taints)) in module_flags_taint()
1142 switch (mk->mod->state) { in show_initstate()
1177 return sprintf(buffer, "%u\n", mk->mod->core_layout.size); in show_coresize()
1186 return sprintf(buffer, "%u\n", mk->mod->init_layout.size); in show_initsize()
1197 l = module_flags_taint(mk->mod, buffer); in show_taint()
1221 static int try_to_force_load(struct module *mod, const char *reason) in try_to_force_load() argument
1225 pr_warn("%s: %s: kernel tainted.\n", mod->name, reason); in try_to_force_load()
1226 add_taint_module(mod, TAINT_FORCED_MODULE, LOCKDEP_NOW_UNRELIABLE); in try_to_force_load()
1242 struct module *mod, in check_version() argument
1256 return try_to_force_load(mod, symname) == 0; in check_version()
1290 struct module *mod) in check_modstruct_version() argument
1307 return check_version(info, "module_layout", mod, fsa.crc); in check_modstruct_version()
1323 struct module *mod, in check_version() argument
1330 struct module *mod) in check_modstruct_version() argument
1348 struct module *mod) in verify_namespace_is_imported() argument
1368 mod->name, kernel_symbol_name(sym), namespace); in verify_namespace_is_imported()
1376 static bool inherit_taint(struct module *mod, struct module *owner) in inherit_taint() argument
1381 if (mod->using_gplonly_symbols) { in inherit_taint()
1383 mod->name, owner->name); in inherit_taint()
1387 if (!test_bit(TAINT_PROPRIETARY_MODULE, &mod->taints)) { in inherit_taint()
1389 mod->name, owner->name); in inherit_taint()
1390 set_bit(TAINT_PROPRIETARY_MODULE, &mod->taints); in inherit_taint()
1396 static const struct kernel_symbol *resolve_symbol(struct module *mod, in resolve_symbol() argument
1403 .gplok = !(mod->taints & (1 << TAINT_PROPRIETARY_MODULE)), in resolve_symbol()
1419 mod->using_gplonly_symbols = true; in resolve_symbol()
1421 if (!inherit_taint(mod, fsa.owner)) { in resolve_symbol()
1426 if (!check_version(info, name, mod, fsa.crc)) { in resolve_symbol()
1431 err = verify_namespace_is_imported(info, fsa.sym, mod); in resolve_symbol()
1437 err = ref_module(mod, fsa.owner); in resolve_symbol()
1452 resolve_symbol_wait(struct module *mod, in resolve_symbol_wait() argument
1460 !IS_ERR(ksym = resolve_symbol(mod, info, name, owner)) in resolve_symbol_wait()
1464 mod->name, owner); in resolve_symbol_wait()
1533 static void add_sect_attrs(struct module *mod, const struct load_info *info) in add_sect_attrs() argument
1576 if (sysfs_create_group(&mod->mkobj.kobj, §_attrs->grp)) in add_sect_attrs()
1579 mod->sect_attrs = sect_attrs; in add_sect_attrs()
1585 static void remove_sect_attrs(struct module *mod) in remove_sect_attrs() argument
1587 if (mod->sect_attrs) { in remove_sect_attrs()
1588 sysfs_remove_group(&mod->mkobj.kobj, in remove_sect_attrs()
1589 &mod->sect_attrs->grp); in remove_sect_attrs()
1594 free_sect_attrs(mod->sect_attrs); in remove_sect_attrs()
1595 mod->sect_attrs = NULL; in remove_sect_attrs()
1632 static void add_notes_attrs(struct module *mod, const struct load_info *info) in add_notes_attrs() argument
1639 if (!mod->sect_attrs) in add_notes_attrs()
1664 nattr->attr.name = mod->sect_attrs->attrs[loaded].battr.attr.name; in add_notes_attrs()
1674 notes_attrs->dir = kobject_create_and_add("notes", &mod->mkobj.kobj); in add_notes_attrs()
1683 mod->notes_attrs = notes_attrs; in add_notes_attrs()
1690 static void remove_notes_attrs(struct module *mod) in remove_notes_attrs() argument
1692 if (mod->notes_attrs) in remove_notes_attrs()
1693 free_notes_attrs(mod->notes_attrs, mod->notes_attrs->notes); in remove_notes_attrs()
1698 static inline void add_sect_attrs(struct module *mod, in add_sect_attrs() argument
1703 static inline void remove_sect_attrs(struct module *mod) in remove_sect_attrs() argument
1707 static inline void add_notes_attrs(struct module *mod, in add_notes_attrs() argument
1712 static inline void remove_notes_attrs(struct module *mod) in remove_notes_attrs() argument
1717 static void del_usage_links(struct module *mod) in del_usage_links() argument
1723 list_for_each_entry(use, &mod->target_list, target_list) in del_usage_links()
1724 sysfs_remove_link(use->target->holders_dir, mod->name); in del_usage_links()
1729 static int add_usage_links(struct module *mod) in add_usage_links() argument
1736 list_for_each_entry(use, &mod->target_list, target_list) { in add_usage_links()
1738 &mod->mkobj.kobj, mod->name); in add_usage_links()
1744 del_usage_links(mod); in add_usage_links()
1749 static void module_remove_modinfo_attrs(struct module *mod, int end);
1751 static int module_add_modinfo_attrs(struct module *mod) in module_add_modinfo_attrs() argument
1758 mod->modinfo_attrs = kzalloc((sizeof(struct module_attribute) * in module_add_modinfo_attrs()
1761 if (!mod->modinfo_attrs) in module_add_modinfo_attrs()
1764 temp_attr = mod->modinfo_attrs; in module_add_modinfo_attrs()
1766 if (!attr->test || attr->test(mod)) { in module_add_modinfo_attrs()
1769 error = sysfs_create_file(&mod->mkobj.kobj, in module_add_modinfo_attrs()
1781 module_remove_modinfo_attrs(mod, --i); in module_add_modinfo_attrs()
1783 kfree(mod->modinfo_attrs); in module_add_modinfo_attrs()
1787 static void module_remove_modinfo_attrs(struct module *mod, int end) in module_remove_modinfo_attrs() argument
1792 for (i = 0; (attr = &mod->modinfo_attrs[i]); i++) { in module_remove_modinfo_attrs()
1798 sysfs_remove_file(&mod->mkobj.kobj, &attr->attr); in module_remove_modinfo_attrs()
1800 attr->free(mod); in module_remove_modinfo_attrs()
1802 kfree(mod->modinfo_attrs); in module_remove_modinfo_attrs()
1805 static void mod_kobject_put(struct module *mod) in mod_kobject_put() argument
1808 mod->mkobj.kobj_completion = &c; in mod_kobject_put()
1809 kobject_put(&mod->mkobj.kobj); in mod_kobject_put()
1813 static int mod_sysfs_init(struct module *mod) in mod_sysfs_init() argument
1819 pr_err("%s: module sysfs not initialized\n", mod->name); in mod_sysfs_init()
1824 kobj = kset_find_obj(module_kset, mod->name); in mod_sysfs_init()
1826 pr_err("%s: module is already loaded\n", mod->name); in mod_sysfs_init()
1832 mod->mkobj.mod = mod; in mod_sysfs_init()
1834 memset(&mod->mkobj.kobj, 0, sizeof(mod->mkobj.kobj)); in mod_sysfs_init()
1835 mod->mkobj.kobj.kset = module_kset; in mod_sysfs_init()
1836 err = kobject_init_and_add(&mod->mkobj.kobj, &module_ktype, NULL, in mod_sysfs_init()
1837 "%s", mod->name); in mod_sysfs_init()
1839 mod_kobject_put(mod); in mod_sysfs_init()
1845 static int mod_sysfs_setup(struct module *mod, in mod_sysfs_setup() argument
1852 err = mod_sysfs_init(mod); in mod_sysfs_setup()
1856 mod->holders_dir = kobject_create_and_add("holders", &mod->mkobj.kobj); in mod_sysfs_setup()
1857 if (!mod->holders_dir) { in mod_sysfs_setup()
1862 err = module_param_sysfs_setup(mod, kparam, num_params); in mod_sysfs_setup()
1866 err = module_add_modinfo_attrs(mod); in mod_sysfs_setup()
1870 err = add_usage_links(mod); in mod_sysfs_setup()
1874 add_sect_attrs(mod, info); in mod_sysfs_setup()
1875 add_notes_attrs(mod, info); in mod_sysfs_setup()
1880 module_remove_modinfo_attrs(mod, -1); in mod_sysfs_setup()
1882 module_param_sysfs_remove(mod); in mod_sysfs_setup()
1884 kobject_put(mod->holders_dir); in mod_sysfs_setup()
1886 mod_kobject_put(mod); in mod_sysfs_setup()
1891 static void mod_sysfs_fini(struct module *mod) in mod_sysfs_fini() argument
1893 remove_notes_attrs(mod); in mod_sysfs_fini()
1894 remove_sect_attrs(mod); in mod_sysfs_fini()
1895 mod_kobject_put(mod); in mod_sysfs_fini()
1898 static void init_param_lock(struct module *mod) in init_param_lock() argument
1900 mutex_init(&mod->param_lock); in init_param_lock()
1904 static int mod_sysfs_setup(struct module *mod, in mod_sysfs_setup() argument
1912 static void mod_sysfs_fini(struct module *mod) in mod_sysfs_fini() argument
1916 static void module_remove_modinfo_attrs(struct module *mod, int end) in module_remove_modinfo_attrs() argument
1920 static void del_usage_links(struct module *mod) in del_usage_links() argument
1924 static void init_param_lock(struct module *mod) in init_param_lock() argument
1929 static void mod_sysfs_teardown(struct module *mod) in mod_sysfs_teardown() argument
1931 del_usage_links(mod); in mod_sysfs_teardown()
1932 module_remove_modinfo_attrs(mod, -1); in mod_sysfs_teardown()
1933 module_param_sysfs_remove(mod); in mod_sysfs_teardown()
1934 kobject_put(mod->mkobj.drivers_dir); in mod_sysfs_teardown()
1935 kobject_put(mod->holders_dir); in mod_sysfs_teardown()
1936 mod_sysfs_fini(mod); in mod_sysfs_teardown()
1969 static void module_enable_x(const struct module *mod) in module_enable_x() argument
1971 frob_text(&mod->core_layout, set_memory_x); in module_enable_x()
1972 frob_text(&mod->init_layout, set_memory_x); in module_enable_x()
1975 static void module_enable_x(const struct module *mod) { } in module_enable_x() argument
2009 static void module_enable_ro(const struct module *mod, bool after_init) in module_enable_ro() argument
2014 set_vm_flush_reset_perms(mod->core_layout.base); in module_enable_ro()
2015 set_vm_flush_reset_perms(mod->init_layout.base); in module_enable_ro()
2016 frob_text(&mod->core_layout, set_memory_ro); in module_enable_ro()
2018 frob_rodata(&mod->core_layout, set_memory_ro); in module_enable_ro()
2019 frob_text(&mod->init_layout, set_memory_ro); in module_enable_ro()
2020 frob_rodata(&mod->init_layout, set_memory_ro); in module_enable_ro()
2023 frob_ro_after_init(&mod->core_layout, set_memory_ro); in module_enable_ro()
2026 static void module_enable_nx(const struct module *mod) in module_enable_nx() argument
2028 frob_rodata(&mod->core_layout, set_memory_nx); in module_enable_nx()
2029 frob_ro_after_init(&mod->core_layout, set_memory_nx); in module_enable_nx()
2030 frob_writable_data(&mod->core_layout, set_memory_nx); in module_enable_nx()
2031 frob_rodata(&mod->init_layout, set_memory_nx); in module_enable_nx()
2032 frob_writable_data(&mod->init_layout, set_memory_nx); in module_enable_nx()
2036 char *secstrings, struct module *mod) in module_enforce_rwx_sections() argument
2044 mod->name, secstrings + sechdrs[i].sh_name, i); in module_enforce_rwx_sections()
2053 static void module_enable_nx(const struct module *mod) { } in module_enable_nx() argument
2054 static void module_enable_ro(const struct module *mod, bool after_init) {} in module_enable_ro() argument
2056 char *secstrings, struct module *mod) in module_enforce_rwx_sections() argument
2068 static int copy_module_elf(struct module *mod, struct load_info *info) in copy_module_elf() argument
2073 size = sizeof(*mod->klp_info); in copy_module_elf()
2074 mod->klp_info = kmalloc(size, GFP_KERNEL); in copy_module_elf()
2075 if (mod->klp_info == NULL) in copy_module_elf()
2079 size = sizeof(mod->klp_info->hdr); in copy_module_elf()
2080 memcpy(&mod->klp_info->hdr, info->hdr, size); in copy_module_elf()
2084 mod->klp_info->sechdrs = kmemdup(info->sechdrs, size, GFP_KERNEL); in copy_module_elf()
2085 if (mod->klp_info->sechdrs == NULL) { in copy_module_elf()
2092 mod->klp_info->secstrings = kmemdup(info->secstrings, size, GFP_KERNEL); in copy_module_elf()
2093 if (mod->klp_info->secstrings == NULL) { in copy_module_elf()
2100 mod->klp_info->symndx = symndx; in copy_module_elf()
2108 mod->klp_info->sechdrs[symndx].sh_addr = \ in copy_module_elf()
2109 (unsigned long) mod->core_kallsyms.symtab; in copy_module_elf()
2114 kfree(mod->klp_info->sechdrs); in copy_module_elf()
2116 kfree(mod->klp_info); in copy_module_elf()
2120 static void free_module_elf(struct module *mod) in free_module_elf() argument
2122 kfree(mod->klp_info->sechdrs); in free_module_elf()
2123 kfree(mod->klp_info->secstrings); in free_module_elf()
2124 kfree(mod->klp_info); in free_module_elf()
2127 static int copy_module_elf(struct module *mod, struct load_info *info) in copy_module_elf() argument
2132 static void free_module_elf(struct module *mod) in free_module_elf() argument
2147 void __weak module_arch_cleanup(struct module *mod) in module_arch_cleanup() argument
2151 void __weak module_arch_freeing_init(struct module *mod) in module_arch_freeing_init() argument
2155 static void cfi_cleanup(struct module *mod);
2158 static void free_module(struct module *mod) in free_module() argument
2160 trace_module_free(mod); in free_module()
2162 mod_sysfs_teardown(mod); in free_module()
2169 mod->state = MODULE_STATE_UNFORMED; in free_module()
2173 ddebug_remove_module(mod->name); in free_module()
2176 module_arch_cleanup(mod); in free_module()
2179 module_unload_free(mod); in free_module()
2182 destroy_params(mod->kp, mod->num_kp); in free_module()
2184 if (is_livepatch_module(mod)) in free_module()
2185 free_module_elf(mod); in free_module()
2190 list_del_rcu(&mod->list); in free_module()
2191 mod_tree_remove(mod); in free_module()
2193 module_bug_cleanup(mod); in free_module()
2199 cfi_cleanup(mod); in free_module()
2202 module_arch_freeing_init(mod); in free_module()
2203 module_memfree(mod->init_layout.base); in free_module()
2204 kfree(mod->args); in free_module()
2205 percpu_modfree(mod); in free_module()
2208 lockdep_free_key_range(mod->core_layout.base, mod->core_layout.size); in free_module()
2211 module_memfree(mod->core_layout.base); in free_module()
2238 static int verify_exported_symbols(struct module *mod) in verify_exported_symbols() argument
2246 { mod->syms, mod->num_syms }, in verify_exported_symbols()
2247 { mod->gpl_syms, mod->num_gpl_syms }, in verify_exported_symbols()
2259 mod->name, kernel_symbol_name(s), in verify_exported_symbols()
2284 static int simplify_symbols(struct module *mod, const struct load_info *info) in simplify_symbols() argument
2308 mod->name); in simplify_symbols()
2323 ksym = resolve_symbol_wait(mod, info, name); in simplify_symbols()
2338 mod->name, name, ret); in simplify_symbols()
2344 secbase = (unsigned long)mod_percpu(mod); in simplify_symbols()
2355 static int apply_relocations(struct module *mod, const struct load_info *info) in apply_relocations() argument
2373 err = klp_apply_section_relocs(mod, info->sechdrs, in apply_relocations()
2380 info->index.sym, i, mod); in apply_relocations()
2383 info->index.sym, i, mod); in apply_relocations()
2391 unsigned int __weak arch_mod_section_prepend(struct module *mod, in arch_mod_section_prepend() argument
2399 static long get_offset(struct module *mod, unsigned int *size, in get_offset() argument
2404 *size += arch_mod_section_prepend(mod, section); in get_offset()
2425 static void layout_sections(struct module *mod, struct load_info *info) in layout_sections() argument
2455 s->sh_entsize = get_offset(mod, &mod->core_layout.size, s, i); in layout_sections()
2460 mod->core_layout.size = debug_align(mod->core_layout.size); in layout_sections()
2461 mod->core_layout.text_size = mod->core_layout.size; in layout_sections()
2464 mod->core_layout.size = debug_align(mod->core_layout.size); in layout_sections()
2465 mod->core_layout.ro_size = mod->core_layout.size; in layout_sections()
2468 mod->core_layout.size = debug_align(mod->core_layout.size); in layout_sections()
2469 mod->core_layout.ro_after_init_size = mod->core_layout.size; in layout_sections()
2472 mod->core_layout.size = debug_align(mod->core_layout.size); in layout_sections()
2488 s->sh_entsize = (get_offset(mod, &mod->init_layout.size, s, i) in layout_sections()
2494 mod->init_layout.size = debug_align(mod->init_layout.size); in layout_sections()
2495 mod->init_layout.text_size = mod->init_layout.size; in layout_sections()
2498 mod->init_layout.size = debug_align(mod->init_layout.size); in layout_sections()
2499 mod->init_layout.ro_size = mod->init_layout.size; in layout_sections()
2506 mod->init_layout.ro_after_init_size = mod->init_layout.ro_size; in layout_sections()
2509 mod->init_layout.size = debug_align(mod->init_layout.size); in layout_sections()
2515 static void set_license(struct module *mod, const char *license) in set_license() argument
2523 mod->name, license); in set_license()
2524 add_taint_module(mod, TAINT_PROPRIETARY_MODULE, in set_license()
2579 static void setup_modinfo(struct module *mod, struct load_info *info) in setup_modinfo() argument
2586 attr->setup(mod, get_modinfo(info, attr->attr.name)); in setup_modinfo()
2590 static void free_modinfo(struct module *mod) in free_modinfo() argument
2597 attr->free(mod); in free_modinfo()
2613 const struct module *mod) in is_exported() argument
2616 if (!mod) in is_exported()
2619 ks = lookup_exported_symbol(name, mod->syms, mod->syms + mod->num_syms); in is_exported()
2698 static void layout_symtab(struct module *mod, struct load_info *info) in layout_symtab() argument
2707 symsect->sh_entsize = get_offset(mod, &mod->init_layout.size, symsect, in layout_symtab()
2716 if (i == 0 || is_livepatch_module(mod) || in layout_symtab()
2725 info->symoffs = ALIGN(mod->core_layout.size, symsect->sh_addralign ?: 1); in layout_symtab()
2726 info->stroffs = mod->core_layout.size = info->symoffs + ndst * sizeof(Elf_Sym); in layout_symtab()
2727 mod->core_layout.size += strtab_size; in layout_symtab()
2728 info->core_typeoffs = mod->core_layout.size; in layout_symtab()
2729 mod->core_layout.size += ndst * sizeof(char); in layout_symtab()
2730 mod->core_layout.size = debug_align(mod->core_layout.size); in layout_symtab()
2734 strsect->sh_entsize = get_offset(mod, &mod->init_layout.size, strsect, in layout_symtab()
2739 mod->init_layout.size = ALIGN(mod->init_layout.size, in layout_symtab()
2741 info->mod_kallsyms_init_off = mod->init_layout.size; in layout_symtab()
2742 mod->init_layout.size += sizeof(struct mod_kallsyms); in layout_symtab()
2743 info->init_typeoffs = mod->init_layout.size; in layout_symtab()
2744 mod->init_layout.size += nsrc * sizeof(char); in layout_symtab()
2745 mod->init_layout.size = debug_align(mod->init_layout.size); in layout_symtab()
2753 static void add_kallsyms(struct module *mod, const struct load_info *info) in add_kallsyms() argument
2762 mod->kallsyms = mod->init_layout.base + info->mod_kallsyms_init_off; in add_kallsyms()
2764 mod->kallsyms->symtab = (void *)symsec->sh_addr; in add_kallsyms()
2765 mod->kallsyms->num_symtab = symsec->sh_size / sizeof(Elf_Sym); in add_kallsyms()
2767 mod->kallsyms->strtab = (void *)info->sechdrs[info->index.str].sh_addr; in add_kallsyms()
2768 mod->kallsyms->typetab = mod->init_layout.base + info->init_typeoffs; in add_kallsyms()
2774 mod->core_kallsyms.symtab = dst = mod->core_layout.base + info->symoffs; in add_kallsyms()
2775 mod->core_kallsyms.strtab = s = mod->core_layout.base + info->stroffs; in add_kallsyms()
2776 mod->core_kallsyms.typetab = mod->core_layout.base + info->core_typeoffs; in add_kallsyms()
2777 src = mod->kallsyms->symtab; in add_kallsyms()
2778 for (ndst = i = 0; i < mod->kallsyms->num_symtab; i++) { in add_kallsyms()
2779 mod->kallsyms->typetab[i] = elf_type(src + i, info); in add_kallsyms()
2780 if (i == 0 || is_livepatch_module(mod) || in add_kallsyms()
2783 mod->core_kallsyms.typetab[ndst] = in add_kallsyms()
2784 mod->kallsyms->typetab[i]; in add_kallsyms()
2786 dst[ndst++].st_name = s - mod->core_kallsyms.strtab; in add_kallsyms()
2787 s += strlcpy(s, &mod->kallsyms->strtab[src[i].st_name], in add_kallsyms()
2791 mod->core_kallsyms.num_symtab = ndst; in add_kallsyms()
2794 static inline void layout_symtab(struct module *mod, struct load_info *info) in layout_symtab() argument
2798 static void add_kallsyms(struct module *mod, const struct load_info *info) in add_kallsyms() argument
2804 static void init_build_id(struct module *mod, const struct load_info *info) in init_build_id() argument
2812 !build_id_parse_buf((void *)sechdr->sh_addr, mod->build_id, in init_build_id()
2818 static void init_build_id(struct module *mod, const struct load_info *info) in init_build_id() argument
2823 static void dynamic_debug_setup(struct module *mod, struct _ddebug *debug, unsigned int num) in dynamic_debug_setup() argument
2827 ddebug_add_module(debug, num, mod->name); in dynamic_debug_setup()
2830 static void dynamic_debug_remove(struct module *mod, struct _ddebug *debug) in dynamic_debug_remove() argument
2833 ddebug_remove_module(mod->name); in dynamic_debug_remove()
2854 static void kmemleak_load_module(const struct module *mod, in kmemleak_load_module() argument
2860 kmemleak_scan_area(mod, sizeof(struct module), GFP_KERNEL); in kmemleak_load_module()
2874 static inline void kmemleak_load_module(const struct module *mod, in kmemleak_load_module() argument
2886 const void *mod = info->hdr; in module_sig_check() local
2894 memcmp(mod + info->len - markerlen, MODULE_SIG_STRING, markerlen) == 0) { in module_sig_check()
2897 err = mod_verify_sig(mod, info); in module_sig_check()
3111 static int check_modinfo_livepatch(struct module *mod, struct load_info *info) in check_modinfo_livepatch() argument
3114 mod->klp = true; in check_modinfo_livepatch()
3115 add_taint_module(mod, TAINT_LIVEPATCH, LOCKDEP_STILL_OK); in check_modinfo_livepatch()
3117 mod->name); in check_modinfo_livepatch()
3123 static int check_modinfo_livepatch(struct module *mod, struct load_info *info) in check_modinfo_livepatch() argument
3127 mod->name); in check_modinfo_livepatch()
3135 static void check_modinfo_retpoline(struct module *mod, struct load_info *info) in check_modinfo_retpoline() argument
3141 mod->name); in check_modinfo_retpoline()
3241 info->index.mod = find_sec(info, ".gnu.linkonce.this_module"); in setup_load_info()
3242 if (!info->index.mod) { in setup_load_info()
3248 info->mod = (void *)info->hdr + info->sechdrs[info->index.mod].sh_offset; in setup_load_info()
3255 info->name = info->mod->name; in setup_load_info()
3267 static int check_modinfo(struct module *mod, struct load_info *info, int flags) in check_modinfo() argument
3277 err = try_to_force_load(mod, "bad vermagic"); in check_modinfo()
3289 mod->name); in check_modinfo()
3290 add_taint_module(mod, TAINT_OOT_MODULE, LOCKDEP_STILL_OK); in check_modinfo()
3293 check_modinfo_retpoline(mod, info); in check_modinfo()
3296 add_taint_module(mod, TAINT_CRAP, LOCKDEP_STILL_OK); in check_modinfo()
3298 "is unknown, you have been warned.\n", mod->name); in check_modinfo()
3301 err = check_modinfo_livepatch(mod, info); in check_modinfo()
3306 set_license(mod, get_modinfo(info, "license")); in check_modinfo()
3311 static int find_module_sections(struct module *mod, struct load_info *info) in find_module_sections() argument
3313 mod->kp = section_objs(info, "__param", in find_module_sections()
3314 sizeof(*mod->kp), &mod->num_kp); in find_module_sections()
3315 mod->syms = section_objs(info, "__ksymtab", in find_module_sections()
3316 sizeof(*mod->syms), &mod->num_syms); in find_module_sections()
3317 mod->crcs = section_addr(info, "__kcrctab"); in find_module_sections()
3318 mod->gpl_syms = section_objs(info, "__ksymtab_gpl", in find_module_sections()
3319 sizeof(*mod->gpl_syms), in find_module_sections()
3320 &mod->num_gpl_syms); in find_module_sections()
3321 mod->gpl_crcs = section_addr(info, "__kcrctab_gpl"); in find_module_sections()
3324 mod->ctors = section_objs(info, ".ctors", in find_module_sections()
3325 sizeof(*mod->ctors), &mod->num_ctors); in find_module_sections()
3326 if (!mod->ctors) in find_module_sections()
3327 mod->ctors = section_objs(info, ".init_array", in find_module_sections()
3328 sizeof(*mod->ctors), &mod->num_ctors); in find_module_sections()
3335 mod->name); in find_module_sections()
3340 mod->noinstr_text_start = section_objs(info, ".noinstr.text", 1, in find_module_sections()
3341 &mod->noinstr_text_size); in find_module_sections()
3344 mod->tracepoints_ptrs = section_objs(info, "__tracepoints_ptrs", in find_module_sections()
3345 sizeof(*mod->tracepoints_ptrs), in find_module_sections()
3346 &mod->num_tracepoints); in find_module_sections()
3349 mod->srcu_struct_ptrs = section_objs(info, "___srcu_struct_ptrs", in find_module_sections()
3350 sizeof(*mod->srcu_struct_ptrs), in find_module_sections()
3351 &mod->num_srcu_structs); in find_module_sections()
3354 mod->bpf_raw_events = section_objs(info, "__bpf_raw_tp_map", in find_module_sections()
3355 sizeof(*mod->bpf_raw_events), in find_module_sections()
3356 &mod->num_bpf_raw_events); in find_module_sections()
3359 mod->btf_data = any_section_objs(info, ".BTF", 1, &mod->btf_data_size); in find_module_sections()
3362 mod->jump_entries = section_objs(info, "__jump_table", in find_module_sections()
3363 sizeof(*mod->jump_entries), in find_module_sections()
3364 &mod->num_jump_entries); in find_module_sections()
3367 mod->trace_events = section_objs(info, "_ftrace_events", in find_module_sections()
3368 sizeof(*mod->trace_events), in find_module_sections()
3369 &mod->num_trace_events); in find_module_sections()
3370 mod->trace_evals = section_objs(info, "_ftrace_eval_map", in find_module_sections()
3371 sizeof(*mod->trace_evals), in find_module_sections()
3372 &mod->num_trace_evals); in find_module_sections()
3375 mod->trace_bprintk_fmt_start = section_objs(info, "__trace_printk_fmt", in find_module_sections()
3376 sizeof(*mod->trace_bprintk_fmt_start), in find_module_sections()
3377 &mod->num_trace_bprintk_fmt); in find_module_sections()
3381 mod->ftrace_callsites = section_objs(info, FTRACE_CALLSITE_SECTION, in find_module_sections()
3382 sizeof(*mod->ftrace_callsites), in find_module_sections()
3383 &mod->num_ftrace_callsites); in find_module_sections()
3386 mod->ei_funcs = section_objs(info, "_error_injection_whitelist", in find_module_sections()
3387 sizeof(*mod->ei_funcs), in find_module_sections()
3388 &mod->num_ei_funcs); in find_module_sections()
3391 mod->kprobes_text_start = section_objs(info, ".kprobes.text", 1, in find_module_sections()
3392 &mod->kprobes_text_size); in find_module_sections()
3393 mod->kprobe_blacklist = section_objs(info, "_kprobe_blacklist", in find_module_sections()
3395 &mod->num_kprobe_blacklist); in find_module_sections()
3398 mod->printk_index_start = section_objs(info, ".printk_index", in find_module_sections()
3399 sizeof(*mod->printk_index_start), in find_module_sections()
3400 &mod->printk_index_size); in find_module_sections()
3403 mod->static_call_sites = section_objs(info, ".static_call_sites", in find_module_sections()
3404 sizeof(*mod->static_call_sites), in find_module_sections()
3405 &mod->num_static_call_sites); in find_module_sections()
3407 mod->extable = section_objs(info, "__ex_table", in find_module_sections()
3408 sizeof(*mod->extable), &mod->num_exentries); in find_module_sections()
3411 pr_warn("%s: Ignoring obsolete parameters\n", mod->name); in find_module_sections()
3419 static int move_module(struct module *mod, struct load_info *info) in move_module() argument
3425 ptr = module_alloc(mod->core_layout.size); in move_module()
3435 memset(ptr, 0, mod->core_layout.size); in move_module()
3436 mod->core_layout.base = ptr; in move_module()
3438 if (mod->init_layout.size) { in move_module()
3439 ptr = module_alloc(mod->init_layout.size); in move_module()
3448 module_memfree(mod->core_layout.base); in move_module()
3451 memset(ptr, 0, mod->init_layout.size); in move_module()
3452 mod->init_layout.base = ptr; in move_module()
3454 mod->init_layout.base = NULL; in move_module()
3466 dest = mod->init_layout.base in move_module()
3469 dest = mod->core_layout.base + shdr->sh_entsize; in move_module()
3482 static int check_module_license_and_versions(struct module *mod) in check_module_license_and_versions() argument
3491 if (strcmp(mod->name, "ndiswrapper") == 0) in check_module_license_and_versions()
3495 if (strcmp(mod->name, "driverloader") == 0) in check_module_license_and_versions()
3496 add_taint_module(mod, TAINT_PROPRIETARY_MODULE, in check_module_license_and_versions()
3500 if (strcmp(mod->name, "lve") == 0) in check_module_license_and_versions()
3501 add_taint_module(mod, TAINT_PROPRIETARY_MODULE, in check_module_license_and_versions()
3505 pr_warn("%s: module license taints kernel.\n", mod->name); in check_module_license_and_versions()
3508 if ((mod->num_syms && !mod->crcs) || in check_module_license_and_versions()
3509 (mod->num_gpl_syms && !mod->gpl_crcs)) { in check_module_license_and_versions()
3510 return try_to_force_load(mod, in check_module_license_and_versions()
3517 static void flush_module_icache(const struct module *mod) in flush_module_icache() argument
3524 if (mod->init_layout.base) in flush_module_icache()
3525 flush_icache_range((unsigned long)mod->init_layout.base, in flush_module_icache()
3526 (unsigned long)mod->init_layout.base in flush_module_icache()
3527 + mod->init_layout.size); in flush_module_icache()
3528 flush_icache_range((unsigned long)mod->core_layout.base, in flush_module_icache()
3529 (unsigned long)mod->core_layout.base + mod->core_layout.size); in flush_module_icache()
3535 struct module *mod) in module_frob_arch_sections() argument
3563 struct module *mod; in layout_and_allocate() local
3567 err = check_modinfo(info->mod, info, flags); in layout_and_allocate()
3573 info->secstrings, info->mod); in layout_and_allocate()
3578 info->secstrings, info->mod); in layout_and_allocate()
3608 layout_sections(info->mod, info); in layout_and_allocate()
3609 layout_symtab(info->mod, info); in layout_and_allocate()
3612 err = move_module(info->mod, info); in layout_and_allocate()
3617 mod = (void *)info->sechdrs[info->index.mod].sh_addr; in layout_and_allocate()
3618 kmemleak_load_module(mod, info); in layout_and_allocate()
3619 return mod; in layout_and_allocate()
3623 static void module_deallocate(struct module *mod, struct load_info *info) in module_deallocate() argument
3625 percpu_modfree(mod); in module_deallocate()
3626 module_arch_freeing_init(mod); in module_deallocate()
3627 module_memfree(mod->init_layout.base); in module_deallocate()
3628 module_memfree(mod->core_layout.base); in module_deallocate()
3638 static int post_relocation(struct module *mod, const struct load_info *info) in post_relocation() argument
3641 sort_extable(mod->extable, mod->extable + mod->num_exentries); in post_relocation()
3644 percpu_modcopy(mod, (void *)info->sechdrs[info->index.pcpu].sh_addr, in post_relocation()
3648 add_kallsyms(mod, info); in post_relocation()
3651 return module_finalize(info->hdr, info->sechdrs, mod); in post_relocation()
3657 struct module *mod; in finished_loading() local
3667 mod = find_module_all(name, strlen(name), true); in finished_loading()
3668 ret = !mod || mod->state == MODULE_STATE_LIVE; in finished_loading()
3675 static void do_mod_ctors(struct module *mod) in do_mod_ctors() argument
3680 for (i = 0; i < mod->num_ctors; i++) in do_mod_ctors()
3681 mod->ctors[i](); in do_mod_ctors()
3713 static noinline int do_init_module(struct module *mod) in do_init_module() argument
3723 freeinit->module_init = mod->init_layout.base; in do_init_module()
3731 do_mod_ctors(mod); in do_init_module()
3733 if (mod->init != NULL) in do_init_module()
3734 ret = do_one_initcall(mod->init); in do_init_module()
3742 __func__, mod->name, ret, __func__); in do_init_module()
3747 mod->state = MODULE_STATE_LIVE; in do_init_module()
3749 MODULE_STATE_LIVE, mod); in do_init_module()
3752 kobject_uevent(&mod->mkobj.kobj, KOBJ_ADD); in do_init_module()
3771 if (!mod->async_probe_requested && (current->flags & PF_USED_ASYNC)) in do_init_module()
3774 ftrace_free_mem(mod, mod->init_layout.base, mod->init_layout.base + in do_init_module()
3775 mod->init_layout.size); in do_init_module()
3778 module_put(mod); in do_init_module()
3779 trim_init_extable(mod); in do_init_module()
3782 rcu_assign_pointer(mod->kallsyms, &mod->core_kallsyms); in do_init_module()
3784 module_enable_ro(mod, true); in do_init_module()
3785 mod_tree_remove_init(mod); in do_init_module()
3786 module_arch_freeing_init(mod); in do_init_module()
3787 mod->init_layout.base = NULL; in do_init_module()
3788 mod->init_layout.size = 0; in do_init_module()
3789 mod->init_layout.ro_size = 0; in do_init_module()
3790 mod->init_layout.ro_after_init_size = 0; in do_init_module()
3791 mod->init_layout.text_size = 0; in do_init_module()
3794 mod->btf_data = NULL; in do_init_module()
3821 mod->state = MODULE_STATE_GOING; in do_init_module()
3823 module_put(mod); in do_init_module()
3825 MODULE_STATE_GOING, mod); in do_init_module()
3826 klp_module_going(mod); in do_init_module()
3827 ftrace_release_mod(mod); in do_init_module()
3828 free_module(mod); in do_init_module()
3846 static int add_unformed_module(struct module *mod) in add_unformed_module() argument
3851 mod->state = MODULE_STATE_UNFORMED; in add_unformed_module()
3855 old = find_module_all(mod->name, strlen(mod->name), true); in add_unformed_module()
3861 finished_loading(mod->name)); in add_unformed_module()
3869 mod_update_bounds(mod); in add_unformed_module()
3870 list_add_rcu(&mod->list, &modules); in add_unformed_module()
3871 mod_tree_insert(mod); in add_unformed_module()
3880 static int complete_formation(struct module *mod, struct load_info *info) in complete_formation() argument
3887 err = verify_exported_symbols(mod); in complete_formation()
3892 module_bug_finalize(info->hdr, info->sechdrs, mod); in complete_formation()
3894 module_enable_ro(mod, false); in complete_formation()
3895 module_enable_nx(mod); in complete_formation()
3896 module_enable_x(mod); in complete_formation()
3902 mod->state = MODULE_STATE_COMING; in complete_formation()
3912 static int prepare_coming_module(struct module *mod) in prepare_coming_module() argument
3916 ftrace_module_enable(mod); in prepare_coming_module()
3917 err = klp_module_coming(mod); in prepare_coming_module()
3922 MODULE_STATE_COMING, MODULE_STATE_GOING, mod); in prepare_coming_module()
3925 klp_module_going(mod); in prepare_coming_module()
3933 struct module *mod = arg; in unknown_module_param_cb() local
3937 mod->async_probe_requested = true; in unknown_module_param_cb()
3948 static void cfi_init(struct module *mod);
3957 struct module *mod; in load_module() local
4008 if (!check_modstruct_version(info, info->mod)) { in load_module()
4014 mod = layout_and_allocate(info, flags); in load_module()
4015 if (IS_ERR(mod)) { in load_module()
4016 err = PTR_ERR(mod); in load_module()
4020 audit_log_kern_module(mod->name); in load_module()
4023 err = add_unformed_module(mod); in load_module()
4028 mod->sig_ok = info->sig_ok; in load_module()
4029 if (!mod->sig_ok) { in load_module()
4032 "kernel\n", mod->name); in load_module()
4033 add_taint_module(mod, TAINT_UNSIGNED_MODULE, LOCKDEP_STILL_OK); in load_module()
4038 err = percpu_modalloc(mod, info); in load_module()
4043 err = module_unload_init(mod); in load_module()
4047 init_param_lock(mod); in load_module()
4053 err = find_module_sections(mod, info); in load_module()
4057 err = check_module_license_and_versions(mod); in load_module()
4062 setup_modinfo(mod, info); in load_module()
4065 err = simplify_symbols(mod, info); in load_module()
4069 err = apply_relocations(mod, info); in load_module()
4073 err = post_relocation(mod, info); in load_module()
4077 flush_module_icache(mod); in load_module()
4080 cfi_init(mod); in load_module()
4083 mod->args = strndup_user(uargs, ~0UL >> 1); in load_module()
4084 if (IS_ERR(mod->args)) { in load_module()
4085 err = PTR_ERR(mod->args); in load_module()
4089 init_build_id(mod, info); in load_module()
4090 dynamic_debug_setup(mod, info->debug, info->num_debug); in load_module()
4093 ftrace_module_init(mod); in load_module()
4096 err = complete_formation(mod, info); in load_module()
4100 err = prepare_coming_module(mod); in load_module()
4105 after_dashes = parse_args(mod->name, mod->args, mod->kp, mod->num_kp, in load_module()
4106 -32768, 32767, mod, in load_module()
4113 mod->name, after_dashes); in load_module()
4117 err = mod_sysfs_setup(mod, info, mod->kp, mod->num_kp); in load_module()
4121 if (is_livepatch_module(mod)) { in load_module()
4122 err = copy_module_elf(mod, info); in load_module()
4131 trace_module_load(mod); in load_module()
4133 return do_init_module(mod); in load_module()
4136 mod_sysfs_teardown(mod); in load_module()
4138 mod->state = MODULE_STATE_GOING; in load_module()
4139 destroy_params(mod->kp, mod->num_kp); in load_module()
4141 MODULE_STATE_GOING, mod); in load_module()
4142 klp_module_going(mod); in load_module()
4144 mod->state = MODULE_STATE_GOING; in load_module()
4147 module_bug_cleanup(mod); in load_module()
4151 ftrace_release_mod(mod); in load_module()
4152 dynamic_debug_remove(mod, info->debug); in load_module()
4154 kfree(mod->args); in load_module()
4156 cfi_cleanup(mod); in load_module()
4157 module_arch_cleanup(mod); in load_module()
4159 free_modinfo(mod); in load_module()
4161 module_unload_free(mod); in load_module()
4165 list_del_rcu(&mod->list); in load_module()
4166 mod_tree_remove(mod); in load_module()
4173 lockdep_free_key_range(mod->core_layout.base, mod->core_layout.size); in load_module()
4175 module_deallocate(mod, info); in load_module()
4254 static const char *find_kallsyms_symbol(struct module *mod, in find_kallsyms_symbol() argument
4261 struct mod_kallsyms *kallsyms = rcu_dereference_sched(mod->kallsyms); in find_kallsyms_symbol()
4264 if (within_module_init(addr, mod)) in find_kallsyms_symbol()
4265 nextval = (unsigned long)mod->init_layout.base+mod->init_layout.text_size; in find_kallsyms_symbol()
4267 nextval = (unsigned long)mod->core_layout.base+mod->core_layout.text_size; in find_kallsyms_symbol()
4309 void * __weak dereference_module_function_descriptor(struct module *mod, in dereference_module_function_descriptor() argument
4327 struct module *mod; in module_address_lookup() local
4330 mod = __module_address(addr); in module_address_lookup()
4331 if (mod) { in module_address_lookup()
4333 *modname = mod->name; in module_address_lookup()
4336 *modbuildid = mod->build_id; in module_address_lookup()
4342 ret = find_kallsyms_symbol(mod, addr, size, offset); in module_address_lookup()
4356 struct module *mod; in lookup_module_symbol_name() local
4359 list_for_each_entry_rcu(mod, &modules, list) { in lookup_module_symbol_name()
4360 if (mod->state == MODULE_STATE_UNFORMED) in lookup_module_symbol_name()
4362 if (within_module(addr, mod)) { in lookup_module_symbol_name()
4365 sym = find_kallsyms_symbol(mod, addr, NULL, NULL); in lookup_module_symbol_name()
4382 struct module *mod; in lookup_module_symbol_attrs() local
4385 list_for_each_entry_rcu(mod, &modules, list) { in lookup_module_symbol_attrs()
4386 if (mod->state == MODULE_STATE_UNFORMED) in lookup_module_symbol_attrs()
4388 if (within_module(addr, mod)) { in lookup_module_symbol_attrs()
4391 sym = find_kallsyms_symbol(mod, addr, size, offset); in lookup_module_symbol_attrs()
4395 strlcpy(modname, mod->name, MODULE_NAME_LEN); in lookup_module_symbol_attrs()
4410 struct module *mod; in module_get_kallsym() local
4413 list_for_each_entry_rcu(mod, &modules, list) { in module_get_kallsym()
4416 if (mod->state == MODULE_STATE_UNFORMED) in module_get_kallsym()
4418 kallsyms = rcu_dereference_sched(mod->kallsyms); in module_get_kallsym()
4425 strlcpy(module_name, mod->name, MODULE_NAME_LEN); in module_get_kallsym()
4426 *exported = is_exported(name, *value, mod); in module_get_kallsym()
4437 static unsigned long find_kallsyms_symbol_value(struct module *mod, const char *name) in find_kallsyms_symbol_value() argument
4440 struct mod_kallsyms *kallsyms = rcu_dereference_sched(mod->kallsyms); in find_kallsyms_symbol_value()
4455 struct module *mod; in module_kallsyms_lookup_name() local
4462 if ((mod = find_module_all(name, colon - name, false)) != NULL) in module_kallsyms_lookup_name()
4463 ret = find_kallsyms_symbol_value(mod, colon+1); in module_kallsyms_lookup_name()
4465 list_for_each_entry_rcu(mod, &modules, list) { in module_kallsyms_lookup_name()
4466 if (mod->state == MODULE_STATE_UNFORMED) in module_kallsyms_lookup_name()
4468 if ((ret = find_kallsyms_symbol_value(mod, name)) != 0) in module_kallsyms_lookup_name()
4481 struct module *mod; in module_kallsyms_on_each_symbol() local
4486 list_for_each_entry(mod, &modules, list) { in module_kallsyms_on_each_symbol()
4488 struct mod_kallsyms *kallsyms = mod->kallsyms; in module_kallsyms_on_each_symbol()
4490 if (mod->state == MODULE_STATE_UNFORMED) in module_kallsyms_on_each_symbol()
4499 mod, kallsyms_symbol_value(sym)); in module_kallsyms_on_each_symbol()
4511 static void cfi_init(struct module *mod) in cfi_init() argument
4518 mod->cfi_check = (cfi_check_fn) in cfi_init()
4519 find_kallsyms_symbol_value(mod, "__cfi_check"); in cfi_init()
4521 find_kallsyms_symbol_value(mod, "__cfi_jt_init_module"); in cfi_init()
4523 find_kallsyms_symbol_value(mod, "__cfi_jt_cleanup_module"); in cfi_init()
4528 mod->init = *init; in cfi_init()
4531 mod->exit = *exit; in cfi_init()
4534 cfi_module_add(mod, module_addr_min); in cfi_init()
4538 static void cfi_cleanup(struct module *mod) in cfi_cleanup() argument
4541 cfi_module_remove(mod, module_addr_min); in cfi_cleanup()
4549 static char *module_flags(struct module *mod, char *buf) in module_flags() argument
4553 BUG_ON(mod->state == MODULE_STATE_UNFORMED); in module_flags()
4554 if (mod->taints || in module_flags()
4555 mod->state == MODULE_STATE_GOING || in module_flags()
4556 mod->state == MODULE_STATE_COMING) { in module_flags()
4558 bx += module_flags_taint(mod, buf + bx); in module_flags()
4560 if (mod->state == MODULE_STATE_GOING) in module_flags()
4563 if (mod->state == MODULE_STATE_COMING) in module_flags()
4592 struct module *mod = list_entry(p, struct module, list); in m_show() local
4597 if (mod->state == MODULE_STATE_UNFORMED) in m_show()
4601 mod->name, mod->init_layout.size + mod->core_layout.size); in m_show()
4602 print_unload_info(m, mod); in m_show()
4606 mod->state == MODULE_STATE_GOING ? "Unloading" : in m_show()
4607 mod->state == MODULE_STATE_COMING ? "Loading" : in m_show()
4610 value = m->private ? NULL : mod->core_layout.base; in m_show()
4614 if (mod->taints) in m_show()
4615 seq_printf(m, " %s", module_flags(mod, buf)); in m_show()
4673 struct module *mod; in search_module_extables() local
4676 mod = __module_address(addr); in search_module_extables()
4677 if (!mod) in search_module_extables()
4680 if (!mod->num_exentries) in search_module_extables()
4683 e = search_extable(mod->extable, in search_module_extables()
4684 mod->num_exentries, in search_module_extables()
4723 struct module *mod; in __module_address() local
4730 mod = mod_find(addr); in __module_address()
4731 if (mod) { in __module_address()
4732 BUG_ON(!within_module(addr, mod)); in __module_address()
4733 if (mod->state == MODULE_STATE_UNFORMED) in __module_address()
4734 mod = NULL; in __module_address()
4736 return mod; in __module_address()
4767 struct module *mod = __module_address(addr); in __module_text_address() local
4768 if (mod) { in __module_text_address()
4770 if (!within(addr, mod->init_layout.base, mod->init_layout.text_size) in __module_text_address()
4771 && !within(addr, mod->core_layout.base, mod->core_layout.text_size)) in __module_text_address()
4772 mod = NULL; in __module_text_address()
4774 return mod; in __module_text_address()
4780 struct module *mod; in print_modules() local
4786 list_for_each_entry_rcu(mod, &modules, list) { in print_modules()
4787 if (mod->state == MODULE_STATE_UNFORMED) in print_modules()
4789 pr_cont(" %s%s", mod->name, module_flags(mod, buf)); in print_modules()
4802 void module_layout(struct module *mod, in module_layout() argument