#define INCLUDE_VERMAGIC

#include <linux/export.h>

#include <linux/extable.h>

#include <linux/moduleloader.h>

#include <linux/module_signature.h>

#include <linux/trace_events.h>

#include <linux/init.h>

#include <linux/kallsyms.h>

#include <linux/buildid.h>

#include <linux/fs.h>

#include <linux/kernel.h>

#include <linux/kernel_read_file.h>

#include <linux/kstrtox.h>

#include <linux/slab.h>

#include <linux/vmalloc.h>

#include <linux/elf.h>

#include <linux/seq_file.h>

#include <linux/syscalls.h>

#include <linux/fcntl.h>

#include <linux/rcupdate.h>

#include <linux/capability.h>

#include <linux/cpu.h>

#include <linux/moduleparam.h>

#include <linux/errno.h>

#include <linux/err.h>

#include <linux/vermagic.h>

#include <linux/notifier.h>

#include <linux/sched.h>

#include <linux/device.h>

#include <linux/string.h>

#include <linux/mutex.h>

#include <linux/rculist.h>

#include <linux/uaccess.h>

#include <asm/cacheflush.h>

#include <linux/set_memory.h>

#include <asm/mmu_context.h>

#include <linux/license.h>

#include <asm/sections.h>

#include <linux/tracepoint.h>

#include <linux/ftrace.h>

#include <linux/livepatch.h>

#include <linux/async.h>

#include <linux/percpu.h>

#include <linux/kmemleak.h>

#include <linux/jump_label.h>

#include <linux/pfn.h>

#include <linux/bsearch.h>

#include <linux/dynamic_debug.h>

#include <linux/audit.h>

#include <linux/cfi.h>

#include <linux/codetag.h>

#include <linux/debugfs.h>

#include <linux/execmem.h>

#include <uapi/linux/module.h>

#include "internal.h"

#define CREATE_TRACE_POINTS

#include <trace/events/module.h>

DEFINE_MUTEX(module_mutex);

LIST_HEAD(modules);

static void do_free_init(struct work_struct *w);

static DECLARE_WORK(init_free_wq, do_free_init);

static LLIST_HEAD(init_free_list);

struct mod_tree_root mod_tree __cacheline_aligned = {

.addr_min = -1UL,

};

struct symsearch {

};

static void __mod_update_bounds(enum mod_mem_type type __maybe_unused, void *base,

unsigned int size, struct mod_tree_root *tree)

{

unsigned long min = (unsigned long)base;

unsigned long max = min + size;

#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC

if (mod_mem_type_is_core_data(type)) {

if (min < tree->data_addr_min)

tree->data_addr_min = min;

if (max > tree->data_addr_max)

tree->data_addr_max = max;

return;

}

if (min < tree->addr_min)

tree->addr_min = min;

if (max > tree->addr_max)

tree->addr_max = max;

}

static void mod_update_bounds(struct module *mod)

{

for_each_mod_mem_type(type) {

struct module_memory *mod_mem = &mod->mem[type];

if (mod_mem->size)

__mod_update_bounds(type, mod_mem->base, mod_mem->size, &mod_tree);

}

}

int modules_disabled;

core_param(nomodule, modules_disabled, bint, 0);

static DECLARE_WAIT_QUEUE_HEAD(module_wq);

static BLOCKING_NOTIFIER_HEAD(module_notify_list);

int register_module_notifier(struct notifier_block *nb)

{

return blocking_notifier_chain_register(&module_notify_list, nb);

}

EXPORT_SYMBOL(register_module_notifier);

int unregister_module_notifier(struct notifier_block *nb)

{

return blocking_notifier_chain_unregister(&module_notify_list, nb);

}

EXPORT_SYMBOL(unregister_module_notifier);

static inline int strong_try_module_get(struct module *mod)

{

BUG_ON(mod && mod->state == MODULE_STATE_UNFORMED);

if (mod && mod->state == MODULE_STATE_COMING)

return -EBUSY;

if (try_module_get(mod))

return 0;

else

return -ENOENT;

}

static inline void add_taint_module(struct module *mod, unsigned flag,

enum lockdep_ok lockdep_ok)

{

add_taint(flag, lockdep_ok);

set_bit(flag, &mod->taints);

}

static int mod_strncmp(const char *str_a, const char *str_b, size_t n)

{

for (int i = 0; i < n; i++) {

char a = str_a[i];

char b = str_b[i];

int d;

if (a == '-') a = '_';

if (b == '-') b = '_';

d = a - b;

if (d)

return d;

if (!a)

break;

}

return 0;

}

void __noreturn __module_put_and_kthread_exit(struct module *mod, long code)

{

module_put(mod);

kthread_exit(code);

}

EXPORT_SYMBOL(__module_put_and_kthread_exit);

static unsigned int find_sec(const struct load_info *info, const char *name)

{

unsigned int i;

for (i = 1; i < info->hdr->e_shnum; i++) {

Elf_Shdr *shdr = &info->sechdrs[i];

/* Alloc bit cleared means "ignore it." */

if ((shdr->sh_flags & SHF_ALLOC)

&& strcmp(info->secstrings + shdr->sh_name, name) == 0)

return i;

}

return 0;

}

static int find_any_unique_sec(const struct load_info *info, const char *name)

{

unsigned int idx;

unsigned int count = 0;

int i;

for (i = 1; i < info->hdr->e_shnum; i++) {

if (strcmp(info->secstrings + info->sechdrs[i].sh_name,

name) == 0) {

count++;

idx = i;

}

}

if (count == 1) {

return idx;

} else if (count == 0) {

return 0;

} else {

return -count;

}

}

static void *section_addr(const struct load_info *info, const char *name)

{

/* Section 0 has sh_addr 0. */

return (void *)info->sechdrs[find_sec(info, name)].sh_addr;

}

static void *section_objs(const struct load_info *info,

const char *name,

size_t object_size,

unsigned int *num)

{

unsigned int sec = find_sec(info, name);

/* Section 0 has sh_addr 0 and sh_size 0. */

*num = info->sechdrs[sec].sh_size / object_size;

return (void *)info->sechdrs[sec].sh_addr;

}

static unsigned int find_any_sec(const struct load_info *info, const char *name)

{

unsigned int i;

for (i = 1; i < info->hdr->e_shnum; i++) {

Elf_Shdr *shdr = &info->sechdrs[i];

if (strcmp(info->secstrings + shdr->sh_name, name) == 0)

return i;

}

return 0;

}

static __maybe_unused void *any_section_objs(const struct load_info *info,

const char *name,

size_t object_size,

unsigned int *num)

{

unsigned int sec = find_any_sec(info, name);

/* Section 0 has sh_addr 0 and sh_size 0. */

*num = info->sechdrs[sec].sh_size / object_size;

return (void *)info->sechdrs[sec].sh_addr;

}

#ifndef CONFIG_MODVERSIONS

#define symversion(base, idx) NULL

#define symversion(base, idx) ((base != NULL) ? ((base) + (idx)) : NULL)

static const char *kernel_symbol_name(const struct kernel_symbol *sym)

{

#ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS

return offset_to_ptr(&sym->name_offset);

return sym->name;

}

static const char *kernel_symbol_namespace(const struct kernel_symbol *sym)

{

#ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS

if (!sym->namespace_offset)

return NULL;

return offset_to_ptr(&sym->namespace_offset);

return sym->namespace;

}

int cmp_name(const void *name, const void *sym)

{

return strcmp(name, kernel_symbol_name(sym));

}

static bool find_exported_symbol_in_section(const struct symsearch *syms,

struct module *owner,

struct find_symbol_arg *fsa)

{

struct kernel_symbol *sym;

if (!fsa->gplok && syms->license == GPL_ONLY)

return false;

sym = bsearch(fsa->name, syms->start, syms->stop - syms->start,

sizeof(struct kernel_symbol), cmp_name);

if (!sym)

return false;

fsa->owner = owner;

fsa->crc = symversion(syms->crcs, sym - syms->start);

fsa->sym = sym;

fsa->license = syms->license;

return true;

}

bool find_symbol(struct find_symbol_arg *fsa)

{

static const struct symsearch arr[] = {

{ __start___ksymtab, __stop___ksymtab, __start___kcrctab,

NOT_GPL_ONLY },

{ __start___ksymtab_gpl, __stop___ksymtab_gpl,

__start___kcrctab_gpl,

GPL_ONLY },

};

struct module *mod;

unsigned int i;

for (i = 0; i < ARRAY_SIZE(arr); i++)

if (find_exported_symbol_in_section(&arr[i], NULL, fsa))

return true;

list_for_each_entry_rcu(mod, &modules, list,

lockdep_is_held(&module_mutex)) {

struct symsearch arr[] = {

{ mod->syms, mod->syms + mod->num_syms, mod->crcs,

NOT_GPL_ONLY },

{ mod->gpl_syms, mod->gpl_syms + mod->num_gpl_syms,

mod->gpl_crcs,

GPL_ONLY },

};

if (mod->state == MODULE_STATE_UNFORMED)

continue;

for (i = 0; i < ARRAY_SIZE(arr); i++)

if (find_exported_symbol_in_section(&arr[i], mod, fsa))

return true;

}

pr_debug("Failed to find symbol %s\n", fsa->name);

return false;

}

struct module *find_module_all(const char *name, size_t len,

bool even_unformed)

{

struct module *mod;

list_for_each_entry_rcu(mod, &modules, list,

lockdep_is_held(&module_mutex)) {

if (!even_unformed && mod->state == MODULE_STATE_UNFORMED)

continue;

if (strlen(mod->name) == len && !memcmp(mod->name, name, len))

return mod;

}

return NULL;

}

struct module *find_module(const char *name)

{

return find_module_all(name, strlen(name), false);

}

#ifdef CONFIG_SMP

static inline void __percpu *mod_percpu(struct module *mod)

{

return mod->percpu;

}

static int percpu_modalloc(struct module *mod, struct load_info *info)

{

Elf_Shdr *pcpusec = &info->sechdrs[info->index.pcpu];

unsigned long align = pcpusec->sh_addralign;

if (!pcpusec->sh_size)

return 0;

if (align > PAGE_SIZE) {

pr_warn("%s: per-cpu alignment %li > %li\n",

mod->name, align, PAGE_SIZE);

align = PAGE_SIZE;

}

mod->percpu = __alloc_reserved_percpu(pcpusec->sh_size, align);

if (!mod->percpu) {

pr_warn("%s: Could not allocate %lu bytes percpu data\n",

mod->name, (unsigned long)pcpusec->sh_size);

return -ENOMEM;

}

mod->percpu_size = pcpusec->sh_size;

return 0;

}

static void percpu_modfree(struct module *mod)

{

free_percpu(mod->percpu);

}

static unsigned int find_pcpusec(struct load_info *info)

{

return find_sec(info, ".data..percpu");

}

static void percpu_modcopy(struct module *mod,

const void *from, unsigned long size)

{

int cpu;

for_each_possible_cpu(cpu)

memcpy(per_cpu_ptr(mod->percpu, cpu), from, size);

}

bool __is_module_percpu_address(unsigned long addr, unsigned long *can_addr)

{

struct module *mod;

unsigned int cpu;

guard(rcu)();

list_for_each_entry_rcu(mod, &modules, list) {

if (mod->state == MODULE_STATE_UNFORMED)

continue;

if (!mod->percpu_size)

continue;

for_each_possible_cpu(cpu) {

void *start = per_cpu_ptr(mod->percpu, cpu);

void *va = (void *)addr;

if (va >= start && va < start + mod->percpu_size) {

if (can_addr) {

*can_addr = (unsigned long) (va - start);

*can_addr += (unsigned long)

per_cpu_ptr(mod->percpu,

get_boot_cpu_id());

}

return true;

}

}

}

return false;

}

bool is_module_percpu_address(unsigned long addr)

{

return __is_module_percpu_address(addr, NULL);

}

#else /* ... !CONFIG_SMP */

static inline void __percpu *mod_percpu(struct module *mod)

{

return NULL;

}

static int percpu_modalloc(struct module *mod, struct load_info *info)

{

/* UP modules shouldn't have this section: ENOMEM isn't quite right */

if (info->sechdrs[info->index.pcpu].sh_size != 0)

return -ENOMEM;

return 0;

}

static inline void percpu_modfree(struct module *mod)

{

}

static unsigned int find_pcpusec(struct load_info *info)

{

return 0;

}

static inline void percpu_modcopy(struct module *mod,

const void *from, unsigned long size)

{

/* pcpusec should be 0, and size of that section should be 0. */

BUG_ON(size != 0);

}

bool is_module_percpu_address(unsigned long addr)

{

return false;

}

bool __is_module_percpu_address(unsigned long addr, unsigned long *can_addr)

{

return false;

}

#endif /* CONFIG_SMP */

#define MODINFO_ATTR(field) \

static void setup_modinfo_##field(struct module *mod, const char *s) \

{ \

mod->field = kstrdup(s, GFP_KERNEL); \

} \

static ssize_t show_modinfo_##field(const struct module_attribute *mattr, \

struct module_kobject *mk, char *buffer) \

{ \

return scnprintf(buffer, PAGE_SIZE, "%s\n", mk->mod->field); \

} \

static int modinfo_##field##_exists(struct module *mod) \

{ \

return mod->field != NULL; \

} \

static void free_modinfo_##field(struct module *mod) \

{ \

kfree(mod->field); \

mod->field = NULL; \

} \

static const struct module_attribute modinfo_##field = { \

.attr = { .name = __stringify(field), .mode = 0444 }, \

.show = show_modinfo_##field, \

.setup = setup_modinfo_##field, \

.test = modinfo_##field##_exists, \

.free = free_modinfo_##field, \

};

MODINFO_ATTR(version);

MODINFO_ATTR(srcversion);

static struct {

char name[MODULE_NAME_LEN + 1];

char taints[MODULE_FLAGS_BUF_SIZE];

} last_unloaded_module;

#ifdef CONFIG_MODULE_UNLOAD

EXPORT_TRACEPOINT_SYMBOL(module_get);

#define MODULE_REF_BASE 1

static int module_unload_init(struct module *mod)

{

/*

atomic_set(&mod->refcnt, MODULE_REF_BASE);

INIT_LIST_HEAD(&mod->source_list);

INIT_LIST_HEAD(&mod->target_list);

/* Hold reference count during initialization. */

atomic_inc(&mod->refcnt);

return 0;

}

static int already_uses(struct module *a, struct module *b)

{

struct module_use *use;

list_for_each_entry(use, &b->source_list, source_list) {

if (use->source == a)

return 1;

}

pr_debug("%s does not use %s!\n", a->name, b->name);

return 0;

}

static int add_module_usage(struct module *a, struct module *b)

{

struct module_use *use;

pr_debug("Allocating new usage for %s.\n", a->name);

use = kmalloc(sizeof(*use), GFP_ATOMIC);

if (!use)

return -ENOMEM;

use->source = a;

use->target = b;

list_add(&use->source_list, &b->source_list);

list_add(&use->target_list, &a->target_list);

return 0;

}

static int ref_module(struct module *a, struct module *b)

{

int err;

if (b == NULL || already_uses(a, b))

return 0;

/* If module isn't available, we fail. */

err = strong_try_module_get(b);

if (err)

return err;

err = add_module_usage(a, b);

if (err) {

module_put(b);

return err;

}

return 0;

}

static void module_unload_free(struct module *mod)

{

struct module_use *use, *tmp;

mutex_lock(&module_mutex);

list_for_each_entry_safe(use, tmp, &mod->target_list, target_list) {

struct module *i = use->target;

pr_debug("%s unusing %s\n", mod->name, i->name);

module_put(i);

list_del(&use->source_list);

list_del(&use->target_list);

kfree(use);

}

mutex_unlock(&module_mutex);

}

#ifdef CONFIG_MODULE_FORCE_UNLOAD

static inline int try_force_unload(unsigned int flags)

{

int ret = (flags & O_TRUNC);

if (ret)

add_taint(TAINT_FORCED_RMMOD, LOCKDEP_NOW_UNRELIABLE);

return ret;

}

static inline int try_force_unload(unsigned int flags)

{

return 0;

}

#endif /* CONFIG_MODULE_FORCE_UNLOAD */

static int try_release_module_ref(struct module *mod)

{

int ret;

/* Try to decrement refcnt which we set at loading */

ret = atomic_sub_return(MODULE_REF_BASE, &mod->refcnt);

BUG_ON(ret < 0);

if (ret)

/* Someone can put this right now, recover with checking */

ret = atomic_add_unless(&mod->refcnt, MODULE_REF_BASE, 0);

return ret;

}

static int try_stop_module(struct module *mod, int flags, int *forced)

{

/* If it's not unused, quit unless we're forcing. */

if (try_release_module_ref(mod) != 0) {

*forced = try_force_unload(flags);

if (!(*forced))

return -EWOULDBLOCK;

}

/* Mark it as dying. */

mod->state = MODULE_STATE_GOING;

return 0;

}

int module_refcount(struct module *mod)

{

return atomic_read(&mod->refcnt) - MODULE_REF_BASE;

}

EXPORT_SYMBOL(module_refcount);

static void free_module(struct module *mod);

SYSCALL_DEFINE2(delete_module, const char __user *, name_user,

unsigned int, flags)

{

struct module *mod;

char name[MODULE_NAME_LEN];

char buf[MODULE_FLAGS_BUF_SIZE];

int ret, forced = 0;

if (!capable(CAP_SYS_MODULE) || modules_disabled)

return -EPERM;

if (strncpy_from_user(name, name_user, MODULE_NAME_LEN-1) < 0)

return -EFAULT;

name[MODULE_NAME_LEN-1] = '\0';

audit_log_kern_module(name);

if (mutex_lock_interruptible(&module_mutex) != 0)

return -EINTR;

mod = find_module(name);

if (!mod) {

ret = -ENOENT;

goto out;

}

if (!list_empty(&mod->source_list)) {

/* Other modules depend on us: get rid of them first. */

ret = -EWOULDBLOCK;

goto out;

}

/* Doing init or already dying? */

if (mod->state != MODULE_STATE_LIVE) {

/* FIXME: if (force), slam module count damn the torpedoes */

pr_debug("%s already dying\n", mod->name);

ret = -EBUSY;

goto out;

}

/* If it has an init func, it must have an exit func to unload */

if (mod->init && !mod->exit) {

forced = try_force_unload(flags);

if (!forced) {

/* This module can't be removed */

ret = -EBUSY;

goto out;

}

}

ret = try_stop_module(mod, flags, &forced);

if (ret != 0)

goto out;

mutex_unlock(&module_mutex);

/* Final destruction now no one is using it. */

if (mod->exit != NULL)

mod->exit();

blocking_notifier_call_chain(&module_notify_list,

MODULE_STATE_GOING, mod);

klp_module_going(mod);

ftrace_release_mod(mod);

async_synchronize_full();

/* Store the name and taints of the last unloaded module for diagnostic purposes */

strscpy(last_unloaded_module.name, mod->name);

strscpy(last_unloaded_module.taints, module_flags(mod, buf, false));

free_module(mod);

/* someone could wait for the module in add_unformed_module() */

wake_up_all(&module_wq);

return 0;

out:

mutex_unlock(&module_mutex);

return ret;

}

void __symbol_put(const char *symbol)

{

struct find_symbol_arg fsa = {

.name = symbol,

.gplok = true,

};

guard(rcu)();

BUG_ON(!find_symbol(&fsa));

module_put(fsa.owner);

}

EXPORT_SYMBOL(__symbol_put);

void symbol_put_addr(void *addr)

{

struct module *modaddr;

unsigned long a = (unsigned long)dereference_function_descriptor(addr);

if (core_kernel_text(a))

return;

/*

guard(rcu)();

modaddr = __module_text_address(a);

BUG_ON(!modaddr);

module_put(modaddr);

}

EXPORT_SYMBOL_GPL(symbol_put_addr);

static ssize_t show_refcnt(const struct module_attribute *mattr,

struct module_kobject *mk, char *buffer)

{

return sprintf(buffer, "%i\n", module_refcount(mk->mod));

}

static const struct module_attribute modinfo_refcnt =

__ATTR(refcnt, 0444, show_refcnt, NULL);

void __module_get(struct module *module)

{

if (module) {

atomic_inc(&module->refcnt);

trace_module_get(module, _RET_IP_);

}

}

EXPORT_SYMBOL(__module_get);

bool try_module_get(struct module *module)

{

bool ret = true;

if (module) {

/* Note: here, we can fail to get a reference */

if (likely(module_is_live(module) &&

atomic_inc_not_zero(&module->refcnt) != 0))

trace_module_get(module, _RET_IP_);

else

ret = false;

}

return ret;

}

EXPORT_SYMBOL(try_module_get);

void module_put(struct module *module)

{

int ret;

if (module) {

ret = atomic_dec_if_positive(&module->refcnt);

WARN_ON(ret < 0); /* Failed to put refcount */

trace_module_put(module, _RET_IP_);

}

}

EXPORT_SYMBOL(module_put);

#else /* !CONFIG_MODULE_UNLOAD */

static inline void module_unload_free(struct module *mod)

{

}

static int ref_module(struct module *a, struct module *b)

{

return strong_try_module_get(b);

}

static inline int module_unload_init(struct module *mod)

{

return 0;

}

#endif /* CONFIG_MODULE_UNLOAD */

size_t module_flags_taint(unsigned long taints, char *buf)

{

size_t l = 0;

int i;

for (i = 0; i < TAINT_FLAGS_COUNT; i++) {

if (taint_flags[i].module && test_bit(i, &taints))

buf[l++] = taint_flags[i].c_true;

}

return l;

}

static ssize_t show_initstate(const struct module_attribute *mattr,

struct module_kobject *mk, char *buffer)

{

const char *state = "unknown";

switch (mk->mod->state) {

case MODULE_STATE_LIVE:

state = "live";

break;

case MODULE_STATE_COMING:

state = "coming";

break;

case MODULE_STATE_GOING:

state = "going";

break;

default:

BUG();

}

return sprintf(buffer, "%s\n", state);

}

static const struct module_attribute modinfo_initstate =

__ATTR(initstate, 0444, show_initstate, NULL);

static ssize_t store_uevent(const struct module_attribute *mattr,

struct module_kobject *mk,

const char *buffer, size_t count)

{

int rc;

rc = kobject_synth_uevent(&mk->kobj, buffer, count);

return rc ? rc : count;

}

const struct module_attribute module_uevent =

__ATTR(uevent, 0200, NULL, store_uevent);

static ssize_t show_coresize(const struct module_attribute *mattr,

struct module_kobject *mk, char *buffer)

{

unsigned int size = mk->mod->mem[MOD_TEXT].size;

if (!IS_ENABLED(CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC)) {

for_class_mod_mem_type(type, core_data)

size += mk->mod->mem[type].size;

}

return sprintf(buffer, "%u\n", size);

}

static const struct module_attribute modinfo_coresize =

__ATTR(coresize, 0444, show_coresize, NULL);

#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC

static ssize_t show_datasize(const struct module_attribute *mattr,

struct module_kobject *mk, char *buffer)

{

unsigned int size = 0;

for_class_mod_mem_type(type, core_data)

size += mk->mod->mem[type].size;

return sprintf(buffer, "%u\n", size);

}

static const struct module_attribute modinfo_datasize =

__ATTR(datasize, 0444, show_datasize, NULL);