#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/smp.h>
#include <linux/sched.h>
#include <linux/kthread.h>
#include <linux/delay.h>

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Hu Yu <[email protected]>");
MODULE_DESCRIPTION("kthread test");
MODULE_VERSION("0.1");

static struct task_struct *my_task = NULL;

static int my_kthread(void *data) {
    char *str = (char *)data;

    pr_info("my kthread data: %s\n", str);
    pr_info("my kthread smp_processor_id %d\n", smp_processor_id());
    while(!kthread_should_stop()) {
        msleep(5000);
        pr_info("my kthread: living. smp_processor_id %d\n", smp_processor_id());
        pr_info("=========================================\n");
    }
    pr_info("my kthread: stop\n");
    return 0;
}

static int __init my_init(void)
{
    pr_info("my init.\n");
    pr_info("smp_processor_id %d\n", smp_processor_id());

    my_task = kthread_run(my_kthread, "hello my kthread", "mykthread-%s", "test");

    pr_info("my init finish.\n");
    pr_info("=========================================\n");
    return 0;
}

static void __exit my_exit(void)
{
    pr_info("my exit.\n");
    pr_info("smp_processor_id %d\n", smp_processor_id());

    if (my_task) {
        pr_info("stop kthread\n");
        kthread_stop(my_task);
    }

    pr_info("my exit finish.\n");
    pr_info("=========================================\n");
}

module_init(my_init);
module_exit(my_exit);

/**
 * kthread_create_on_node - create a kthread.
 * @threadfn: the function to run until signal_pending(current).
 * @data: data ptr for @threadfn.
 * @node: memory node number.
 * @namefmt: printf-style name for the thread.
 *
 * Description: This helper function creates and names a kernel
 * thread.  The thread will be stopped: use wake_up_process() to start
 * it.  See also kthread_run().
 *
 * If thread is going to be bound on a particular cpu, give its node
 * in @node, to get NUMA affinity for kthread stack, or else give -1.
 * When woken, the thread will run @threadfn() with @data as its
 * argument. @threadfn() can either call do_exit() directly if it is a
 * standalone thread for which no one will call kthread_stop(), or
 * return when 'kthread_should_stop()' is true (which means
 * kthread_stop() has been called).  The return value should be zero
 * or a negative error number; it will be passed to kthread_stop().
 *
 * Returns a task_struct or ERR_PTR(-ENOMEM).
 */
struct task_struct *kthread_create_on_node(int (*threadfn)(void *data),
        void *data, int node,
        const char namefmt[],
        ...)
{
    struct kthread_create_info create;

    create.threadfn = threadfn;
    create.data = data;
    create.node = node;
    init_completion(&create.done);

    spin_lock(&kthread_create_lock);
    list_add_tail(&create.list, &kthread_create_list);
    spin_unlock(&kthread_create_lock);

    wake_up_process(kthreadd_task);
    wait_for_completion(&create.done);

    if (!IS_ERR(create.result)) {
        static const struct sched_param param = { .sched_priority = 0 };
        va_list args;

        va_start(args, namefmt);
        vsnprintf(create.result->comm, sizeof(create.result->comm),
                namefmt, args);
        va_end(args);
        /*
         * root may have changed our (kthreadd's) priority or CPU mask.
         * The kernel thread should not inherit these properties.
         */
        sched_setscheduler_nocheck(create.result, SCHED_NORMAL, &param);
        set_cpus_allowed_ptr(create.result, cpu_all_mask);
    }
    return create.result;
}

struct kthread_create_info
{
    /* Information passed to kthread() from kthreadd. */
    /* 内核线程需要执行的函数 */
    int (*threadfn)(void *data);
    /* 执行函数的唯一一个参数 */
    void *data;
    /* numa node */
    int node;

    /* Result passed back to kthread_create() from kthreadd. */
    /* kthread_create_on_node函数的返回值 */
    struct task_struct *result;
    /* kthreadd创建内核线程完成的标识 */
    struct completion done;

    /* 用于链接到kthread_create_list队列 */
    struct list_head list;
};

static noinline void __init_refok rest_init(void)
{
    int pid;

    rcu_scheduler_starting();
    /*
     * We need to spawn init first so that it obtains pid 1, however
     * the init task will end up wanting to create kthreads, which, if
     * we schedule it before we create kthreadd, will OOPS.
     */
    // 这里创建出了1号线程，优先于kthreadd创建，而且其父线程为0号线程
    kernel_thread(kernel_init, NULL, CLONE_FS | CLONE_SIGHAND);
    numa_default_policy();
    // 这里创建除了2号线程，kthreadd
    pid = kernel_thread(kthreadd, NULL, CLONE_FS | CLONE_FILES);
    rcu_read_lock();
    kthreadd_task = find_task_by_pid_ns(pid, &init_pid_ns);
    rcu_read_unlock();
    // 全局标记kthreadd是否启动完成，1号线程需要等待kthreadd启动完成才能进行后续工作。
    complete(&kthreadd_done);

    /*
     * The boot idle thread must execute schedule()
     * at least once to get things moving:
     */
    init_idle_bootup_task(current);
    schedule_preempt_disabled();
    /* Call into cpu_idle with preempt disabled */
    cpu_startup_entry(CPUHP_ONLINE);
}

int kthreadd(void *unused)
{
    struct task_struct *tsk = current;

    /* Setup a clean context for our children to inherit. */
    /* 设置进程名 */
    set_task_comm(tsk, "kthreadd");
    /* 忽略所有信号 */
    ignore_signals(tsk);
    /* 设置该进程cpu亲和度 */
    set_cpus_allowed_ptr(tsk, cpu_all_mask);
    /* 设置该进程内存分配 */
    set_mems_allowed(node_states[N_MEMORY]);

    /* 设置该进程不允许冻结 */
    current->flags |= PF_NOFREEZE;

    for (;;) {
        set_current_state(TASK_INTERRUPTIBLE);
        /* 如果链表kthread_create_list为空则主动让出cpu */
        if (list_empty(&kthread_create_list))
            schedule();
        /* 重新唤醒运行了 */
        __set_current_state(TASK_RUNNING);

        /* 为了防止冲突，链表操作需要上锁 */
        spin_lock(&kthread_create_lock);
        while (!list_empty(&kthread_create_list)) {
            struct kthread_create_info *create;

            /* 如果链表不空，每次取出一个kthread_create_info结构的实例 */
            create = list_entry(kthread_create_list.next,
                    struct kthread_create_info, list);
            /* 从链表摘除 */
            list_del_init(&create->list);
            spin_unlock(&kthread_create_lock);

            /* 创建内核线程 */
            create_kthread(create);

            spin_lock(&kthread_create_lock);
        }
        spin_unlock(&kthread_create_lock);
    }

    return 0;
}

static void create_kthread(struct kthread_create_info *create)
{
    int pid;

#ifdef CONFIG_NUMA
    current->pref_node_fork = create->node;
#endif
    /* We want our own signal handler (we take no signals by default). */
    pid = kernel_thread(kthread, create, CLONE_FS | CLONE_FILES | SIGCHLD);
    if (pid < 0) {
        create->result = ERR_PTR(pid);
        complete(&create->done);
    }
}

static int kthread(void *_create)
{
    /* Copy data: it's on kthread's stack */
    /*
     * 将参数kthread_create_info中的指针保存到当前函数栈上，
     * 因为参数内容所在的内存位于kthread_create_on_node函数的栈上，
     * 因为kthread_create_on_node还在阻塞等待内核线程创建完成，因此函数栈内存还是有效的，
     * 一旦complete(&create->done);这句运行完成。
     * kthread_create_on_node随时可能运行完成返回，这里的参数内容所在的栈内存将无效，
     * 将无法找到其中的指针数据。
     * result指针不需要保存是因为其作为返回值由kthread_create_on_node返回了。
     */
    struct kthread_create_info *create = _create;
    int (*threadfn)(void *data) = create->threadfn;
    void *data = create->data;
    /* 这里kthread类型的结构体，对内核线程的很多操作会用到 */
    struct kthread self;
    int ret;

    self.flags = 0;
    self.data = data;
    init_completion(&self.exited);
    init_completion(&self.parked);
    /* 这里的赋值使通过task_struct定位上面的kthread结构体地址成为可能 */
    current->vfork_done = &self.exited;

    /* OK, tell user we're spawned, wait for stop or wakeup */
    /* 设置当前线程状态 */
    __set_current_state(TASK_UNINTERRUPTIBLE);
    /* 用于提供给kthread_create_on_node的返回值 */
    create->result = current;
    /* 设置标识，通知kthread_create_on_node内核创建完成。个人理解这个操作应该包含了内存屏障，没有进一步验证*/
    complete(&create->done);
    /* 让出cpu，等待wake_up_process唤醒当前内核线程 */
    schedule();

    ret = -EINTR;

    /* 唤醒后首先检查是否应该停止 */
    if (!test_bit(KTHREAD_SHOULD_STOP, &self.flags)) {
        /* 其次检查是否应该进入park状态 */
        __kthread_parkme(&self);
        /* 运行创建内核线程时传入的指定函数，并传入指定的参数 */
        ret = threadfn(data);
    }
    /* we can't just return, we must preserve "self" on stack */
    /* 
     * 创建内核线程时传入的指定函数可以直接return，
     * 但kthread函数不能直接return，还需要负责清理线程相关的所有资源
     * 最后通过调度器让出cpu。
     */
    do_exit(ret);
}

struct kthread {
    unsigned long flags;
    unsigned int cpu;
    void *data;
    struct completion parked;
    struct completion exited;
};