笔记

openwrt procd启动流程分析

openwrt procd启动流程分析

kernel_init

Linux内核执行start_kernel函数时会调用kernel_init来启动init进程，流程如下图

start_kernel–>rest_init–>kernel_init–>try_to_run_init_process

kernel_init()(位于 linux-4.1.52/init/main.c)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

    /*
     * We try each of these until one succeeds.
     *
     * The Bourne shell can be used instead of init if we are
     * trying to recover a really broken machine.
     */
    if (execute_command) {
        ret = run_init_process(execute_command);
        if (!ret)
            return 0;
        panic("Requested init %s failed (error %d).",
              execute_command, ret);
    }
    if (!try_to_run_init_process("/sbin/init") ||
        !try_to_run_init_process("/etc/init") ||
        !try_to_run_init_process("/bin/init") ||
        !try_to_run_init_process("/bin/sh"))
        return 0;

    panic("No working init found.  Try passing init= option to kernel. "
          "See Linux Documentation/init.txt for guidance.");
}

/sbin/init

openwrt 源码 openwrt/package/system/procd/Makefile

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

define Package/procd/install
    $(INSTALL_DIR) $(1)/sbin $(1)/etc $(1)/lib/functions

    $(INSTALL_BIN) $(PKG_INSTALL_DIR)/usr/sbin/{init,procd,askfirst,udevtrigger,upgraded} $(1)/sbin/
    $(INSTALL_DATA) $(PKG_INSTALL_DIR)/usr/lib/libsetlbf.so $(1)/lib
    $(INSTALL_BIN) ./files/reload_config $(1)/sbin/
    $(INSTALL_CONF) ./files/hotplug*.json $(1)/etc/
    $(INSTALL_DATA) ./files/procd.sh $(1)/lib/functions/
    $(INSTALL_BIN) ./files/service $(1)/sbin/service
endef

procd源码procd/CMakeList.txt

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

IF(DISABLE_INIT)
ADD_DEFINITIONS(-DDISABLE_INIT)
ELSE()
ADD_EXECUTABLE(init initd/init.c initd/early.c initd/preinit.c initd/mkdev.c sysupgrade.c watchdog.c
    utils/utils.c)
TARGET_INCLUDE_DIRECTORIES(init PUBLIC ${SELINUX_INCLUDE_DIRS})
TARGET_LINK_LIBRARIES(init ${LIBS} ${SELINUX_LIBRARIES})
INSTALL(TARGETS init
    RUNTIME DESTINATION ${CMAKE_INSTALL_SBINDIR}
)

procd 启动流程

/sbin/init main 函数入口位于 procd/initd/init.c

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47

int main(int argc, char **argv)
{
    pid_t pid;

    ulog_open(ULOG_KMSG, LOG_DAEMON, "init");

    sigaction(SIGTERM, &sa_shutdown, NULL);
    sigaction(SIGUSR1, &sa_shutdown, NULL);
    sigaction(SIGUSR2, &sa_shutdown, NULL);
    sigaction(SIGPWR, &sa_shutdown, NULL);

    if (selinux(argv))
        exit(-1);
    early();
    cmdline();
    watchdog_init(1);

    pid = fork();
    if (!pid) {
        char *kmod[] = { "/sbin/kmodloader", "/etc/modules-boot.d/", NULL };

        if (debug < 3)
            patch_stdio("/dev/null");

        execvp(kmod[0], kmod);
        ERROR("Failed to start kmodloader: %m\n");
        exit(EXIT_FAILURE);
    }
    if (pid <= 0) {
        ERROR("Failed to start kmodloader instance: %m\n");
    } else {
        const struct timespec req = {0, 10 * 1000 * 1000};
        int i;

        for (i = 0; i < 1200; i++) {
            if (waitpid(pid, NULL, WNOHANG) > 0)
                break;
            nanosleep(&req, NULL);
            watchdog_ping();
        }
    }
    uloop_init();
    preinit();
    uloop_run();

    return 0;
}

kmodloader 先启动的是kmodloader(实现于openwrt/ubox/kmodloader.c),会insmod位于/etc/modules.d/下的kernel module list

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62


static int main_loader(int argc, char **argv)
{
	int gl_flags = GLOB_NOESCAPE | GLOB_MARK;
	char *dir = "/etc/modules.d/";
	struct module_node *mn;
	struct module *m;
	glob_t gl;
	char *path;
	int ret = 0, fail, j;

	if (argc > 1)
		dir = argv[1];

	path = malloc(strlen(dir) + 2);
	if (!path) {
		ULOG_ERR("out of memory\n");
		return -1;
	}

	strcpy(path, dir);
	strcat(path, "*");

	if (scan_module_folders()) {
		ret = -1;
		goto free_path;
	}

	if (scan_loaded_modules()) {
		ret = -1;
		goto free_path;
	}

	ULOG_INFO("loading kernel modules from %s\n", path);
	......
}

int main(int argc, char **argv)
{
	char *exec = basename(*argv);

	avl_init(&modules, avl_modcmp, true, NULL);
	if (!strcmp(exec, "insmod"))
		return main_insmod(argc, argv);

	if (!strcmp(exec, "rmmod"))
		return main_rmmod(argc, argv);

	if (!strcmp(exec, "lsmod"))
		return main_lsmod(argc, argv);

	if (!strcmp(exec, "modinfo"))
		return main_modinfo(argc, argv);

	load_options();

	if (!strcmp(exec, "modprobe"))
		return main_modprobe(argc, argv);

	ulog_open(ULOG_KMSG, LOG_USER, "kmodloader");
	return main_loader(argc, argv);
}

uloop_init实现位于libubox源码uloop.c

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

int uloop_init(void)
{
    if (uloop_init_pollfd() < 0)
        return -1;

    if (waker_init() < 0) {
        uloop_done();
        return -1;
    }

    return 0;
}

static int uloop_init_pollfd(void)
{
    if (poll_fd >= 0)
        return 0;

    poll_fd = epoll_create(32);    
    if (poll_fd < 0)
        return -1;

    fcntl(poll_fd, F_SETFD, fcntl(poll_fd, F_GETFD) | FD_CLOEXEC);
    return 0;
}

preinit实现位于procd源码文件initd/preinit.c

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49

static struct uloop_process preinit_proc;
static struct uloop_process plugd_proc;

void
preinit(void)
{
    char *init[] = { "/bin/sh", "/etc/preinit", NULL };
    char *plug[] = { "/sbin/procd", "-h", "/etc/hotplug-preinit.json", NULL };
    int fd;

    LOG("- preinit -\n");

    plugd_proc.cb = plugd_proc_cb;
    plugd_proc.pid = fork();
    if (!plugd_proc.pid) {
        execvp(plug[0], plug);
        ERROR("Failed to start plugd: %m\n");
        exit(EXIT_FAILURE);
    }
    if (plugd_proc.pid <= 0) {
        ERROR("Failed to start new plugd instance: %m\n");
        return;
    }
    uloop_process_add(&plugd_proc);

    setenv("PREINIT", "1", 1);

    fd = creat("/tmp/.preinit", 0600);

    if (fd < 0)
        ERROR("Failed to create sentinel file: %m\n");
    else
        close(fd);

    preinit_proc.cb = spawn_procd;
    preinit_proc.pid = fork();
    if (!preinit_proc.pid) {
        execvp(init[0], init);
        ERROR("Failed to start preinit: %m\n");
        exit(EXIT_FAILURE);
    }
    if (preinit_proc.pid <= 0) {
        ERROR("Failed to start new preinit instance: %m\n");
        return;
    }
    uloop_process_add(&preinit_proc);

    DEBUG(4, "Launched preinit instance, pid=%d\n", (int) preinit_proc.pid);
}
  • 创建子进程执行 /sbin/procd -h /etc/hotplug-preinit.json ，主进程同时使用 uloop_process_add()把 /sbin/procd 子进程加入 uloop 进行监控，当 /sbin/procd 进程结束时回调 plugd_proc_cb 函数。
  • 创建子进程执行 /etc/preinit 脚本，此时 PREINIT环境变量被设置为1，主进程同时使用 uloop_process_add() 把/etc/preinit 子进程加入 uloop 进行监控，当 /etc/preinit 执行结束时回调 spawn_procd函数
  • spawn_procd()函数繁行后继真正使用的 /sbin/procd 进程，从 /tmp/debuglevel 读出 debug 级别并设置到环境变量 DBGLVL 中，把 watchdog fd 设置到环境变量 WDTFD 中，最后调用 execvp()繁行 /sbin/procd 进程

首先看procd，因为带有参数“-h /etc/hotplug-preinit.json”，所以会执行hotplug_run函数。

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45

int main(int argc, char **argv)
{
    int ch;
    char *dbglvl = getenv("DBGLVL");
    int ulog_channels = ULOG_KMSG;

    if (dbglvl) {
        debug = atoi(dbglvl);
        unsetenv("DBGLVL");
    }

    while ((ch = getopt(argc, argv, "d:s:h:S")) != -1) {
        switch (ch) {
        case 'h':
            return hotplug_run(optarg); // hotplug
        case 's':
            ubus_socket = optarg;
            break;
        case 'd':
            debug = atoi(optarg);
            break;
        case 'S':
            ulog_channels = ULOG_STDIO;
            break;
        default:
            return usage(argv[0]);
        }
    }

    ulog_open(ulog_channels, LOG_DAEMON, "procd");
    ulog_threshold(LOG_DEBUG + 1);

    setsid();
    uloop_init();
    procd_signal();
    procd_udebug_set_enabled(true);
    if (getpid() != 1)
        procd_connect_ubus();
    else
        procd_state_next();
    uloop_run();
    uloop_done();

    return 0;
}

hotplug实现如下，这里是建立netlink通信机制，完成用户层和内核的交互，监听内核的uevent事件。

procd/plug/hotplug.c

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36

void hotplug(char *rules)
{
    struct sockaddr_nl nls;
    int nlbufsize = 512 * 1024;

    rule_file = strdup(rules);
    memset(&nls,0,sizeof(struct sockaddr_nl));
    nls.nl_family = AF_NETLINK;
    nls.nl_pid = getpid();
    nls.nl_groups = -1;

    if ((hotplug_fd.fd = socket(PF_NETLINK, SOCK_DGRAM | SOCK_CLOEXEC, NETLINK_KOBJECT_UEVENT)) == -1) {
        ERROR("Failed to open hotplug socket: %s\n", strerror(errno));
        exit(1);
    }
    if (bind(hotplug_fd.fd, (void *)&nls, sizeof(struct sockaddr_nl))) {
        ERROR("Failed to bind hotplug socket: %s\n", strerror(errno));
        exit(1);
    }

    if (setsockopt(hotplug_fd.fd, SOL_SOCKET, SO_RCVBUFFORCE, &nlbufsize, sizeof(nlbufsize)))
        ERROR("Failed to resize receive buffer: %s\n", strerror(errno));

    json_script_init(&jctx);
    queue_proc.cb = queue_proc_cb;
    uloop_fd_add(&hotplug_fd, ULOOP_READ);
}

int hotplug_run(char *rules)
{
    uloop_init();
    hotplug(rules);
    uloop_run();

    return 0;
}
  • 内核发出uevent事件 内核使用 uevent 事件通知用户空间， uevent 首先在内核中调用 netink_kemel_create() 函数创建一个 socket 套接字，该函数原型在 netink.h 中定义。这是一种特殊类型的 socket ，专门用于内核空间与用户空间的异步通信。kobject_uevent()产生uevent 事件 (/lib/kobject_uevent.c)，事件的部分信息通过环境变量传递，如$ACTION,$DEVPATH,$SUBSYSTEM 等，产生的 uevent 先由 netlink_broadcast_filtered()发出，最后调用uevent helper 所指定的程序来处理。在linux 中，uevent_helper 里默认指定"/sbin/hotplug”，但可以通过 /sys/kemel/uevent helper (kernel/ksysfs.c) /proc/kernel/uevent_elper(kernel/sysctl.c)来修改成指定的程序。在新 OpenWRT 中，并不使用 user helper 指定程序来处理 uevent(/sbin/hotplug 不存在，在以前版本中存在)，而是通过PF_NETLINK套接字来获取来自内核空间的 uevent 。
  • 用户空间监听uevent 在 procd/plug/hotplug.c 中，创建一个 PF_NETLINK 套接字来监听内核 netlink_broadcast_fitered() 发出的 uevent 。收到uevent 之后，在根据 /etc/hotplug.json 里的描述，定位到对应的执行函数来处理.通常情况下， /etc/hotplug.json 会调用 /sbin/hotplug-call 来处理 uevent ，它根据 uevent 的 $SUBSYSTEM 变量来分别调用 /etc/hotplug.d 下不同目录中的脚本。

/etc/preinit脚本大致内容如下，先调用另外的shell脚本，获取函数定义

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20

. /lib/functions.sh
. /lib/functions/preinit.sh
. /lib/functions/system.sh

# 初始化hook链
boot_hook_init preinit_essential
boot_hook_init preinit_main
boot_hook_init failsafe
boot_hook_init initramfs

# 依次执行/lib/preinit目录中的脚本，将函数调用添加到hook链中
for pi_source_file in /lib/preinit/*; do
    . $pi_source_file
done

# 执行preinit_essential注册的hook链的所有函数
boot_run_hook preinit_essential

# 执行preinit_main注册的hook链的所有函数
boot_run_hook preinit_main

/etc/preinit脚本执行完成后，调用spawn_procd,spawn_procd会调用 execvp()执行 /sbin/procd进程

procd/initd/preinit.c

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

spawn_procd(struct uloop_process *proc, int ret)
{
    char *wdt_fd = watchdog_fd();
    char *argv[] = { "/sbin/procd", NULL};
    char dbg[2];

    if (plugd_proc.pid > 0)
        kill(plugd_proc.pid, SIGKILL);

    unsetenv("PREINIT");
    unlink("/tmp/.preinit");

    check_sysupgrade();

    DEBUG(2, "Exec to real procd now\n");
    if (wdt_fd)
        setenv("WDTFD", wdt_fd, 1);
    check_dbglvl();
    if (debug > 0) {
        snprintf(dbg, 2, "%d", debug);
        setenv("DBGLVL", dbg, 1);
    }

    execvp(argv[0], argv);
}

procd/procd.c中

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

setsid();
uloop_init();
procd_signal();
procd_udebug_set_enabled(true);
if (getpid() != 1)
    procd_connect_ubus();
else
    procd_state_next();
uloop_run();
uloop_done();

此时getpid()等于1，所以调用procd_state_next，进入到状态机处理中。

procd_state不断迁移，包括STATE_EARLY，STATE_UBUS，STATE_INIT等。

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21

[    3.161338@3] init: Console is alive        
[    3.173921@3] init: Ping        
[    3.184207@3] init: Ping
[    3.192558@1] kmodloader: loading kernel modules from /etc/modules-boot.d/*    
[    3.194447@3] init: Ping
[    3.196209@1] kmodloader: done loading kernel modules from /etc/modules-boot.d/*
[    3.204716@3] init: Ping
[    3.206180@3] init: - preinit -
[    3.208671@3] init: Launched preinit instance, pid=1308
[    3.302967@3] init: Exec to real procd now
[    3.308865@3] procd: - early -
[    3.524654@2] procd: Finished udevtrigger
[    4.024929@2] procd: Coldplug complete
[    4.028061@2] procd: - ubus -
[    4.029198@2] procd: Create service ubus
[    4.030829@2] procd: Create instance ubus::instance1
[    4.032109@2] procd: Started instance ubus::instance1[1554]
[    4.098895@2] procd: Connected to ubus, id=459ede6c
[    4.099092@2] procd: - init -
[    4.102474@2] procd: Launched new askconsole action, pid=1555
[    4.104142@2] procd: Launched new askfirst action, pid=1556

以STATE_INIT为例，执行procd_inittab_run(“xxx”)会调用对应handlers的callback，对应所有的init_action是在procd_inittab()中添加的。

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

    case STATE_INIT:
        LOG("- init -\n");
        procd_inittab();
        procd_inittab_run("respawn");
        procd_inittab_run("askconsole");
        procd_inittab_run("askfirst");
        procd_inittab_run("sysinit");

static struct init_handler handlers[] = {
    {
        .name = "sysinit",
        .cb = runrc,
    }, {
        .name = "shutdown",
        .cb = runrc,
    }, {
        .name = "askfirst",
        .cb = askfirst,
        .multi = 1,
    }, {
        .name = "askconsole",
        .cb = askconsole,
        .multi = 1,
    }, {
        .name = "respawn",
        .cb = rcrespawn,
        .multi = 1,
    }
};

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79

static const char *tab = "/etc/inittab";
static char *ask = "/sbin/askfirst";

static int add_action(struct init_action *a, const char *name)
{
    int i;

    for (i = 0; i < ARRAY_SIZE(handlers); i++)
        if (!strcmp(handlers[i].name, name)) {
            a->handler = &handlers[i];
            list_add_tail(&a->list, &actions);
            return 0;
        }
    ERROR("Unknown init handler %s\n", name);
    return -1;
}


void procd_inittab(void)
{
#define LINE_LEN    128
    FILE *fp = fopen(tab, "r");
    struct init_action *a;
    regex_t pat_inittab;
    regmatch_t matches[5];
    char *line;

    if (!fp) {
        ERROR("Failed to open %s: %m\n", tab);
        return;
    }

    regcomp(&pat_inittab, "([a-zA-Z0-9]*):([a-zA-Z0-9]*):([a-zA-Z0-9]*):(.*)", REG_EXTENDED);
    line = malloc(LINE_LEN);
    a = calloc(1, sizeof(struct init_action));

    while (fgets(line, LINE_LEN, fp)) {
        char *tags[TAG_PROCESS + 1];
        char *tok;
        int i;
        int len = strlen(line);

        while (isspace(line[len - 1]))
            len--;
        line[len] = 0;

        if (*line == '#')
            continue;

        if (regexec(&pat_inittab, line, 5, matches, 0))
            continue;

        DEBUG(4, "Parsing inittab - %s\n", line);

        for (i = TAG_ID; i <= TAG_PROCESS; i++) {
            line[matches[i].rm_eo] = '\0';
            tags[i] = &line[matches[i + 1].rm_so];
        };

        tok = strtok(tags[TAG_PROCESS], " ");
        for (i = 0; i < (MAX_ARGS - 1) && tok; i++) {
            a->argv[i] = tok;
            tok = strtok(NULL, " ");
        }
        a->argv[i] = NULL;
        a->id = tags[TAG_ID];
        a->line = line;

        if (add_action(a, tags[TAG_ACTION]))
            continue;
        line = malloc(LINE_LEN);
        a = calloc(1, sizeof(struct init_action));
    }

    fclose(fp);
    free(line);
    free(a);
    regfree(&pat_inittab);
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

void procd_inittab_run(const char *handler)
{
    struct init_action *a;

    list_for_each_entry(a, &actions, list) {
        if (!strcmp(a->handler->name, handler)) {
            if (a->handler->multi) {
                a->handler->cb(a);
                continue;
            }
            a->handler->cb(a);
            break;
        }
    }

}

/etc/inittab

1
2
3

::sysinit:/etc/init.d/rcS S boot
::shutdown:/etc/init.d/rcS K shutdown
::askconsole:/usr/libexec/login.sh

这里来看runrc的实现，代码位于inittab.c

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

static void runrc(struct init_action *a)
{
	if (!a->argv[1] || !a->argv[2]) {
		ERROR("valid format is rcS <S|K> <param>\n");
		return;
	}

	/* proceed even if no init or shutdown scripts run */
	if (rcS(a->argv[1], a->argv[2], rcdone))
		rcdone(NULL);
}

rcS.c

1
2
3
4
5
6
7
8

int rcS(char *pattern, char *param, void (*q_empty)(struct runqueue *))
{
    runqueue_init(&q);
    q.empty_cb = q_empty;
    q.max_running_tasks = 1;

    return _rc(&q, "/etc/rc.d", pattern, "*", param);
}

执行/etc/rc.d目录下S开头的脚本

Openwrt
Licensed under CC BY-NC-SA 4.0
© 2025 - 2026 Charles Lv
使用 Hugo 构建
主题 Stack 由 Jimmy 设计