From eb3e8dd9e87e926a121715d103cc6c3cb20388a7 Mon Sep 17 00:00:00 2001 From: zhongtao Date: Fri, 17 Feb 2023 14:33:15 +0800 Subject: [PATCH 49/53] add runc doc Signed-off-by: zhongtao --- docs/design/README_zh.md | 1 + .../design/detailed/Runtime/runc_design_zh.md | 264 ++++++++++++++++++ docs/images/isulad_shim_flow_chart.svg | 16 ++ docs/images/k8s_isulad_flow_chart.svg | 16 ++ docs/manual/README_zh.md | 6 +- docs/manual/runc_config_zh.md | 134 +++++++++ 6 files changed, 436 insertions(+), 1 deletion(-) create mode 100644 docs/design/detailed/Runtime/runc_design_zh.md create mode 100755 docs/images/isulad_shim_flow_chart.svg create mode 100644 docs/images/k8s_isulad_flow_chart.svg create mode 100644 docs/manual/runc_config_zh.md diff --git a/docs/design/README_zh.md b/docs/design/README_zh.md index 1f1c94b5..e7763a42 100644 --- a/docs/design/README_zh.md +++ b/docs/design/README_zh.md @@ -49,6 +49,7 @@ ## Runtime - 查看 runtime 模块的设计文档： [runtime_design](./detailed/Runtime/runtime_design_zh.md) 。 +- 查看 isulad 支持runc 的设计文档：[runc_design](./detailed/Runtime/runc_design_zh.md)。 ## Security diff --git a/docs/design/detailed/Runtime/runc_design_zh.md b/docs/design/detailed/Runtime/runc_design_zh.md new file mode 100644 index 00000000..3af3ea02 --- /dev/null +++ b/docs/design/detailed/Runtime/runc_design_zh.md @@ -0,0 +1,264 @@ +| Author | zhongtao | +| ------ | --------------------- | +| Date | 2023-03-21 | +| Email | zhongtao17@huawei.com | + +# 方案目标 + +isulad 支持使用runc作为容器runtime。用户可以在启动容器时配置`--runtime=runc`创建runtime为runc的容器： +```bash +$ isula run -tid -n test --runtime=runc busybox sh +eb77c672a148cc4cefa0d6e7c5847f5a87d0c5353e245461b68820bd9af90c67 +$ isula inspect eb77 | grep -i runc + "ResolvConfPath": "/var/lib/isulad/engines/runc/eb77c672a148cc4cefa0d6e7c5847f5a87d0c5353e245461b68820bd9af90c67/resolv.conf", + "HostsPath": "/var/lib/isulad/engines/runc/eb77c672a148cc4cefa0d6e7c5847f5a87d0c5353e245461b68820bd9af90c67/hosts", + "LogPath": "/var/lib/isulad/engines/runc/eb77c672a148cc4cefa0d6e7c5847f5a87d0c5353e245461b68820bd9af90c67/console.log", + "Runtime": "runc", + "log.console.file": "/var/lib/isulad/engines/runc/eb77c672a148cc4cefa0d6e7c5847f5a87d0c5353e245461b68820bd9af90c67/console.log", +``` + +同时，也可以在`/etc/isulad/daemon.json`中配置`default-runtime`为runc并重启isulad，修改isulad创建容器时默认使用的runtime。 +```sh +$ vim /etc/isulad/daemon.json + ... + "default-runtime": "runc" + ... +$ sudo isulad +``` + +# 总体设计 + +由于isulad与runc之间的交互存在gap，且将容器创建成功之后，容器进程的生命周期与isulad进程的生命周期没有必然联系，因此我们设计了一个isulad-shim进程，用于isulad与runc的交互并将isulad与容器实例解耦。同时，由于只有create以及exec涉及到在容器中新建进程，因此只有这两个子命令需要创建isulad-shim。其他的子命令直接通过调用runc二进制实现。 + +## 时序图 + +```mermaid +sequenceDiagram + participant isula + participant kubelet + participant isulad + participant supervisor + participant shim + participant runc + participant container + + isula->>isulad: request + kubelet->>isulad:request + alt create + isulad->>shim:shim_creat() + shim->>runc: execvp(runc, params) + runc ->> container:create request + container ->> runc:success + runc ->> shim:get process pid + isulad ->> isulad:get process pid + isulad ->> supervisor:add monitor + loop epoll exit_fd + supervisor ->> shim:if exit? + end + else exec + isulad->>shim:shim_creat() + par + shim->>runc: execvp(runc, params) + runc ->> container:exec request + container ->> runc:success + runc ->> shim:get process pid + shim ->> container:wait process pid + shim ->> shim:exit + and + isulad ->> isulad: wait isulad-shim pid + end + else others container cmd + isulad->>runc: runtime_call_simple() + runc ->> container:cmd + container ->>runc:success + runc ->>isulad:success + end + isulad ->> isula:response + isulad ->> kubelet:response +``` + +# 接口描述 + +## cri接口 + +### PodSandboxManagerService + +```h +auto RunPodSandbox(const runtime::v1alpha2::PodSandboxConfig &config, const std::string &runtimeHandler,Errors &error) -> std::string; + +void StopPodSandbox(const std::string &podSandboxID, Errors &error); + +void RemovePodSandbox(const std::string &podSandboxID, Errors &error); + +auto PodSandboxStatus(const std::string &podSandboxID, Errors &error) + -> std::unique_ptr; + +void ListPodSandbox(const runtime::v1alpha2::PodSandboxFilter *filter,std::vector> *pods, Errors &error); + +// This feature is temporarily not supported +void PortForward(const runtime::v1alpha2::PortForwardRequest &req,runtime::v1alpha2::PortForwardResponse *resp,Errors &error); + ... ... +}; +} // namespace CRI +``` + +### ContainerManagerService + +```c +auto CreateContainer(const std::string &podSandboxID, const runtime::v1alpha2::ContainerConfig &containerConfig,const runtime::v1alpha2::PodSandboxConfig &podSandboxConfig, Errors &error)-> std::string override; + +void StartContainer(const std::string &containerID, Errors &error) override; + +void StopContainer(const std::string &containerID, int64_t timeout, Errors &error) override; + +void RemoveContainer(const std::string &containerID, Errors &error) override; + +void ListContainers(const runtime::v1alpha2::ContainerFilter *filter, + std::vector> *containers, Errors &error) override; + +void ListContainerStats(const runtime::v1alpha2::ContainerStatsFilter *filter,std::vector> *containerstats,Errors &error) override; + +auto ContainerStats(const std::string &containerID, Errors &error) + -> std::unique_ptr override; + +auto ContainerStatus(const std::string &containerID, Errors &error) -> std::unique_ptr override; + +void ExecSync(const std::string &containerID, const google::protobuf::RepeatedPtrField &cmd, int64_t timeout, runtime::v1alpha2::ExecSyncResponse *reply, Errors &error) override; + +void Exec(const runtime::v1alpha2::ExecRequest &req, runtime::v1alpha2::ExecResponse *resp, Errors &error) override; + +void Attach(const runtime::v1alpha2::AttachRequest &req, runtime::v1alpha2::AttachResponse *resp, Errors &error) override; + +void UpdateContainerResources(const std::string &containerID, + const runtime::v1alpha2::LinuxContainerResources &resources, Errors &error) override; +``` + +### RuntimeManagerService + +```c +void UpdateRuntimeConfig(const runtime::v1alpha2::RuntimeConfig &config, Errors &error) override; + +// 对应crictl info +auto Status(Errors &error) -> std::unique_ptr override; +``` + +### ImageManagerService + +```c +void ListImages(const runtime::v1alpha2::ImageFilter &filter, std::vector> *images, Errors &error) override; std::unique_ptr ImageStatus(const runtime::v1alpha2::ImageSpec &image, Errors &error) override; + +std::string PullImage(const runtime::v1alpha2::ImageSpec &image, const runtime::v1alpha2::AuthConfig &auth, Errors &error) override; + +void RemoveImage(const runtime::v1alpha2::ImageSpec &image, Errors &error) override; + +void ImageFsInfo(std::vector> *usages, Errors &error) override +``` + + + +## isula_rt_ops模块 + +```c +// 检测runtime是否为isula_rt_ops模块处理的目标runtime +bool rt_isula_detect(const char *runtime); + +int rt_isula_create(const char *name, const char *runtime, const rt_create_params_t *params); + +int rt_isula_start(const char *name, const char *runtime, const rt_start_params_t *params, pid_ppid_info_t *pid_info); + +// restart not implemented +int rt_isula_restart(const char *name, const char *runtime, const rt_restart_params_t *params); + +int rt_isula_clean_resource(const char *name, const char *runtime, const rt_clean_params_t *params); + +int rt_isula_rm(const char *name, const char *runtime, const rt_rm_params_t *params); + +int rt_isula_exec(const char *id, const char *runtime, const rt_exec_params_t *params, int *exit_code); + +int rt_isula_status(const char *name, const char *runtime, const rt_status_params_t *params, + struct runtime_container_status_info *status); + +// isula attach not support on isulad-shim +int rt_isula_attach(const char *id, const char *runtime, const rt_attach_params_t *params); + +int rt_isula_update(const char *id, const char *runtime, const rt_update_params_t *params); + +int rt_isula_pause(const char *id, const char *runtime, const rt_pause_params_t *params); + +int rt_isula_resume(const char *id, const char *runtime, const rt_resume_params_t *params); + +// isula top/listpids not support on isulad-shim +int rt_isula_listpids(const char *name, const char *runtime, const rt_listpids_params_t *params, + rt_listpids_out_t *out); + +int rt_isula_resources_stats(const char *name, const char *runtime, const rt_stats_params_t *params, + struct runtime_container_resources_stats_info *rs_stats); +// rt_isula_resize not impl +int rt_isula_resize(const char *id, const char *runtime, const rt_resize_params_t *params); + +int rt_isula_exec_resize(const char *id, const char *runtime, const rt_exec_resize_params_t *params); + +int rt_isula_kill(const char *id, const char *runtime, const rt_kill_params_t *params); + +``` + + + +# 详细设计 + +## create 实现流程 + +isulad端： + +1. 创建process文件：create_process_json_file(); +2. 获得runtime二进制：get_runtime_cmd(runtime, &cmd); +3. 利用两次fork()创建isulad-shim进程， 此时的isulad-shim进程是1号进程的子进程，与isulad无父子关系:shim_create(); +4. 若shim创建成功则直接返回成功，若失败则调用runtime_call_delete_force()将容器进程都force delete。 + +isulad-shim端： + +1. 根据解析process文件新建一个process，new_process()； +2. 开启exit_fifo：open_no_inherit("exit_fifo", O_WRONLY, -1); +3. 为io_copy创建io_epoll_loop线程：process_io_init(); +4. 创建进行io copy的线程，并将isulad的console与runtime的console连接起来:open_io(); +5. 创建运行runtime的子进程，获得容器中进程的pid，create_process(); +6. 循环wait子进程，直到wait到的子进程为容器进程pid则退出shim进程，process_signal_handle_routine(); + +## exec 实现流程 + +isulad端： + +1. 创建process文件：create_process_json_file(); +2. 获得runtime二进制：get_runtime_cmd(runtime, &cmd); +3. 若`--detach=false`，直接fork()创建isulad-shim进程，此时的isulad-shim进程是isulad进程的子进程，isulad进程wait等待isulad-shim进程退出；若-`-detach=true`，则与create一样，创建独立的isulad-shim进程:shim_create(); +4. 循环读取pid文件获得容器进程id说明exec命令成功，get_container_process_pid。 + +isulad-shim端： + +1. 根据解析process文件新建一个process，new_process()； +2. 为io_copy创建io_epoll_loop线程：process_io_init(); +3. 创建进行io copy的线程，并将isulad的console与runtime的console连接起来:open_io(); +4. 创建运行runtime的子进程，获得容器中进程的pid，create_process(); +5. 循环wait子进程，直到wait到的子进程为容器进程pid则退出shim进程，process_signal_handle_routine(); + + + +## start 实现流程 + +isulad端： + +1. 分别读取pid以及shim-pid文件获得容器进程pid以及isulad-shim pid； +2. 依次获得容器进程以及isulad-shim进程的proc信息：util_get_process_proc_info(); +3. 根据proc信息为pid_ppid_info_t结构体赋值； +4. 直接调用runc二进制start容器：runtime_call_simple(); +5. 之后isulad根据监听exit_fifo_fd，感知容器是否退出，从而更新容器状态。 + +其他子命令与satrt类似，均是调用runtime_call_simple()函数直接调用runc二进制，此处不再赘述。 + + + +## 流程图 + +### isulad与isulad-shim交互流程图 + +![isulad_shim_flow_chart](../../../images/isulad_shim_flow_chart.svg) diff --git a/docs/images/isulad_shim_flow_chart.svg b/docs/images/isulad_shim_flow_chart.svg new file mode 100755 index 00000000..9a509b80 --- /dev/null +++ b/docs/images/isulad_shim_flow_chart.svg @@ -0,0 +1,16 @@ + + + + + + + isulad-shimmain()isulad-shim id bundle runtime info 2m0s从process.json中加载process：new_processif p->state->exec?将create的process存储在process.jsonwe文件中：create_process_json_fileisuladget_runtime_cmdret = shim_createret = -1runtime_call_delete_forcert_isula_creatert_isula_execget_runtime_cmdret = shim_createget_container_process_pid将exec的process存储在process.jsonwe文件中create_process_json_fileget_container_process_pidfile_read_int(shim_pid_file_name, &shim_pid);util_get_process_proc_inforuntime_call_simple(workdir,runtime, "start", NULL, 0, id, NULL)rt_isula_startfg=false;exit_code=NULLshim_createfork()若设置了--detach，则fg=true;否则为false；exit_code!=NULLparent(isulad)child(shim)从读fifo中读取isulad-shimxier写入的内容num = util_read_nointrif num > 0nowait isulad-shim或者shim parent 进程util_waitpid_with_timeoutif ret != 0show_shim_runtime_errlogif timeout <= 0kill(pid, SIGKILL)yeschdir(workdir)if fgnounsetenv("NOTIFY_SOCKET")fork()parent(shim parent)child(isulad-shim)将shim的pid写入文件中：file_write_int(fpid, pid)_exit(EXIT_SUCCESS);setsid()将从父进程继承的fd关闭util_check_inheritedexecvp(SHIM_BINARY, (char * const *)params);设置启动isulad-shim超时的定时器set_timeout_exit(DEFAULT_TIMEOUT);set_subreaper()解析参数：parse_argsyes打开exit_fd用于感知退出：open_no_inherit("exit_fifo", O_WRONLY, -1)为isulad与runtime之间的io copychuan构建main loop和epoll：：process_io_init(p);open_io(p, &tid_accept);no创建io copy线程：start_io_copy_threads(p);if (p->state->terminal) true开勇socket实现isulad与runtime的通信：open_terminal_io(p, tid_accept);利用pipe实现通信：open_generic_io(p);falsecreate_process(p);fork()child:runtime processparent:isulad-shim process拼接params，并调用runc二进制执行命令：exec_runtime_process(p, exec_fd[1]);读取runtime写入的信息： read_nointr(exec_fd[0],exec_buff, sizeof(exec_buff) - 1);等待runtime子进程退出：waitpid(pid, NULL, 0);获得runtime写入的容器中进程的pid：read_text_file("pid");将其赋值给process：p->ctr_pid = ctr_pid;清除超时计时器：released_timeout_exit();process_signal_handle_routine(p, tid_accept);wait任意一个子进程退出，当pid为ctr_pid时ret为0且exit_shim = true：reap_container(p->ctr_pid, &status);if ret == 0? noyesif exit_shim? yes调用runc kill命令杀死容器进程：process_kill_all(p);等待所有子进程被杀死：DO_RETRY_CALL(120, 1000000, nret, try_wait_all_child);调用runc delete命令删除容器进程：process_delete(p)if p->exit_fd > 0yeswrite_nointr(p->exit_fd, &status, sizeof(int));pthread_timedjoin_np(tid_accept, NULL, &ts);nodestroy_io_thread(p, i); \ No newline at end of file diff --git a/docs/images/k8s_isulad_flow_chart.svg b/docs/images/k8s_isulad_flow_chart.svg new file mode 100644 index 00000000..700b5b6d --- /dev/null +++ b/docs/images/k8s_isulad_flow_chart.svg @@ -0,0 +1,16 @@ + + + + + + + kubeletclientisuladservergrpcCRIPodSandboxManagerServiceruntime service implImageManagerServiceservice_executor_t：callback函数Network::PluginManagerRunPodSandbox1. Pull the image for the sandboxEnsureSandboxImageExists2. Create the sandbox containerCreateSandboxContainerRuntimeVersionerServiceContainerManagerServicePodSandboxManagerServiceRuntimeManagerServiceRuntimeManagerServiceContainerManagerServiceRuntimeVersionerServiceservice_executor_t：callback函数Network::PluginManagerservice_executor_tservice_executor_tNetwork::PluginManager1. 创建create容器的requestGenerateSandboxCreateContainerRequest2. 发送请求m_cb->container.create(create_request, &create_response)3: Enable network SetNetworkReady4: Inspect container：直接从container_t中获得容器信息：inspect_data =CRIHelpers::InspectContainer5: Get networking info GetSandboxNetworkInfo 6: Mount network namespace when network mode is cninamespace_is_cn；prepare_network_namespace7: Setup networking for the sandbox. SetupSandboxNetwork8: Start the sandbox container. StartSandboxContainer 9: Save network settings json to disk ips_request = GenerateUpdateNetworkSettingsReqestinput：run pod的config文件解析出来的结构体output：pause容器的idCreateContainerinput：podSandboxID、containerConfig、podSandboxConfigoutput：pod中容器的id1. 根据podsandbox的id或者名字获得容器idGetRealContainerOrSandboxID2. 获得pod的runtimeGetContainerOrSandboxRuntime3. 创建pod中的容器的create请求GenerateCreateContainerRequest4. 调用创建cbm_cb->container.create \ No newline at end of file diff --git a/docs/manual/README_zh.md b/docs/manual/README_zh.md index 8de5604b..1f3f37f8 100644 --- a/docs/manual/README_zh.md +++ b/docs/manual/README_zh.md @@ -22,4 +22,8 @@ Device Mapper 是一个基于内核的框架，它支持 Linux 上的许多高 ## isula search使用指南 -关于如何使用isula search请参考[isula_search](isula_search_zh.md) \ No newline at end of file +关于如何使用isula search请参考[isula_search](isula_search_zh.md) + +## runc使用指南 + +关于如何在isulad中使用runc请参考[runc_config](runc_config_zh.md) \ No newline at end of file diff --git a/docs/manual/runc_config_zh.md b/docs/manual/runc_config_zh.md new file mode 100644 index 00000000..003cd2c5 --- /dev/null +++ b/docs/manual/runc_config_zh.md @@ -0,0 +1,134 @@ +# runc使用指南 +本文主要是指导iSulad社区开发者和使用者，如何配置isulad使用runc作为runtime创建容器。 + +## 一、runc的安装 + +`tips`: 在安装runc之前需要安装好go环境。 + +isulad当前推荐的runc验证版本为v1.0.0-rc5。 + +runc可以使用以下两种安装方式： + +1. 直接使用包管理器安装runc: + +```sh +# centOS +sudo yum install runc +# Ubuntu +sudo apt-get install runc +``` + +2. 源码编译安装runc（注意建议切换成isulad推荐的runc版本：`git checkout v1.0.0-rc5`） + +```sh +# 在GOPATH/src下创建 'github.com/opencontainers' 文件夹 +cd github.com/opencontainers +git clone https://github.com/opencontainers/runc +cd runc + +make +sudo make install +``` + +还可以使用go get安装到`GOPATH`路径下（需要在GOPATH/src下创建github.com父文件夹）： + +```sh +go get github.com/opencontainers/runc +cd $GOPATH/src/github.com/opencontainers/runc +make +sudo make install +``` + +最终安装好的runc会在`/usr/local/sbin/runc`目录下。 + +## 二、配置iSulad使用runc + +### 配置文件配置 + +1. 修改isulad的daemon.json，配置isulad默认使用的runtime。 + +```sh +$ vim /etc/isulad/daemon.json + ... + "default-runtime": "runc" + ... +``` + +2. 也可以在配置文件中配置runtimes，在其中指定使用的`path`(用于修改isulad使用的runc路径)以及`runtime-args`(对runtime所有命令配置的参数)。 + +```sh +"runtimes": { + "runc": { + "path": "/usr/local/sbin/runc", + "runtime-args": [ + ] + } + }, +``` + +之后使用root权限启动isulad服务，使修改后的配置生效即可： + +```sh +$ sudo isulad +``` + +### 单个容器配置 + +使用`--runtime=runc`启动一个runtime为runc的容器。 + +```sh +isula run -tid -n test --runtime=runc busybox sh +``` + +## 三、K8s中配置pod的runtime为runc + +如何与kubernetes集成请参考[k8s_integration](https://gitee.com/openeuler/iSulad/blob/master/docs/manual/k8s_integration_zh.md)。 + +### 全局配置 + +直接参照第二节中配置文件配置的方式修改isulad默认使用的runtime为runc，则后续使用k8s启动容器时会默认使用的runtime即为runc。 + +### 使用RuntimeClass配置 + +RuntimeClass 是K8s的一种内置集群资源，是一种容器运行时配置，用于运行pod中的容器。 + +1. 在`/etc/isulad/daemon.json`中配置`isulad`： + + ```json + "runtimes": { + "runc-runtime": { + "path": "/usr/local/sbin/runc", + "runtime-args": [ + ] + } + }, + ``` + +2. 定义 `runc-runtime.yaml`，例如创建一个`runc-runtime.yaml`内容如下：(注意handler需要与daemon.json中的名称一致) + + ```yamlapiVersion: v1 + apiVersion: node.k8s.io/v1beta1 + kind: RuntimeClass + metadata: + name: runc-runtime + handler: runc-runtime + ``` + + 之后运行`kubectl apply -f runc-runtime.yaml`命令在kubectl中让这个配置生效。 + +3. 之后在创建pod时，可以在其定义的yaml文件中的`spec.runtimeClassName`中设置pod使用的runtime： + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: runc-pod-example +spec: + runtimeClassName: runc-runtime + containers: + - name: runc-pod + image: busybox:latest + command: ["/bin/sh"] + args: ["-c", "sleep 1000"] +``` + -- 2.25.1