From 5de4bc645d087edfbc848d43ac0019a6535a3c73 Mon Sep 17 00:00:00 2001 From: CoprDistGit Date: Mon, 17 Apr 2023 07:36:31 +0000 Subject: automatic import of iSulad --- 0049-add-runc-doc.patch | 506 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 506 insertions(+) create mode 100644 0049-add-runc-doc.patch (limited to '0049-add-runc-doc.patch') diff --git a/0049-add-runc-doc.patch b/0049-add-runc-doc.patch new file mode 100644 index 0000000..11f8de2 --- /dev/null +++ b/0049-add-runc-doc.patch @@ -0,0 +1,506 @@ +From eb3e8dd9e87e926a121715d103cc6c3cb20388a7 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Fri, 17 Feb 2023 14:33:15 +0800 +Subject: [PATCH 49/53] add runc doc + +Signed-off-by: zhongtao +--- + docs/design/README_zh.md | 1 + + .../design/detailed/Runtime/runc_design_zh.md | 264 ++++++++++++++++++ + docs/images/isulad_shim_flow_chart.svg | 16 ++ + docs/images/k8s_isulad_flow_chart.svg | 16 ++ + docs/manual/README_zh.md | 6 +- + docs/manual/runc_config_zh.md | 134 +++++++++ + 6 files changed, 436 insertions(+), 1 deletion(-) + create mode 100644 docs/design/detailed/Runtime/runc_design_zh.md + create mode 100755 docs/images/isulad_shim_flow_chart.svg + create mode 100644 docs/images/k8s_isulad_flow_chart.svg + create mode 100644 docs/manual/runc_config_zh.md + +diff --git a/docs/design/README_zh.md b/docs/design/README_zh.md +index 1f1c94b5..e7763a42 100644 +--- a/docs/design/README_zh.md ++++ b/docs/design/README_zh.md +@@ -49,6 +49,7 @@ + ## Runtime + + - 查看 runtime 模块的设计文档： [runtime_design](./detailed/Runtime/runtime_design_zh.md) 。 ++- 查看 isulad 支持runc 的设计文档：[runc_design](./detailed/Runtime/runc_design_zh.md)。 + + ## Security + +diff --git a/docs/design/detailed/Runtime/runc_design_zh.md b/docs/design/detailed/Runtime/runc_design_zh.md +new file mode 100644 +index 00000000..3af3ea02 +--- /dev/null ++++ b/docs/design/detailed/Runtime/runc_design_zh.md +@@ -0,0 +1,264 @@ ++| Author | zhongtao | ++| ------ | --------------------- | ++| Date | 2023-03-21 | ++| Email | zhongtao17@huawei.com | ++ ++# 方案目标 ++ ++isulad 支持使用runc作为容器runtime。用户可以在启动容器时配置`--runtime=runc`创建runtime为runc的容器： ++```bash ++$ isula run -tid -n test --runtime=runc busybox sh ++eb77c672a148cc4cefa0d6e7c5847f5a87d0c5353e245461b68820bd9af90c67 ++$ isula inspect eb77 | grep -i runc ++ "ResolvConfPath": "/var/lib/isulad/engines/runc/eb77c672a148cc4cefa0d6e7c5847f5a87d0c5353e245461b68820bd9af90c67/resolv.conf", ++ "HostsPath": "/var/lib/isulad/engines/runc/eb77c672a148cc4cefa0d6e7c5847f5a87d0c5353e245461b68820bd9af90c67/hosts", ++ "LogPath": "/var/lib/isulad/engines/runc/eb77c672a148cc4cefa0d6e7c5847f5a87d0c5353e245461b68820bd9af90c67/console.log", ++ "Runtime": "runc", ++ "log.console.file": "/var/lib/isulad/engines/runc/eb77c672a148cc4cefa0d6e7c5847f5a87d0c5353e245461b68820bd9af90c67/console.log", ++``` ++ ++同时，也可以在`/etc/isulad/daemon.json`中配置`default-runtime`为runc并重启isulad，修改isulad创建容器时默认使用的runtime。 ++```sh ++$ vim /etc/isulad/daemon.json ++ ... ++ "default-runtime": "runc" ++ ... ++$ sudo isulad ++``` ++ ++# 总体设计 ++ ++由于isulad与runc之间的交互存在gap，且将容器创建成功之后，容器进程的生命周期与isulad进程的生命周期没有必然联系，因此我们设计了一个isulad-shim进程，用于isulad与runc的交互并将isulad与容器实例解耦。同时，由于只有create以及exec涉及到在容器中新建进程，因此只有这两个子命令需要创建isulad-shim。其他的子命令直接通过调用runc二进制实现。 ++ ++## 时序图 ++ ++```mermaid ++sequenceDiagram ++ participant isula ++ participant kubelet ++ participant isulad ++ participant supervisor ++ participant shim ++ participant runc ++ participant container ++ ++ isula->>isulad: request ++ kubelet->>isulad:request ++ alt create ++ isulad->>shim:shim_creat() ++ shim->>runc: execvp(runc, params) ++ runc ->> container:create request ++ container ->> runc:success ++ runc ->> shim:get process pid ++ isulad ->> isulad:get process pid ++ isulad ->> supervisor:add monitor ++ loop epoll exit_fd ++ supervisor ->> shim:if exit? ++ end ++ else exec ++ isulad->>shim:shim_creat() ++ par ++ shim->>runc: execvp(runc, params) ++ runc ->> container:exec request ++ container ->> runc:success ++ runc ->> shim:get process pid ++ shim ->> container:wait process pid ++ shim ->> shim:exit ++ and ++ isulad ->> isulad: wait isulad-shim pid ++ end ++ else others container cmd ++ isulad->>runc: runtime_call_simple() ++ runc ->> container:cmd ++ container ->>runc:success ++ runc ->>isulad:success ++ end ++ isulad ->> isula:response ++ isulad ->> kubelet:response ++``` ++ ++# 接口描述 ++ ++## cri接口 ++ ++### PodSandboxManagerService ++ ++```h ++auto RunPodSandbox(const runtime::v1alpha2::PodSandboxConfig &config, const std::string &runtimeHandler,Errors &error) -> std::string; ++ ++void StopPodSandbox(const std::string &podSandboxID, Errors &error); ++ ++void RemovePodSandbox(const std::string &podSandboxID, Errors &error); ++ ++auto PodSandboxStatus(const std::string &podSandboxID, Errors &error) ++ -> std::unique_ptr; ++ ++void ListPodSandbox(const runtime::v1alpha2::PodSandboxFilter *filter,std::vector> *pods, Errors &error); ++ ++// This feature is temporarily not supported ++void PortForward(const runtime::v1alpha2::PortForwardRequest &req,runtime::v1alpha2::PortForwardResponse *resp,Errors &error); ++ ... ... ++}; ++} // namespace CRI ++``` ++ ++### ContainerManagerService ++ ++```c ++auto CreateContainer(const std::string &podSandboxID, const runtime::v1alpha2::ContainerConfig &containerConfig,const runtime::v1alpha2::PodSandboxConfig &podSandboxConfig, Errors &error)-> std::string override; ++ ++void StartContainer(const std::string &containerID, Errors &error) override; ++ ++void StopContainer(const std::string &containerID, int64_t timeout, Errors &error) override; ++ ++void RemoveContainer(const std::string &containerID, Errors &error) override; ++ ++void ListContainers(const runtime::v1alpha2::ContainerFilter *filter, ++ std::vector> *containers, Errors &error) override; ++ ++void ListContainerStats(const runtime::v1alpha2::ContainerStatsFilter *filter,std::vector> *containerstats,Errors &error) override; ++ ++auto ContainerStats(const std::string &containerID, Errors &error) ++ -> std::unique_ptr override; ++ ++auto ContainerStatus(const std::string &containerID, Errors &error) -> std::unique_ptr override; ++ ++void ExecSync(const std::string &containerID, const google::protobuf::RepeatedPtrField &cmd, int64_t timeout, runtime::v1alpha2::ExecSyncResponse *reply, Errors &error) override; ++ ++void Exec(const runtime::v1alpha2::ExecRequest &req, runtime::v1alpha2::ExecResponse *resp, Errors &error) override; ++ ++void Attach(const runtime::v1alpha2::AttachRequest &req, runtime::v1alpha2::AttachResponse *resp, Errors &error) override; ++ ++void UpdateContainerResources(const std::string &containerID, ++ const runtime::v1alpha2::LinuxContainerResources &resources, Errors &error) override; ++``` ++ ++### RuntimeManagerService ++ ++```c ++void UpdateRuntimeConfig(const runtime::v1alpha2::RuntimeConfig &config, Errors &error) override; ++ ++// 对应crictl info ++auto Status(Errors &error) -> std::unique_ptr override; ++``` ++ ++### ImageManagerService ++ ++```c ++void ListImages(const runtime::v1alpha2::ImageFilter &filter, std::vector> *images, Errors &error) override; std::unique_ptr ImageStatus(const runtime::v1alpha2::ImageSpec &image, Errors &error) override; ++ ++std::string PullImage(const runtime::v1alpha2::ImageSpec &image, const runtime::v1alpha2::AuthConfig &auth, Errors &error) override; ++ ++void RemoveImage(const runtime::v1alpha2::ImageSpec &image, Errors &error) override; ++ ++void ImageFsInfo(std::vector> *usages, Errors &error) override ++``` ++ ++ ++ ++## isula_rt_ops模块 ++ ++```c ++// 检测runtime是否为isula_rt_ops模块处理的目标runtime ++bool rt_isula_detect(const char *runtime); ++ ++int rt_isula_create(const char *name, const char *runtime, const rt_create_params_t *params); ++ ++int rt_isula_start(const char *name, const char *runtime, const rt_start_params_t *params, pid_ppid_info_t *pid_info); ++ ++// restart not implemented ++int rt_isula_restart(const char *name, const char *runtime, const rt_restart_params_t *params); ++ ++int rt_isula_clean_resource(const char *name, const char *runtime, const rt_clean_params_t *params); ++ ++int rt_isula_rm(const char *name, const char *runtime, const rt_rm_params_t *params); ++ ++int rt_isula_exec(const char *id, const char *runtime, const rt_exec_params_t *params, int *exit_code); ++ ++int rt_isula_status(const char *name, const char *runtime, const rt_status_params_t *params, ++ struct runtime_container_status_info *status); ++ ++// isula attach not support on isulad-shim ++int rt_isula_attach(const char *id, const char *runtime, const rt_attach_params_t *params); ++ ++int rt_isula_update(const char *id, const char *runtime, const rt_update_params_t *params); ++ ++int rt_isula_pause(const char *id, const char *runtime, const rt_pause_params_t *params); ++ ++int rt_isula_resume(const char *id, const char *runtime, const rt_resume_params_t *params); ++ ++// isula top/listpids not support on isulad-shim ++int rt_isula_listpids(const char *name, const char *runtime, const rt_listpids_params_t *params, ++ rt_listpids_out_t *out); ++ ++int rt_isula_resources_stats(const char *name, const char *runtime, const rt_stats_params_t *params, ++ struct runtime_container_resources_stats_info *rs_stats); ++// rt_isula_resize not impl ++int rt_isula_resize(const char *id, const char *runtime, const rt_resize_params_t *params); ++ ++int rt_isula_exec_resize(const char *id, const char *runtime, const rt_exec_resize_params_t *params); ++ ++int rt_isula_kill(const char *id, const char *runtime, const rt_kill_params_t *params); ++ ++``` ++ ++ ++ ++# 详细设计 ++ ++## create 实现流程 ++ ++isulad端： ++ ++1. 创建process文件：create_process_json_file(); ++2. 获得runtime二进制：get_runtime_cmd(runtime, &cmd); ++3. 利用两次fork()创建isulad-shim进程， 此时的isulad-shim进程是1号进程的子进程，与isulad无父子关系:shim_create(); ++4. 若shim创建成功则直接返回成功，若失败则调用runtime_call_delete_force()将容器进程都force delete。 ++ ++isulad-shim端： ++ ++1. 根据解析process文件新建一个process，new_process()； ++2. 开启exit_fifo：open_no_inherit("exit_fifo", O_WRONLY, -1); ++3. 为io_copy创建io_epoll_loop线程：process_io_init(); ++4. 创建进行io copy的线程，并将isulad的console与runtime的console连接起来:open_io(); ++5. 创建运行runtime的子进程，获得容器中进程的pid，create_process(); ++6. 循环wait子进程，直到wait到的子进程为容器进程pid则退出shim进程，process_signal_handle_routine(); ++ ++## exec 实现流程 ++ ++isulad端： ++ ++1. 创建process文件：create_process_json_file(); ++2. 获得runtime二进制：get_runtime_cmd(runtime, &cmd); ++3. 若`--detach=false`，直接fork()创建isulad-shim进程，此时的isulad-shim进程是isulad进程的子进程，isulad进程wait等待isulad-shim进程退出；若-`-detach=true`，则与create一样，创建独立的isulad-shim进程:shim_create(); ++4. 循环读取pid文件获得容器进程id说明exec命令成功，get_container_process_pid。 ++ ++isulad-shim端： ++ ++1. 根据解析process文件新建一个process，new_process()； ++2. 为io_copy创建io_epoll_loop线程：process_io_init(); ++3. 创建进行io copy的线程，并将isulad的console与runtime的console连接起来:open_io(); ++4. 创建运行runtime的子进程，获得容器中进程的pid，create_process(); ++5. 循环wait子进程，直到wait到的子进程为容器进程pid则退出shim进程，process_signal_handle_routine(); ++ ++ ++ ++## start 实现流程 ++ ++isulad端： ++ ++1. 分别读取pid以及shim-pid文件获得容器进程pid以及isulad-shim pid； ++2. 依次获得容器进程以及isulad-shim进程的proc信息：util_get_process_proc_info(); ++3. 根据proc信息为pid_ppid_info_t结构体赋值； ++4. 直接调用runc二进制start容器：runtime_call_simple(); ++5. 之后isulad根据监听exit_fifo_fd，感知容器是否退出，从而更新容器状态。 ++ ++其他子命令与satrt类似，均是调用runtime_call_simple()函数直接调用runc二进制，此处不再赘述。 ++ ++ ++ ++## 流程图 ++ ++### isulad与isulad-shim交互流程图 ++ ++![isulad_shim_flow_chart](../../../images/isulad_shim_flow_chart.svg) +diff --git a/docs/images/isulad_shim_flow_chart.svg b/docs/images/isulad_shim_flow_chart.svg +new file mode 100755 +index 00000000..9a509b80 +--- /dev/null ++++ b/docs/images/isulad_shim_flow_chart.svg +@@ -0,0 +1,16 @@ ++ ++ ++ ++ ++ ++ ++ isulad-shimmain()isulad-shim id bundle runtime info 2m0s从process.json中加载process：new_processif p->state->exec?将create的process存储在process.jsonwe文件中：create_process_json_fileisuladget_runtime_cmdret = shim_createret = -1runtime_call_delete_forcert_isula_creatert_isula_execget_runtime_cmdret = shim_createget_container_process_pid将exec的process存储在process.jsonwe文件中create_process_json_fileget_container_process_pidfile_read_int(shim_pid_file_name, &shim_pid);util_get_process_proc_inforuntime_call_simple(workdir,runtime, "start", NULL, 0, id, NULL)rt_isula_startfg=false;exit_code=NULLshim_createfork()若设置了--detach，则fg=true;否则为false；exit_code!=NULLparent(isulad)child(shim)从读fifo中读取isulad-shimxier写入的内容num = util_read_nointrif num > 0nowait isulad-shim或者shim parent 进程util_waitpid_with_timeoutif ret != 0show_shim_runtime_errlogif timeout <= 0kill(pid, SIGKILL)yeschdir(workdir)if fgnounsetenv("NOTIFY_SOCKET")fork()parent(shim parent)child(isulad-shim)将shim的pid写入文件中：file_write_int(fpid, pid)_exit(EXIT_SUCCESS);setsid()将从父进程继承的fd关闭util_check_inheritedexecvp(SHIM_BINARY, (char * const *)params);设置启动isulad-shim超时的定时器set_timeout_exit(DEFAULT_TIMEOUT);set_subreaper()解析参数：parse_argsyes打开exit_fd用于感知退出：open_no_inherit("exit_fifo", O_WRONLY, -1)为isulad与runtime之间的io copychuan构建main loop和epoll：：process_io_init(p);open_io(p, &tid_accept);no创建io copy线程：start_io_copy_threads(p);if (p->state->terminal) true开勇socket实现isulad与runtime的通信：open_terminal_io(p, tid_accept);利用pipe实现通信：open_generic_io(p);falsecreate_process(p);fork()child:runtime processparent:isulad-shim process拼接params，并调用runc二进制执行命令：exec_runtime_process(p, exec_fd[1]);读取runtime写入的信息： read_nointr(exec_fd[0],exec_buff, sizeof(exec_buff) - 1);等待runtime子进程退出：waitpid(pid, NULL, 0);获得runtime写入的容器中进程的pid：read_text_file("pid");将其赋值给process：p->ctr_pid = ctr_pid;清除超时计时器：released_timeout_exit();process_signal_handle_routine(p, tid_accept);wait任意一个子进程退出，当pid为ctr_pid时ret为0且exit_shim = true：reap_container(p->ctr_pid, &status);if ret == 0? noyesif exit_shim? yes调用runc kill命令杀死容器进程：process_kill_all(p);等待所有子进程被杀死：DO_RETRY_CALL(120, 1000000, nret, try_wait_all_child);调用runc delete命令删除容器进程：process_delete(p)if p->exit_fd > 0yeswrite_nointr(p->exit_fd, &status, sizeof(int));pthread_timedjoin_np(tid_accept, NULL, &ts);nodestroy_io_thread(p, i); +\ No newline at end of file +diff --git a/docs/images/k8s_isulad_flow_chart.svg b/docs/images/k8s_isulad_flow_chart.svg +new file mode 100644 +index 00000000..700b5b6d +--- /dev/null ++++ b/docs/images/k8s_isulad_flow_chart.svg +@@ -0,0 +1,16 @@ ++ ++ ++ ++ ++ ++ ++ kubeletclientisuladservergrpcCRIPodSandboxManagerServiceruntime service implImageManagerServiceservice_executor_t：callback函数Network::PluginManagerRunPodSandbox1. Pull the image for the sandboxEnsureSandboxImageExists2. Create the sandbox containerCreateSandboxContainerRuntimeVersionerServiceContainerManagerServicePodSandboxManagerServiceRuntimeManagerServiceRuntimeManagerServiceContainerManagerServiceRuntimeVersionerServiceservice_executor_t：callback函数Network::PluginManagerservice_executor_tservice_executor_tNetwork::PluginManager1. 创建create容器的requestGenerateSandboxCreateContainerRequest2. 发送请求m_cb->container.create(create_request, &create_response)3: Enable network SetNetworkReady4: Inspect container：直接从container_t中获得容器信息：inspect_data =CRIHelpers::InspectContainer5: Get networking info GetSandboxNetworkInfo 6: Mount network namespace when network mode is cninamespace_is_cn；prepare_network_namespace7: Setup networking for the sandbox. SetupSandboxNetwork8: Start the sandbox container. StartSandboxContainer 9: Save network settings json to disk ips_request = GenerateUpdateNetworkSettingsReqestinput：run pod的config文件解析出来的结构体output：pause容器的idCreateContainerinput：podSandboxID、containerConfig、podSandboxConfigoutput：pod中容器的id1. 根据podsandbox的id或者名字获得容器idGetRealContainerOrSandboxID2. 获得pod的runtimeGetContainerOrSandboxRuntime3. 创建pod中的容器的create请求GenerateCreateContainerRequest4. 调用创建cbm_cb->container.create +\ No newline at end of file +diff --git a/docs/manual/README_zh.md b/docs/manual/README_zh.md +index 8de5604b..1f3f37f8 100644 +--- a/docs/manual/README_zh.md ++++ b/docs/manual/README_zh.md +@@ -22,4 +22,8 @@ Device Mapper 是一个基于内核的框架，它支持 Linux 上的许多高 + + ## isula search使用指南 + +-关于如何使用isula search请参考[isula_search](isula_search_zh.md) +\ No newline at end of file ++关于如何使用isula search请参考[isula_search](isula_search_zh.md) ++ ++## runc使用指南 ++ ++关于如何在isulad中使用runc请参考[runc_config](runc_config_zh.md) +\ No newline at end of file +diff --git a/docs/manual/runc_config_zh.md b/docs/manual/runc_config_zh.md +new file mode 100644 +index 00000000..003cd2c5 +--- /dev/null ++++ b/docs/manual/runc_config_zh.md +@@ -0,0 +1,134 @@ ++# runc使用指南 ++本文主要是指导iSulad社区开发者和使用者，如何配置isulad使用runc作为runtime创建容器。 ++ ++## 一、runc的安装 ++ ++`tips`: 在安装runc之前需要安装好go环境。 ++ ++isulad当前推荐的runc验证版本为v1.0.0-rc5。 ++ ++runc可以使用以下两种安装方式： ++ ++1. 直接使用包管理器安装runc: ++ ++```sh ++# centOS ++sudo yum install runc ++# Ubuntu ++sudo apt-get install runc ++``` ++ ++2. 源码编译安装runc（注意建议切换成isulad推荐的runc版本：`git checkout v1.0.0-rc5`） ++ ++```sh ++# 在GOPATH/src下创建 'github.com/opencontainers' 文件夹 ++cd github.com/opencontainers ++git clone https://github.com/opencontainers/runc ++cd runc ++ ++make ++sudo make install ++``` ++ ++还可以使用go get安装到`GOPATH`路径下（需要在GOPATH/src下创建github.com父文件夹）： ++ ++```sh ++go get github.com/opencontainers/runc ++cd $GOPATH/src/github.com/opencontainers/runc ++make ++sudo make install ++``` ++ ++最终安装好的runc会在`/usr/local/sbin/runc`目录下。 ++ ++## 二、配置iSulad使用runc ++ ++### 配置文件配置 ++ ++1. 修改isulad的daemon.json，配置isulad默认使用的runtime。 ++ ++```sh ++$ vim /etc/isulad/daemon.json ++ ... ++ "default-runtime": "runc" ++ ... ++``` ++ ++2. 也可以在配置文件中配置runtimes，在其中指定使用的`path`(用于修改isulad使用的runc路径)以及`runtime-args`(对runtime所有命令配置的参数)。 ++ ++```sh ++"runtimes": { ++ "runc": { ++ "path": "/usr/local/sbin/runc", ++ "runtime-args": [ ++ ] ++ } ++ }, ++``` ++ ++之后使用root权限启动isulad服务，使修改后的配置生效即可： ++ ++```sh ++$ sudo isulad ++``` ++ ++### 单个容器配置 ++ ++使用`--runtime=runc`启动一个runtime为runc的容器。 ++ ++```sh ++isula run -tid -n test --runtime=runc busybox sh ++``` ++ ++## 三、K8s中配置pod的runtime为runc ++ ++如何与kubernetes集成请参考[k8s_integration](https://gitee.com/openeuler/iSulad/blob/master/docs/manual/k8s_integration_zh.md)。 ++ ++### 全局配置 ++ ++直接参照第二节中配置文件配置的方式修改isulad默认使用的runtime为runc，则后续使用k8s启动容器时会默认使用的runtime即为runc。 ++ ++### 使用RuntimeClass配置 ++ ++RuntimeClass 是K8s的一种内置集群资源，是一种容器运行时配置，用于运行pod中的容器。 ++ ++1. 在`/etc/isulad/daemon.json`中配置`isulad`： ++ ++ ```json ++ "runtimes": { ++ "runc-runtime": { ++ "path": "/usr/local/sbin/runc", ++ "runtime-args": [ ++ ] ++ } ++ }, ++ ``` ++ ++2. 定义 `runc-runtime.yaml`，例如创建一个`runc-runtime.yaml`内容如下：(注意handler需要与daemon.json中的名称一致) ++ ++ ```yamlapiVersion: v1 ++ apiVersion: node.k8s.io/v1beta1 ++ kind: RuntimeClass ++ metadata: ++ name: runc-runtime ++ handler: runc-runtime ++ ``` ++ ++ 之后运行`kubectl apply -f runc-runtime.yaml`命令在kubectl中让这个配置生效。 ++ ++3. 之后在创建pod时，可以在其定义的yaml文件中的`spec.runtimeClassName`中设置pod使用的runtime： ++ ++```yaml ++apiVersion: v1 ++kind: Pod ++metadata: ++ name: runc-pod-example ++spec: ++ runtimeClassName: runc-runtime ++ containers: ++ - name: runc-pod ++ image: busybox:latest ++ command: ["/bin/sh"] ++ args: ["-c", "sleep 1000"] ++``` ++ +-- +2.25.1 + -- cgit v1.2.3