diff options
author | CoprDistGit <infra@openeuler.org> | 2023-04-17 07:36:31 +0000 |
---|---|---|
committer | CoprDistGit <infra@openeuler.org> | 2023-04-17 07:36:31 +0000 |
commit | 5de4bc645d087edfbc848d43ac0019a6535a3c73 (patch) | |
tree | 7354cf7f0504f1020c7c3165c77c21ef9257cbb4 | |
parent | 4a79e40e226e2077b5e8cc274610e473b148f507 (diff) |
automatic import of iSuladopeneuler20.03
56 files changed, 14260 insertions, 0 deletions
@@ -0,0 +1 @@ +/v2.1.1.tar.gz diff --git a/0001-modify-dependence-from-lcr-to-libisula.patch b/0001-modify-dependence-from-lcr-to-libisula.patch new file mode 100644 index 0000000..83d70eb --- /dev/null +++ b/0001-modify-dependence-from-lcr-to-libisula.patch @@ -0,0 +1,206 @@ +From d3d13f374bb341e7e4d389feb1780e43fec91945 Mon Sep 17 00:00:00 2001 +From: zhangxiaoyu <zhangxiaoyu58@huawei.com> +Date: Tue, 7 Feb 2023 14:27:51 +0800 +Subject: [PATCH 01/53] modify dependence from lcr to libisula + +Signed-off-by: zhangxiaoyu <zhangxiaoyu58@huawei.com> +--- + cmake/checker.cmake | 2 +- + iSulad.spec | 10 ++++----- + isulad.pc.in | 6 +++--- + src/CMakeLists.txt | 32 ++++++++++++++-------------- + src/daemon/modules/runtime/runtime.c | 14 ++---------- + 5 files changed, 27 insertions(+), 37 deletions(-) + +diff --git a/cmake/checker.cmake b/cmake/checker.cmake +index 0a4b6fb6..a1ca9360 100644 +--- a/cmake/checker.cmake ++++ b/cmake/checker.cmake +@@ -92,7 +92,7 @@ if (ENABLE_SELINUX) + endif() + + # check iSula libutils +-pkg_check_modules(PC_ISULA_LIBUTILS REQUIRED "lcr") ++pkg_check_modules(PC_ISULA_LIBUTILS REQUIRED "libisula") + find_path(ISULA_LIBUTILS_INCLUDE_DIR isula_libutils/log.h + HINTS ${PC_ISULA_LIBUTILS_INCLUDEDIR} ${PC_ISULA_LIBUTILS_INCLUDE_DIRS}) + _CHECK(ISULA_LIBUTILS_INCLUDE_DIR "ISULA_LIBUTILS_INCLUDE_DIR-NOTFOUND" "isula_libutils/log.h") +diff --git a/iSulad.spec b/iSulad.spec +index 79275885..3cff9c25 100644 +--- a/iSulad.spec ++++ b/iSulad.spec +@@ -16,7 +16,7 @@ ExclusiveArch: x86_64 aarch64 + + %ifarch x86_64 aarch64 + Provides: libhttpclient.so()(64bit) +-Provides: libisula.so()(64bit) ++Provides: libisula_client.so()(64bit) + Provides: libisulad_img.so()(64bit) + Provides: libisulad_tools.so()(64bit) + %endif +@@ -42,14 +42,14 @@ BuildRequires: lib-shim-v2-devel + Requires: lib-shim-v2 + %endif + +-BuildRequires: cmake gcc-c++ lxc-devel lcr-devel yajl-devel ++BuildRequires: cmake gcc-c++ lxc-devel lcr-devel yajl-devel libisula-devel + BuildRequires: grpc-plugins grpc-devel protobuf-devel + BuildRequires: libcurl-devel libarchive-devel device-mapper-devel + BuildRequires: http-parser-devel + BuildRequires: libselinux-devel libwebsockets-devel + BuildRequires: systemd-devel git + +-Requires: lcr lxc ++Requires: libisula lxc + Requires: grpc libcurl http-parser + Requires: libselinux libwebsockets libarchive device-mapper + Requires: systemd +@@ -78,13 +78,13 @@ cd build + rm -rf %{buildroot} + cd build + install -d $RPM_BUILD_ROOT/%{_libdir} +-install -m 0644 ./src/libisula.so %{buildroot}/%{_libdir}/libisula.so ++install -m 0644 ./src/libisula_client.so %{buildroot}/%{_libdir}/libisula_client.so + install -m 0644 ./src/utils/http/libhttpclient.so %{buildroot}/%{_libdir}/libhttpclient.so + chrpath -d ./src/libisulad_tools.so + install -m 0644 ./src/libisulad_tools.so %{buildroot}/%{_libdir}/libisulad_tools.so + chrpath -d ./src/daemon/modules/image/libisulad_img.so + install -m 0644 ./src/daemon/modules/image/libisulad_img.so %{buildroot}/%{_libdir}/libisulad_img.so +-chmod +x %{buildroot}/%{_libdir}/libisula.so ++chmod +x %{buildroot}/%{_libdir}/libisula_client.so + chmod +x %{buildroot}/%{_libdir}/libhttpclient.so + chmod +x %{buildroot}/%{_libdir}/libisulad_img.so + +diff --git a/isulad.pc.in b/isulad.pc.in +index 695ba364..016e406d 100644 +--- a/isulad.pc.in ++++ b/isulad.pc.in +@@ -3,10 +3,10 @@ libdir=@CMAKE_INSTALL_PREFIX@/lib + localstatedir=@CMAKE_INSTALL_PREFIX@/var + includedir=@CMAKE_INSTALL_PREFIX@/include + +-Name: libisula +-Description: light-weighted container runtime daemon library ++Name: libisula_client ++Description: light-weighted container client library + Version: @ISULAD_VERSION@ + URL: iSulad +-Libs: -L@CMAKE_INSTALL_PREFIX@/lib -lisula ++Libs: -L@CMAKE_INSTALL_PREFIX@/lib -lisula_client + Cflags: -I@CMAKE_INSTALL_PREFIX@/include + +diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt +index abce1284..8de18082 100644 +--- a/src/CMakeLists.txt ++++ b/src/CMakeLists.txt +@@ -60,40 +60,40 @@ endif() + # get all c and header files + list(REMOVE_DUPLICATES SHARED_INCS) + +-# ------ build libisula ------ ++# ------ build libisula_client ------ + + add_subdirectory(client) + if (OPENSSL_VERIFY) + list(APPEND CLIENT_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/utils/http/certificate.c) + endif() + +-add_library(libisula ${LIBTYPE} ++add_library(libisula_client ${LIBTYPE} + ${CLIENT_SRCS} + ) + +-target_include_directories(libisula PUBLIC ++target_include_directories(libisula_client PUBLIC + ${SHARED_INCS} + ${CLIENT_INCS} + ${CMAKE_CURRENT_SOURCE_DIR}/utils/http + ) + +-# set libisula FLAGS +-set_target_properties(libisula PROPERTIES PREFIX "") ++# set libisula_client FLAGS ++set_target_properties(libisula_client PROPERTIES PREFIX "") + +-target_link_libraries(libisula libisulad_tools) ++target_link_libraries(libisula_client libisulad_tools) + + if (GRPC_CONNECTOR) +- target_link_libraries(libisula -Wl,--as-needed -lstdc++) +- target_link_libraries(libisula -Wl,--as-needed ${PROTOBUF_LIBRARY}) +- target_link_libraries(libisula -Wl,--no-as-needed ${GRPC_PP_REFLECTION_LIBRARY} ${GRPC_PP_LIBRARY} ${GRPC_LIBRARY} ${GPR_LIBRARY}) ++ target_link_libraries(libisula_client -Wl,--as-needed -lstdc++) ++ target_link_libraries(libisula_client -Wl,--as-needed ${PROTOBUF_LIBRARY}) ++ target_link_libraries(libisula_client -Wl,--no-as-needed ${GRPC_PP_REFLECTION_LIBRARY} ${GRPC_PP_LIBRARY} ${GRPC_LIBRARY} ${GPR_LIBRARY}) + if(ABSL_SYNC_LIB) +- target_link_libraries(libisula -Wl,--no-as-needed ${ABSL_SYNC_LIB}) ++ target_link_libraries(libisula_client -Wl,--no-as-needed ${ABSL_SYNC_LIB}) + endif() + else() +- target_link_libraries(libisula -ldl libhttpclient) +- set_target_properties(libisula PROPERTIES LINKER_LANGUAGE "C") ++ target_link_libraries(libisula_client -ldl libhttpclient) ++ set_target_properties(libisula_client PROPERTIES LINKER_LANGUAGE "C") + endif() +-# ------ build libisula finish ----- ++# ------ build libisula_client finish ----- + + add_subdirectory(cmd) + # ------ build isula ------- +@@ -101,7 +101,7 @@ add_executable(isula + ${ISULA_SRCS} + ) + target_include_directories(isula PUBLIC ${ISULA_INCS} ${SHARED_INCS}) +-target_link_libraries(isula libisula ${LIBYAJL_LIBRARY}) ++target_link_libraries(isula libisula_client ${LIBYAJL_LIBRARY}) + if (ANDROID OR MUSL) + target_link_libraries(isula ${LIBSSL_LIBRARY}) + else() +@@ -192,7 +192,7 @@ endif() + + if (ISULAD_GCOV) + target_link_libraries(isula -lgcov) +- target_link_libraries(libisula -lgcov) ++ target_link_libraries(libisula_client -lgcov) + target_link_libraries(isulad -lgcov) + endif() + +@@ -201,7 +201,7 @@ endif() + # ------ install binary -------- + install(TARGETS libisulad_tools + ${INSTALL_TYPE} DESTINATION ${LIB_INSTALL_DIR_DEFAULT} PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) +-install(TARGETS libisula ++install(TARGETS libisula_client + ${INSTALL_TYPE} DESTINATION ${LIB_INSTALL_DIR_DEFAULT} PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) + install(TARGETS isula + RUNTIME DESTINATION bin PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) +diff --git a/src/daemon/modules/runtime/runtime.c b/src/daemon/modules/runtime/runtime.c +index 23a4d60f..7a3ed87f 100644 +--- a/src/daemon/modules/runtime/runtime.c ++++ b/src/daemon/modules/runtime/runtime.c +@@ -542,20 +542,10 @@ bool is_default_runtime(const char *name) + + int runtime_init() + { +- int ret = 0; +- + if (engines_global_init()) { + ERROR("Init engines global failed"); +- ret = -1; +- goto out; ++ return -1; + } + +- /* Init default engine, now is lcr */ +- if (engines_discovery(DEFAULT_RUNTIME_NAME)) { +- ERROR("Failed to discovery engine %s", DEFAULT_RUNTIME_NAME); +- ret = -1; +- } +- +-out: +- return ret; ++ return 0; + } +-- +2.25.1 + diff --git a/0002-Add-unified-memory_swap_limit_in_bytes-fields-into-C.patch b/0002-Add-unified-memory_swap_limit_in_bytes-fields-into-C.patch new file mode 100644 index 0000000..b492de1 --- /dev/null +++ b/0002-Add-unified-memory_swap_limit_in_bytes-fields-into-C.patch @@ -0,0 +1,243 @@ +From 27c3d00f74c5641685d5781fe0c02c5eead92d23 Mon Sep 17 00:00:00 2001 +From: "ilya.kuksenok" <ilya.kuksenok@huawei.com> +Date: Thu, 2 Feb 2023 14:41:16 +0300 +Subject: [PATCH 02/53] Add unified, memory_swap_limit_in_bytes fields into + ContainerStats; add unified and memory_swap_limit_in_bytes into + UpdateCreateConfig add nullptr for unified. + +--- + src/daemon/common/sysinfo.c | 3 +- + .../cri/cri_container_manager_service.cc | 19 ++++++++++++ + src/daemon/entry/cri/cri_helpers.cc | 17 ++++++++++- + src/daemon/modules/runtime/runtime.c | 30 +++++++++---------- + 4 files changed, 52 insertions(+), 17 deletions(-) + +diff --git a/src/daemon/common/sysinfo.c b/src/daemon/common/sysinfo.c +index 8b5768db..38416db4 100644 +--- a/src/daemon/common/sysinfo.c ++++ b/src/daemon/common/sysinfo.c +@@ -743,7 +743,8 @@ static void check_cgroup_mem(struct layer **layers, bool quiet, cgroup_mem_info_ + return; + } + +- meminfo->limit = true; ++ meminfo->limit = cgroup_enabled(mountpoint, CGROUP_MEMORY_LIMIT); ++ cgroup_do_log(quiet, !(meminfo->limit), "Your kernel does not support memory limit"); + + meminfo->swap = cgroup_enabled(mountpoint, CGROUP_MEMORY_SWAP); + cgroup_do_log(quiet, !(meminfo->swap), "Your kernel does not support swap memory limit"); +diff --git a/src/daemon/entry/cri/cri_container_manager_service.cc b/src/daemon/entry/cri/cri_container_manager_service.cc +index d2f486cf..d044cca8 100644 +--- a/src/daemon/entry/cri/cri_container_manager_service.cc ++++ b/src/daemon/entry/cri/cri_container_manager_service.cc +@@ -1084,6 +1084,12 @@ void ContainerManagerService::UpdateContainerResources(const std::string &contai + struct parser_context ctx { + OPT_GEN_SIMPLIFY, 0 + }; ++ json_map_string_string *unified = nullptr; ++ unified = (json_map_string_string *)util_common_calloc_s(sizeof(json_map_string_string)); ++ if (unified == nullptr) { ++ error.SetError("Out of memory"); ++ goto cleanup; ++ } + request = (container_update_request *)util_common_calloc_s(sizeof(container_update_request)); + if (request == nullptr) { + error.SetError("Out of memory"); +@@ -1100,6 +1106,18 @@ void ContainerManagerService::UpdateContainerResources(const std::string &contai + hostconfig->cpu_period = resources.cpu_period(); + hostconfig->cpu_quota = resources.cpu_quota(); + hostconfig->cpu_shares = resources.cpu_shares(); ++ hostconfig->memory_swap_limit_in_bytes = resources.memory_swap_limit_in_bytes(); ++ ++ if (!resources.unified().empty()) { ++ for (auto &iter : resources.unified()) { ++ if (append_json_map_string_string(unified, iter.first.c_str(), iter.second.c_str()) != 0) { ++ error.SetError("Failed to append string"); ++ goto cleanup; ++ } ++ } ++ } ++ hostconfig->unified = unified; ++ unified = nullptr; + hostconfig->memory = resources.memory_limit_in_bytes(); + if (!resources.cpuset_cpus().empty()) { + hostconfig->cpuset_cpus = util_strdup_s(resources.cpuset_cpus().c_str()); +@@ -1126,6 +1144,7 @@ cleanup: + free_container_update_request(request); + free_container_update_response(response); + free_host_config(hostconfig); ++ free_json_map_string_string(unified); + free(perror); + } + +diff --git a/src/daemon/entry/cri/cri_helpers.cc b/src/daemon/entry/cri/cri_helpers.cc +index ddcc153f..2f6dcf78 100644 +--- a/src/daemon/entry/cri/cri_helpers.cc ++++ b/src/daemon/entry/cri/cri_helpers.cc +@@ -445,8 +445,23 @@ void UpdateCreateConfig(container_config *createConfig, host_config *hc, + hc->cpuset_mems = util_strdup_s(rOpts.cpuset_mems().c_str()); + } + hc->oom_score_adj = rOpts.oom_score_adj(); ++ hc->memory_swap_limit_in_bytes = rOpts.memory_swap_limit_in_bytes(); ++ auto *unified = (json_map_string_string *)util_common_calloc_s(sizeof(json_map_string_string)); ++ if (unified == nullptr) { ++ error.SetError("Out of memory"); ++ return; ++ } ++ if (!rOpts.unified().empty()) { ++ for (auto &iter : rOpts.unified()) { ++ if (append_json_map_string_string(unified, iter.first.c_str(), iter.second.c_str()) != 0) { ++ error.SetError("Failed to append string"); ++ free_json_map_string_string(unified); ++ return; ++ } ++ } ++ } ++ hc->unified = unified; + } +- + createConfig->open_stdin = config.stdin(); + createConfig->tty = config.tty(); + } +diff --git a/src/daemon/modules/runtime/runtime.c b/src/daemon/modules/runtime/runtime.c +index 7a3ed87f..29a64ac1 100644 +--- a/src/daemon/modules/runtime/runtime.c ++++ b/src/daemon/modules/runtime/runtime.c +@@ -122,7 +122,7 @@ int runtime_create(const char *name, const char *runtime, const rt_create_params + const struct rt_ops *ops = NULL; + + if (name == NULL || runtime == NULL) { +- ERROR("Invalide arguments for runtime create"); ++ ERROR("Invalid arguments for runtime create"); + ret = -1; + goto out; + } +@@ -146,7 +146,7 @@ int runtime_start(const char *name, const char *runtime, const rt_start_params_t + const struct rt_ops *ops = NULL; + + if (name == NULL || runtime == NULL || pid_info == NULL) { +- ERROR("Invalide arguments for runtime start"); ++ ERROR("Invalid arguments for runtime start"); + ret = -1; + goto out; + } +@@ -194,7 +194,7 @@ int runtime_restart(const char *name, const char *runtime, const rt_restart_para + const struct rt_ops *ops = NULL; + + if (name == NULL || runtime == NULL) { +- ERROR("Invalide arguments for runtime restart"); ++ ERROR("Invalid arguments for runtime restart"); + ret = -1; + goto out; + } +@@ -218,7 +218,7 @@ int runtime_clean_resource(const char *name, const char *runtime, const rt_clean + const struct rt_ops *ops = NULL; + + if (name == NULL || runtime == NULL) { +- ERROR("Invalide arguments for runtime clean"); ++ ERROR("Invalid arguments for runtime clean"); + ret = -1; + goto out; + } +@@ -242,7 +242,7 @@ int runtime_rm(const char *name, const char *runtime, const rt_rm_params_t *para + const struct rt_ops *ops = NULL; + + if (name == NULL || runtime == NULL) { +- ERROR("Invalide arguments for runtime rm"); ++ ERROR("Invalid arguments for runtime rm"); + ret = -1; + goto out; + } +@@ -267,7 +267,7 @@ int runtime_status(const char *name, const char *runtime, const rt_status_params + const struct rt_ops *ops = NULL; + + if (name == NULL || runtime == NULL || status == NULL) { +- ERROR("Invalide arguments for runtime status"); ++ ERROR("Invalid arguments for runtime status"); + ret = -1; + goto out; + } +@@ -292,7 +292,7 @@ int runtime_resources_stats(const char *name, const char *runtime, const rt_stat + const struct rt_ops *ops = NULL; + + if (name == NULL || runtime == NULL || rs_stats == NULL) { +- ERROR("Invalide arguments for runtime stats"); ++ ERROR("Invalid arguments for runtime stats"); + ret = -1; + goto out; + } +@@ -316,7 +316,7 @@ int runtime_exec(const char *name, const char *runtime, const rt_exec_params_t * + const struct rt_ops *ops = NULL; + + if (name == NULL || runtime == NULL || exit_code == NULL) { +- ERROR("Invalide arguments for runtime exec"); ++ ERROR("Invalid arguments for runtime exec"); + ret = -1; + goto out; + } +@@ -340,7 +340,7 @@ int runtime_pause(const char *name, const char *runtime, const rt_pause_params_t + const struct rt_ops *ops = NULL; + + if (name == NULL || runtime == NULL || params == NULL) { +- ERROR("Invalide arguments for runtime pause"); ++ ERROR("Invalid arguments for runtime pause"); + ret = -1; + goto out; + } +@@ -364,7 +364,7 @@ int runtime_resume(const char *name, const char *runtime, const rt_resume_params + const struct rt_ops *ops = NULL; + + if (name == NULL || runtime == NULL || params == NULL) { +- ERROR("Invalide arguments for runtime resume"); ++ ERROR("Invalid arguments for runtime resume"); + ret = -1; + goto out; + } +@@ -388,7 +388,7 @@ int runtime_attach(const char *name, const char *runtime, const rt_attach_params + const struct rt_ops *ops = NULL; + + if (name == NULL || runtime == NULL || params == NULL) { +- ERROR("Invalide arguments for runtime attach"); ++ ERROR("Invalid arguments for runtime attach"); + ret = -1; + goto out; + } +@@ -412,7 +412,7 @@ int runtime_update(const char *name, const char *runtime, const rt_update_params + const struct rt_ops *ops = NULL; + + if (name == NULL || runtime == NULL || params == NULL) { +- ERROR("Invalide arguments for runtime update"); ++ ERROR("Invalid arguments for runtime update"); + ret = -1; + goto out; + } +@@ -447,7 +447,7 @@ int runtime_listpids(const char *name, const char *runtime, const rt_listpids_pa + const struct rt_ops *ops = NULL; + + if (name == NULL || runtime == NULL || params == NULL || out == NULL) { +- ERROR("Invalide arguments for runtime listpids"); ++ ERROR("Invalid arguments for runtime listpids"); + ret = -1; + goto out; + } +@@ -471,7 +471,7 @@ int runtime_resize(const char *name, const char *runtime, const rt_resize_params + const struct rt_ops *ops = NULL; + + if (name == NULL || runtime == NULL || params == NULL) { +- ERROR("Invalide arguments for runtime resize"); ++ ERROR("Invalid arguments for runtime resize"); + ret = -1; + goto out; + } +@@ -495,7 +495,7 @@ int runtime_exec_resize(const char *name, const char *runtime, const rt_exec_res + const struct rt_ops *ops = NULL; + + if (name == NULL || runtime == NULL || params == NULL) { +- ERROR("Invalide arguments for runtime exec resize"); ++ ERROR("Invalid arguments for runtime exec resize"); + ret = -1; + goto out; + } +-- +2.25.1 + diff --git a/0003-Add-macro-for-protoc-cmake.patch b/0003-Add-macro-for-protoc-cmake.patch new file mode 100644 index 0000000..accf473 --- /dev/null +++ b/0003-Add-macro-for-protoc-cmake.patch @@ -0,0 +1,135 @@ +From 466309bc0aafe61ebed5c71012e28b9912783b60 Mon Sep 17 00:00:00 2001 +From: Xuepeng Xu <xuxuepeng1@huawei.com> +Date: Thu, 9 Feb 2023 14:32:59 +0800 +Subject: [PATCH 03/53] Add macro for protoc cmake + +Signed-off-by: Xuepeng Xu <xuxuepeng1@huawei.com> +--- + cmake/protoc.cmake | 98 +++++++++++++--------------------------------- + 1 file changed, 28 insertions(+), 70 deletions(-) + +diff --git a/cmake/protoc.cmake b/cmake/protoc.cmake +index 5c433e5c..23b8c077 100644 +--- a/cmake/protoc.cmake ++++ b/cmake/protoc.cmake +@@ -11,89 +11,47 @@ if (ENABLE_NATIVE_NETWORK) + set(NETWORK_PROTOS_OUT_PATH ${GRPC_OUT_PRE_PATH}/src/api/services/network) + endif() + ++macro(PROTOC_CPP_GEN proto_name cpp_out_path proto_path) ++ execute_process(COMMAND ${CMD_PROTOC} -I ${PROTOS_PATH}/${proto_name} --cpp_out=${cpp_out_path} ${proto_path} ERROR_VARIABLE cpp_err) ++ if (cpp_err) ++ message("Parse ${proto_path} failed: ") ++ message(FATAL_ERROR ${cpp_err}) ++ endif() ++endmacro(PROTOC_CPP_GEN) ++ ++macro(PROTOC_GRPC_GEN proto_name grpc_out_path proto_path) ++ execute_process(COMMAND ${CMD_PROTOC} -I ${PROTOS_PATH}/${proto_name} --grpc_out=${grpc_out_path} --plugin=protoc-gen-grpc=${CMD_GRPC_CPP_PLUGIN} ${proto_path} ERROR_VARIABLE grpc_err) ++ if (grpc_err) ++ message("Parse ${proto_path} failed: ") ++ message(FATAL_ERROR ${grpc_err}) ++ endif() ++endmacro(PROTOC_GRPC_GEN) ++ + if (GRPC_CONNECTOR) + execute_process(COMMAND mkdir -p ${CONTAINER_PROTOS_OUT_PATH}) + execute_process(COMMAND mkdir -p ${IMAGE_PROTOS_OUT_PATH}) + execute_process(COMMAND mkdir -p ${VOLUME_PROTOS_OUT_PATH}) + execute_process(COMMAND mkdir -p ${CRI_PROTOS_OUT_PATH}) +- execute_process(COMMAND ${CMD_PROTOC} -I ${PROTOS_PATH}/containers --cpp_out=${CONTAINER_PROTOS_OUT_PATH} +- ${PROTOS_PATH}/containers/container.proto ERROR_VARIABLE containers_err) +- if (containers_err) +- message("Parse ${PROTOS_PATH}/containers/container.proto failed: ") +- message(FATAL_ERROR ${containers_err}) +- endif() + +- execute_process(COMMAND ${CMD_PROTOC} -I ${PROTOS_PATH}/containers --grpc_out=${CONTAINER_PROTOS_OUT_PATH} --plugin=protoc-gen-grpc=${CMD_GRPC_CPP_PLUGIN} ${PROTOS_PATH}/containers/container.proto ERROR_VARIABLE containers_err) +- if (containers_err) +- message("Parse ${PROTOS_PATH}/containers/container.proto plugin failed: ") +- message(FATAL_ERROR ${containers_err}) +- endif() ++ PROTOC_CPP_GEN(containers ${CONTAINER_PROTOS_OUT_PATH} ${PROTOS_PATH}/containers/container.proto) ++ PROTOC_GRPC_GEN(containers ${CONTAINER_PROTOS_OUT_PATH} ${PROTOS_PATH}/containers/container.proto) + +- execute_process(COMMAND ${CMD_PROTOC} -I ${PROTOS_PATH}/images --cpp_out=${IMAGE_PROTOS_OUT_PATH} ${PROTOS_PATH}/images/images.proto ERROR_VARIABLE images_err) +- if (images_err) +- message("Parse ${PROTOS_PATH}/images/images.proto failed: ") +- message(FATAL_ERROR ${images_err}) +- endif() ++ PROTOC_CPP_GEN(images ${IMAGE_PROTOS_OUT_PATH} ${PROTOS_PATH}/images/images.proto) ++ PROTOC_GRPC_GEN(images ${IMAGE_PROTOS_OUT_PATH} ${PROTOS_PATH}/images/images.proto) + +- execute_process(COMMAND ${CMD_PROTOC} -I ${PROTOS_PATH}/images --grpc_out=${IMAGE_PROTOS_OUT_PATH} --plugin=protoc-gen-grpc=${CMD_GRPC_CPP_PLUGIN} ${PROTOS_PATH}/images/images.proto ERROR_VARIABLE images_err) +- if (images_err) +- message("Parse ${PROTOS_PATH}/images/images.proto plugin failed: ") +- message(FATAL_ERROR ${images_err}) +- endif() ++ PROTOC_CPP_GEN(volumes ${VOLUME_PROTOS_OUT_PATH} ${PROTOS_PATH}/volumes/volumes.proto) ++ PROTOC_GRPC_GEN(volumes ${VOLUME_PROTOS_OUT_PATH} ${PROTOS_PATH}/volumes/volumes.proto) + +- execute_process(COMMAND ${CMD_PROTOC} -I ${PROTOS_PATH}/volumes --cpp_out=${VOLUME_PROTOS_OUT_PATH} ${PROTOS_PATH}/volumes/volumes.proto ERROR_VARIABLE volumes_err) +- if (volumes_err) +- message("Parse ${PROTOS_PATH}/volumes/volumes.proto failed: ") +- message(FATAL_ERROR ${volumes_err}) +- endif() ++ PROTOC_CPP_GEN(cri ${CRI_PROTOS_OUT_PATH} ${PROTOS_PATH}/cri/api.proto) ++ PROTOC_GRPC_GEN(cri ${CRI_PROTOS_OUT_PATH} ${PROTOS_PATH}/cri/api.proto) + +- execute_process(COMMAND ${CMD_PROTOC} -I ${PROTOS_PATH}/volumes --grpc_out=${VOLUME_PROTOS_OUT_PATH} --plugin=protoc-gen-grpc=${CMD_GRPC_CPP_PLUGIN} ${PROTOS_PATH}/volumes/volumes.proto ERROR_VARIABLE volumes_err) +- if (volumes_err) +- message("Parse ${PROTOS_PATH}/volumes/volumes.proto plugin failed: ") +- message(FATAL_ERROR ${volumes_err}) +- endif() +- +- execute_process(COMMAND ${CMD_PROTOC} -I ${PROTOS_PATH}/cri --cpp_out=${CRI_PROTOS_OUT_PATH} ${PROTOS_PATH}/cri/api.proto +- ERROR_VARIABLE cri_err) +- if (cri_err) +- message("Parse ${PROTOS_PATH}/cri/api.proto failed: ") +- message(FATAL_ERROR ${cri_err}) +- endif() +- +- execute_process(COMMAND ${CMD_PROTOC} -I ${PROTOS_PATH}/cri --grpc_out=${CRI_PROTOS_OUT_PATH} +- --plugin=protoc-gen-grpc=${CMD_GRPC_CPP_PLUGIN} ${PROTOS_PATH}/cri/api.proto ERROR_VARIABLE cri_err) +- if (cri_err) +- message("Parse ${PROTOS_PATH}/cri/api.proto plugin failed: ") +- message(FATAL_ERROR ${cri_err}) +- endif() +- execute_process(COMMAND ${CMD_PROTOC} -I ${PROTOS_PATH}/cri --cpp_out=${CRI_PROTOS_OUT_PATH} ${PROTOS_PATH}/cri/gogo.proto +- ERROR_VARIABLE cri_err) +- if (cri_err) +- message("Parse ${PROTOS_PATH}/cri/gogo.proto failed: ") +- message(FATAL_ERROR ${cri_err}) +- endif() +- execute_process(COMMAND ${CMD_PROTOC} -I ${PROTOS_PATH}/cri --grpc_out=${CRI_PROTOS_OUT_PATH} +- --plugin=protoc-gen-grpc=${CMD_GRPC_CPP_PLUGIN} ${PROTOS_PATH}/cri/gogo.proto ERROR_VARIABLE cri_err) +- if (cri_err) +- message("Parse ${PROTOS_PATH}/cri/gogo.proto plugin failed: ") +- message(FATAL_ERROR ${cri_err}) +- endif() ++ PROTOC_CPP_GEN(cri ${CRI_PROTOS_OUT_PATH} ${PROTOS_PATH}/cri/gogo.proto) ++ PROTOC_GRPC_GEN(cri ${CRI_PROTOS_OUT_PATH} ${PROTOS_PATH}/cri/gogo.proto) + + if (ENABLE_NATIVE_NETWORK) + execute_process(COMMAND mkdir -p ${NETWORK_PROTOS_OUT_PATH}) +- execute_process(COMMAND ${CMD_PROTOC} -I ${PROTOS_PATH}/network +- --cpp_out=${NETWORK_PROTOS_OUT_PATH} ${PROTOS_PATH}/network/network.proto ERROR_VARIABLE network_err) +- if (network_err) +- message("Parse ${PROTOS_PATH}/network/network.proto failed: ") +- message(FATAL_ERROR ${network_err}) +- endif() +- +- execute_process(COMMAND ${CMD_PROTOC} -I ${PROTOS_PATH}/network --grpc_out=${NETWORK_PROTOS_OUT_PATH} +- --plugin=protoc-gen-grpc=${CMD_GRPC_CPP_PLUGIN} ${PROTOS_PATH}/network/network.proto ERROR_VARIABLE network_err) +- if (network_err) +- message("Parse ${PROTOS_PATH}/network/network.proto plugin failed: ") +- message(FATAL_ERROR ${network_err}) +- endif() ++ PROTOC_CPP_GEN(network ${NETWORK_PROTOS_OUT_PATH} ${PROTOS_PATH}/network/network.proto) ++ PROTOC_GRPC_GEN(network ${NETWORK_PROTOS_OUT_PATH} ${PROTOS_PATH}/network/network.proto) + endif() + endif() + +-- +2.25.1 + diff --git a/0004-fix-design-typo.patch b/0004-fix-design-typo.patch new file mode 100644 index 0000000..dafc2c7 --- /dev/null +++ b/0004-fix-design-typo.patch @@ -0,0 +1,25 @@ +From a7df50dc3b51f961f3d2e48dd968cfb115c39fec Mon Sep 17 00:00:00 2001 +From: zhushy <zhushangyuan@foxmail.com> +Date: Sat, 11 Feb 2023 00:05:53 +0800 +Subject: [PATCH 04/53] fix design typo + +--- + README.md | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/README.md b/README.md +index 3d1fc7cb..7e4b6de1 100644 +--- a/README.md ++++ b/README.md +@@ -70,7 +70,7 @@ CRI interface is implemented based on gRPC. iSulad implemented CRI gRPC Server f + + - [user manual](./docs/manual/README.md) + +-- [desgin docs](./docs/design/README.md) ++- [design docs](./docs/design/README.md) + + ### Installing + +-- +2.25.1 + diff --git a/0005-fix-cpu-rt-review-comments.patch b/0005-fix-cpu-rt-review-comments.patch new file mode 100644 index 0000000..d5f812d --- /dev/null +++ b/0005-fix-cpu-rt-review-comments.patch @@ -0,0 +1,44 @@ +From 257054b234debb7b1fcafce6f2ec3df828370aed Mon Sep 17 00:00:00 2001 +From: songbuhuang <544824346@qq.com> +Date: Sun, 12 Feb 2023 15:23:37 +0800 +Subject: [PATCH 05/53] fix cpu rt review comments + +Signed-off-by: songbuhuang <544824346@qq.com> +--- + src/daemon/executor/container_cb/execution_create.c | 10 +++++----- + 1 file changed, 5 insertions(+), 5 deletions(-) + +diff --git a/src/daemon/executor/container_cb/execution_create.c b/src/daemon/executor/container_cb/execution_create.c +index feaa3064..cc9ae716 100644 +--- a/src/daemon/executor/container_cb/execution_create.c ++++ b/src/daemon/executor/container_cb/execution_create.c +@@ -1327,7 +1327,7 @@ static int save_container_config_before_create(const char *id, const char *runti + static int maybe_create_cpu_realtime_file(int64_t value, const char *file, const char *path) + { + int ret; +- int fd = 0; ++ int fd = -1; + ssize_t nwrite; + char fpath[PATH_MAX] = { 0 }; + char buf[ISULAD_NUMSTRLEN64] = { 0 }; +@@ -1342,13 +1342,13 @@ static int maybe_create_cpu_realtime_file(int64_t value, const char *file, const + return -1; + } + +- int nret = snprintf(fpath, sizeof(fpath), "%s/%s", path, file); +- if (nret < 0 || nret >= sizeof(fpath)) { ++ ret = snprintf(fpath, sizeof(fpath), "%s/%s", path, file); ++ if (ret < 0 || ret >= sizeof(fpath)) { + ERROR("Failed to print string"); + return -1; + } +- nret = snprintf(buf, sizeof(buf), "%lld", (long long int)value); +- if (nret < 0 || (size_t)nret >= sizeof(buf)) { ++ ret = snprintf(buf, sizeof(buf), "%lld", (long long int)value); ++ if (ret < 0 || (size_t)ret >= sizeof(buf)) { + ERROR("Failed to print string"); + return -1; + } +-- +2.25.1 + diff --git a/0006-fix-inspect.sh-failed.patch b/0006-fix-inspect.sh-failed.patch new file mode 100644 index 0000000..17f5510 --- /dev/null +++ b/0006-fix-inspect.sh-failed.patch @@ -0,0 +1,31 @@ +From e4993d0e89ca853d74d8b23895de0967b4379441 Mon Sep 17 00:00:00 2001 +From: zhangxiaoyu <zhangxiaoyu58@huawei.com> +Date: Mon, 13 Feb 2023 17:42:30 +0800 +Subject: [PATCH 06/53] fix inspect.sh failed + +Signed-off-by: zhangxiaoyu <zhangxiaoyu58@huawei.com> +--- + CI/test_cases/container_cases/inspect.sh | 7 +------ + 1 file changed, 1 insertion(+), 6 deletions(-) + +diff --git a/CI/test_cases/container_cases/inspect.sh b/CI/test_cases/container_cases/inspect.sh +index 0d4ccb02..cde9ea1f 100755 +--- a/CI/test_cases/container_cases/inspect.sh ++++ b/CI/test_cases/container_cases/inspect.sh +@@ -103,12 +103,7 @@ function test_inspect_spec() + isula inspect --format='{{.Image}}' $containername 2>&1 | grep "sha256:${image_id}" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container with image: ${image}" && ((ret++)) + +- if [ -d /sys/fs/cgroup/files ];then +- grepval="100" +- else +- grepval="0" +- fi +- isula inspect --format='{{json .HostConfig.FilesLimit}}' $containername 2>&1 | grep "$grepval" ++ isula inspect --format='{{json .HostConfig.FilesLimit}}' $containername 2>&1 | grep 0 + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container with image: ${image}" && ((ret++)) + + isula inspect --format='{{json .Config.Env}}' $containername 2>&1 | grep "a=1" +-- +2.25.1 + diff --git a/0007-add-CRI-ContainerStats-Service.patch b/0007-add-CRI-ContainerStats-Service.patch new file mode 100644 index 0000000..e719086 --- /dev/null +++ b/0007-add-CRI-ContainerStats-Service.patch @@ -0,0 +1,490 @@ +From 9bd02c394110180ac7d7cbe80c1f4abe18146ebb Mon Sep 17 00:00:00 2001 +From: zhangxiaoyu <zhangxiaoyu58@huawei.com> +Date: Fri, 10 Feb 2023 17:43:11 +0800 +Subject: [PATCH 07/53] add CRI ContainerStats Service + +Signed-off-by: zhangxiaoyu <zhangxiaoyu58@huawei.com> +--- + .../connect/grpc/runtime_image_service.cc | 12 ++--- + .../connect/grpc/runtime_runtime_service.cc | 45 +++++++++++----- + .../connect/grpc/runtime_runtime_service.h | 3 ++ + .../cri/cri_container_manager_service.cc | 54 +++++++++++++++++++ + .../entry/cri/cri_container_manager_service.h | 3 ++ + src/daemon/entry/cri/cri_runtime_service.h | 3 ++ + .../entry/cri/cri_runtime_service_impl.cc | 6 +++ + .../entry/cri/cri_runtime_service_impl.h | 3 ++ + src/daemon/modules/events/collector.c | 4 +- + src/daemon/modules/image/image.c | 16 +++--- + .../modules/image/oci/oci_common_operators.c | 8 +-- + 11 files changed, 125 insertions(+), 32 deletions(-) + +diff --git a/src/daemon/entry/connect/grpc/runtime_image_service.cc b/src/daemon/entry/connect/grpc/runtime_image_service.cc +index 23447baf..e593a9c6 100644 +--- a/src/daemon/entry/connect/grpc/runtime_image_service.cc ++++ b/src/daemon/entry/connect/grpc/runtime_image_service.cc +@@ -54,7 +54,7 @@ grpc::Status RuntimeImageServiceImpl::ListImages(grpc::ServerContext *context, + std::vector<std::unique_ptr<runtime::v1alpha2::Image>> images; + Errors error; + +- WARN("Event: {Object: CRI, Type: Listing all images}"); ++ INFO("Event: {Object: CRI, Type: Listing all images}"); + + rService->ListImages(request->filter(), &images, error); + if (!error.Empty()) { +@@ -70,7 +70,7 @@ grpc::Status RuntimeImageServiceImpl::ListImages(grpc::ServerContext *context, + *image = *(iter->get()); + } + +- WARN("Event: {Object: CRI, Type: Listed all images}"); ++ INFO("Event: {Object: CRI, Type: Listed all images}"); + + return grpc::Status::OK; + } +@@ -82,7 +82,7 @@ grpc::Status RuntimeImageServiceImpl::ImageStatus(grpc::ServerContext *context, + std::unique_ptr<runtime::v1alpha2::Image> image_info = nullptr; + Errors error; + +- WARN("Event: {Object: CRI, Type: Statusing image %s}", request->image().image().c_str()); ++ INFO("Event: {Object: CRI, Type: Statusing image %s}", request->image().image().c_str()); + + image_info = rService->ImageStatus(request->image(), error); + if (!error.Empty() && !CRIHelpers::IsImageNotFoundError(error.GetMessage())) { +@@ -96,7 +96,7 @@ grpc::Status RuntimeImageServiceImpl::ImageStatus(grpc::ServerContext *context, + *image = *image_info; + } + +- WARN("Event: {Object: CRI, Type: Statused image %s}", request->image().image().c_str()); ++ INFO("Event: {Object: CRI, Type: Statused image %s}", request->image().image().c_str()); + + return grpc::Status::OK; + } +@@ -108,7 +108,7 @@ grpc::Status RuntimeImageServiceImpl::ImageFsInfo(grpc::ServerContext *context, + std::vector<std::unique_ptr<runtime::v1alpha2::FilesystemUsage>> usages; + Errors error; + +- WARN("Event: {Object: CRI, Type: Statusing image fs info}"); ++ INFO("Event: {Object: CRI, Type: Statusing image fs info}"); + + rService->ImageFsInfo(&usages, error); + if (!error.Empty()) { +@@ -125,7 +125,7 @@ grpc::Status RuntimeImageServiceImpl::ImageFsInfo(grpc::ServerContext *context, + *fs_info = *(iter->get()); + } + +- WARN("Event: {Object: CRI, Type: Statused image fs info}"); ++ INFO("Event: {Object: CRI, Type: Statused image fs info}"); + return grpc::Status::OK; + } + +diff --git a/src/daemon/entry/connect/grpc/runtime_runtime_service.cc b/src/daemon/entry/connect/grpc/runtime_runtime_service.cc +index b6f9e751..8fed162b 100644 +--- a/src/daemon/entry/connect/grpc/runtime_runtime_service.cc ++++ b/src/daemon/entry/connect/grpc/runtime_runtime_service.cc +@@ -171,7 +171,7 @@ grpc::Status RuntimeRuntimeServiceImpl::ListContainers(grpc::ServerContext *cont + { + Errors error; + +- WARN("Event: {Object: CRI, Type: Listing all Container}"); ++ INFO("Event: {Object: CRI, Type: Listing all Container}"); + + std::vector<std::unique_ptr<runtime::v1alpha2::Container>> containers; + m_rService->ListContainers(request->has_filter() ? &request->filter() : nullptr, &containers, error); +@@ -189,7 +189,28 @@ grpc::Status RuntimeRuntimeServiceImpl::ListContainers(grpc::ServerContext *cont + *container = *(iter->get()); + } + +- WARN("Event: {Object: CRI, Type: Listed all Container}"); ++ INFO("Event: {Object: CRI, Type: Listed all Container}"); ++ ++ return grpc::Status::OK; ++} ++ ++grpc::Status RuntimeRuntimeServiceImpl::ContainerStats(grpc::ServerContext *context, ++ const runtime::v1alpha2::ContainerStatsRequest *request, ++ runtime::v1alpha2::ContainerStatsResponse *reply) ++{ ++ Errors error; ++ ++ INFO("Event: {Object: CRI, Type: Getting Container Stats: %s}", request->container_id().c_str()); ++ ++ std::unique_ptr<runtime::v1alpha2::ContainerStats> contStats = ++ m_rService->ContainerStats(request->container_id(), error); ++ if (!error.Empty() || !contStats) { ++ ERROR("Object: CRI, Type: Failed to get container stats %s", request->container_id().c_str()); ++ return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ } ++ *(reply->mutable_stats()) = *contStats; ++ ++ INFO("Event: {Object: CRI, Type: Got Container stats: %s}", request->container_id().c_str()); + + return grpc::Status::OK; + } +@@ -200,7 +221,7 @@ grpc::Status RuntimeRuntimeServiceImpl::ListContainerStats(grpc::ServerContext * + { + Errors error; + +- WARN("Event: {Object: CRI, Type: Listing all Container stats}"); ++ INFO("Event: {Object: CRI, Type: Listing all Container stats}"); + + std::vector<std::unique_ptr<runtime::v1alpha2::ContainerStats>> containers; + m_rService->ListContainerStats(request->has_filter() ? &request->filter() : nullptr, &containers, error); +@@ -218,7 +239,7 @@ grpc::Status RuntimeRuntimeServiceImpl::ListContainerStats(grpc::ServerContext * + *container = *(iter->get()); + } + +- WARN("Event: {Object: CRI, Type: Listed all Container stats}"); ++ INFO("Event: {Object: CRI, Type: Listed all Container stats}"); + + return grpc::Status::OK; + } +@@ -229,7 +250,7 @@ grpc::Status RuntimeRuntimeServiceImpl::ContainerStatus(grpc::ServerContext *con + { + Errors error; + +- WARN("Event: {Object: CRI, Type: Statusing Container: %s}", request->container_id().c_str()); ++ INFO("Event: {Object: CRI, Type: Statusing Container: %s}", request->container_id().c_str()); + + std::unique_ptr<runtime::v1alpha2::ContainerStatus> contStatus = + m_rService->ContainerStatus(request->container_id(), error); +@@ -239,7 +260,7 @@ grpc::Status RuntimeRuntimeServiceImpl::ContainerStatus(grpc::ServerContext *con + } + *(reply->mutable_status()) = *contStatus; + +- WARN("Event: {Object: CRI, Type: Statused Container: %s}", request->container_id().c_str()); ++ INFO("Event: {Object: CRI, Type: Statused Container: %s}", request->container_id().c_str()); + + return grpc::Status::OK; + } +@@ -329,7 +350,7 @@ grpc::Status RuntimeRuntimeServiceImpl::PodSandboxStatus(grpc::ServerContext *co + { + Errors error; + +- WARN("Event: {Object: CRI, Type: Status Pod: %s}", request->pod_sandbox_id().c_str()); ++ INFO("Event: {Object: CRI, Type: Status Pod: %s}", request->pod_sandbox_id().c_str()); + + std::unique_ptr<runtime::v1alpha2::PodSandboxStatus> podStatus; + podStatus = m_rService->PodSandboxStatus(request->pod_sandbox_id(), error); +@@ -340,7 +361,7 @@ grpc::Status RuntimeRuntimeServiceImpl::PodSandboxStatus(grpc::ServerContext *co + } + *(reply->mutable_status()) = *podStatus; + +- WARN("Event: {Object: CRI, Type: Statused Pod: %s}", request->pod_sandbox_id().c_str()); ++ INFO("Event: {Object: CRI, Type: Statused Pod: %s}", request->pod_sandbox_id().c_str()); + + return grpc::Status::OK; + } +@@ -351,7 +372,7 @@ grpc::Status RuntimeRuntimeServiceImpl::ListPodSandbox(grpc::ServerContext *cont + { + Errors error; + +- WARN("Event: {Object: CRI, Type: Listing all Pods}"); ++ INFO("Event: {Object: CRI, Type: Listing all Pods}"); + + std::vector<std::unique_ptr<runtime::v1alpha2::PodSandbox>> pods; + m_rService->ListPodSandbox(request->has_filter() ? &request->filter() : nullptr, &pods, error); +@@ -368,7 +389,7 @@ grpc::Status RuntimeRuntimeServiceImpl::ListPodSandbox(grpc::ServerContext *cont + *pod = *(iter->get()); + } + +- WARN("Event: {Object: CRI, Type: Listed all Pods}"); ++ INFO("Event: {Object: CRI, Type: Listed all Pods}"); + + return grpc::Status::OK; + } +@@ -460,7 +481,7 @@ grpc::Status RuntimeRuntimeServiceImpl::Status(grpc::ServerContext *context, + { + Errors error; + +- WARN("Event: {Object: CRI, Type: Statusing daemon}"); ++ INFO("Event: {Object: CRI, Type: Statusing daemon}"); + + std::unique_ptr<runtime::v1alpha2::RuntimeStatus> status = m_rService->Status(error); + if (status == nullptr || error.NotEmpty()) { +@@ -469,7 +490,7 @@ grpc::Status RuntimeRuntimeServiceImpl::Status(grpc::ServerContext *context, + } + *(reply->mutable_status()) = *status; + +- WARN("Event: {Object: CRI, Type: Statused daemon}"); ++ INFO("Event: {Object: CRI, Type: Statused daemon}"); + + return grpc::Status::OK; + } +diff --git a/src/daemon/entry/connect/grpc/runtime_runtime_service.h b/src/daemon/entry/connect/grpc/runtime_runtime_service.h +index 6e8c1009..cb3c5425 100644 +--- a/src/daemon/entry/connect/grpc/runtime_runtime_service.h ++++ b/src/daemon/entry/connect/grpc/runtime_runtime_service.h +@@ -52,6 +52,9 @@ public: + const runtime::v1alpha2::ListContainerStatsRequest *request, + runtime::v1alpha2::ListContainerStatsResponse *reply) override; + ++ grpc::Status ContainerStats(grpc::ServerContext *context, const runtime::v1alpha2::ContainerStatsRequest *request, ++ runtime::v1alpha2::ContainerStatsResponse *reply) override; ++ + grpc::Status ContainerStatus(grpc::ServerContext *context, const runtime::v1alpha2::ContainerStatusRequest *request, + runtime::v1alpha2::ContainerStatusResponse *reply) override; + +diff --git a/src/daemon/entry/cri/cri_container_manager_service.cc b/src/daemon/entry/cri/cri_container_manager_service.cc +index d044cca8..710556a3 100644 +--- a/src/daemon/entry/cri/cri_container_manager_service.cc ++++ b/src/daemon/entry/cri/cri_container_manager_service.cc +@@ -893,6 +893,60 @@ cleanup: + free_container_stats_response(response); + } + ++auto ContainerManagerService::ContainerStats(const std::string &containerID, Errors &error) ++-> std::unique_ptr<runtime::v1alpha2::ContainerStats> ++{ ++ container_stats_request *request { nullptr }; ++ container_stats_response *response { nullptr }; ++ std::unique_ptr<runtime::v1alpha2::ContainerStats> contStats { nullptr }; ++ std::vector<std::unique_ptr<runtime::v1alpha2::ContainerStats>> contStatsVec; ++ ++ if (containerID.empty()) { ++ error.SetError("Empty container id"); ++ return nullptr; ++ } ++ ++ if (m_cb == nullptr || m_cb->container.stats == nullptr) { ++ error.SetError("Unimplemented callback"); ++ return nullptr; ++ } ++ ++ request = (container_stats_request *)util_common_calloc_s(sizeof(container_stats_request)); ++ if (request == nullptr) { ++ error.SetError("Out of memory"); ++ return nullptr; ++ } ++ ++ request->containers = (char **)util_smart_calloc_s(sizeof(char *), 1); ++ if (request->containers == nullptr) { ++ error.SetError("Out of memory"); ++ goto cleanup; ++ } ++ ++ request->containers[0] = util_strdup_s(containerID.c_str()); ++ request->containers_len = 1; ++ ++ if (m_cb->container.stats(request, &response) != 0) { ++ if (response != nullptr && response->errmsg != nullptr) { ++ error.SetError(response->errmsg); ++ } else { ++ error.SetError("Failed to call stats container callback"); ++ } ++ goto cleanup; ++ } ++ ++ ContainerStatsToGRPC(response, &contStatsVec, error); ++ if (error.NotEmpty()) { ++ goto cleanup; ++ } ++ contStats = std::move(contStatsVec[0]); ++ ++cleanup: ++ free_container_stats_request(request); ++ free_container_stats_response(response); ++ return contStats; ++} ++ + void ContainerManagerService::PackContainerImageToStatus( + container_inspect *inspect, std::unique_ptr<runtime::v1alpha2::ContainerStatus> &contStatus, Errors &error) + { +diff --git a/src/daemon/entry/cri/cri_container_manager_service.h b/src/daemon/entry/cri/cri_container_manager_service.h +index 8002b77d..6ec1f21d 100644 +--- a/src/daemon/entry/cri/cri_container_manager_service.h ++++ b/src/daemon/entry/cri/cri_container_manager_service.h +@@ -53,6 +53,9 @@ public: + std::vector<std::unique_ptr<runtime::v1alpha2::ContainerStats>> *containerstats, + Errors &error); + ++ auto ContainerStats(const std::string &containerID, Errors &error) ++ -> std::unique_ptr<runtime::v1alpha2::ContainerStats>; ++ + auto ContainerStatus(const std::string &containerID, Errors &error) + -> std::unique_ptr<runtime::v1alpha2::ContainerStatus>; + +diff --git a/src/daemon/entry/cri/cri_runtime_service.h b/src/daemon/entry/cri/cri_runtime_service.h +index 4727230f..5e4740cb 100644 +--- a/src/daemon/entry/cri/cri_runtime_service.h ++++ b/src/daemon/entry/cri/cri_runtime_service.h +@@ -49,6 +49,9 @@ public: + std::vector<std::unique_ptr<runtime::v1alpha2::ContainerStats>> *containerstats, + Errors &error) = 0; + ++ virtual auto ContainerStats(const std::string &containerID, ++ Errors &error) -> std::unique_ptr<runtime::v1alpha2::ContainerStats> = 0; ++ + virtual auto ContainerStatus(const std::string &containerID, + Errors &error) -> std::unique_ptr<runtime::v1alpha2::ContainerStatus> = 0; + +diff --git a/src/daemon/entry/cri/cri_runtime_service_impl.cc b/src/daemon/entry/cri/cri_runtime_service_impl.cc +index 241d4316..c4b84828 100644 +--- a/src/daemon/entry/cri/cri_runtime_service_impl.cc ++++ b/src/daemon/entry/cri/cri_runtime_service_impl.cc +@@ -70,6 +70,12 @@ void CRIRuntimeServiceImpl::ListContainerStats( + m_containerManager->ListContainerStats(filter, containerstats, error); + } + ++auto CRIRuntimeServiceImpl::ContainerStats(const std::string &containerID, Errors &error) ++-> std::unique_ptr<runtime::v1alpha2::ContainerStats> ++{ ++ return m_containerManager->ContainerStats(containerID, error); ++} ++ + auto CRIRuntimeServiceImpl::ContainerStatus(const std::string &containerID, Errors &error) + -> std::unique_ptr<runtime::v1alpha2::ContainerStatus> + { +diff --git a/src/daemon/entry/cri/cri_runtime_service_impl.h b/src/daemon/entry/cri/cri_runtime_service_impl.h +index f2e25e42..7355c5cb 100644 +--- a/src/daemon/entry/cri/cri_runtime_service_impl.h ++++ b/src/daemon/entry/cri/cri_runtime_service_impl.h +@@ -51,6 +51,9 @@ public: + std::vector<std::unique_ptr<runtime::v1alpha2::ContainerStats>> *containerstats, + Errors &error) override; + ++ auto ContainerStats(const std::string &containerID, Errors &error) ++ -> std::unique_ptr<runtime::v1alpha2::ContainerStats> override; ++ + auto ContainerStatus(const std::string &containerID, Errors &error) + -> std::unique_ptr<runtime::v1alpha2::ContainerStatus> override; + +diff --git a/src/daemon/modules/events/collector.c b/src/daemon/modules/events/collector.c +index fb4e4a98..f53ad319 100644 +--- a/src/daemon/modules/events/collector.c ++++ b/src/daemon/modules/events/collector.c +@@ -484,7 +484,7 @@ static int write_events_log(const struct isulad_events_format *events) + + len = calculate_annaotation_info_len(events); + if (len == 1) { +- WARN("Event: {Object: %s, Type: %s}", events->id, events->opt); ++ INFO("Event: {Object: %s, Type: %s}", events->id, events->opt); + } else { + annotation = (char *)util_common_calloc_s(len); + if (annotation == NULL) { +@@ -502,7 +502,7 @@ static int write_events_log(const struct isulad_events_format *events) + } + (void)strcat(annotation, ")"); + +- WARN("Event: {Object: %s, Type: %s %s}", events->id, events->opt, annotation); ++ INFO("Event: {Object: %s, Type: %s %s}", events->id, events->opt, annotation); + } + + out: +diff --git a/src/daemon/modules/image/image.c b/src/daemon/modules/image/image.c +index d5fbc8dc..a9b9523e 100644 +--- a/src/daemon/modules/image/image.c ++++ b/src/daemon/modules/image/image.c +@@ -416,7 +416,7 @@ int im_get_filesystem_info(const char *image_type, im_fs_info_response **respons + goto out; + } + +- WARN("Event: {Object: get image filesystem info, Type: inspecting}"); ++ INFO("Event: {Object: get image filesystem info, Type: inspecting}"); + ret = q->ops->get_filesystem_info(response); + if (ret != 0) { + if (response != NULL && *response != NULL) { +@@ -426,7 +426,7 @@ int im_get_filesystem_info(const char *image_type, im_fs_info_response **respons + } + goto out; + } +- WARN("Event: {Object: get image filesystem info, Type: inspected}"); ++ INFO("Event: {Object: get image filesystem info, Type: inspected}"); + + out: + return ret; +@@ -466,7 +466,7 @@ int im_get_container_filesystem_usage(const char *image_type, const char *id, im + request->name_id = util_strdup_s(id); + } + +- WARN("Event: {Object: container \'%s\' filesystem info, Type: inspecting}", id != NULL ? id : ""); ++ INFO("Event: {Object: container \'%s\' filesystem info, Type: inspecting}", id != NULL ? id : ""); + ret = q->ops->container_fs_usage(request, &filesystemusage); + if (ret != 0) { + ERROR("Failed to get filesystem usage for container %s", id); +@@ -476,7 +476,7 @@ int im_get_container_filesystem_usage(const char *image_type, const char *id, im + + *fs_usage = filesystemusage; + filesystemusage = NULL; +- WARN("Event: {Object: container \'%s\' filesystem info, Type: inspected}", id != NULL ? id : ""); ++ INFO("Event: {Object: container \'%s\' filesystem info, Type: inspected}", id != NULL ? id : ""); + + out: + free_im_container_fs_usage_request(request); +@@ -916,7 +916,7 @@ int im_list_images(const im_list_request *ctx, im_list_response **response) + return -1; + } + +- WARN("Event: {Object: list images, Type: listing}"); ++ INFO("Event: {Object: list images, Type: listing}"); + + for (i = 0; i < g_numbims; i++) { + if (g_bims[i].ops->list_ims == NULL) { +@@ -936,7 +936,7 @@ int im_list_images(const im_list_request *ctx, im_list_response **response) + images_tmp = NULL; + } + +- WARN("Event: {Object: list images, Type: listed}"); ++ INFO("Event: {Object: list images, Type: listed}"); + + if (g_isulad_errmsg != NULL) { + (*response)->errmsg = util_strdup_s(g_isulad_errmsg); +@@ -1671,7 +1671,7 @@ int im_inspect_image(const im_inspect_request *request, im_inspect_response **re + + image_ref = util_strdup_s(request->image.image); + +- WARN("Event: {Object: %s, Type: image inspecting}", image_ref); ++ INFO("Event: {Object: %s, Type: image inspecting}", image_ref); + + bim_type = bim_query(image_ref); + if (bim_type == NULL) { +@@ -1693,7 +1693,7 @@ int im_inspect_image(const im_inspect_request *request, im_inspect_response **re + goto pack_response; + } + +- WARN("Event: {Object: %s, Type: image inspected}", image_ref); ++ INFO("Event: {Object: %s, Type: image inspected}", image_ref); + + pack_response: + if (g_isulad_errmsg != NULL) { +diff --git a/src/daemon/modules/image/oci/oci_common_operators.c b/src/daemon/modules/image/oci/oci_common_operators.c +index 3d200e09..83cccbe6 100644 +--- a/src/daemon/modules/image/oci/oci_common_operators.c ++++ b/src/daemon/modules/image/oci/oci_common_operators.c +@@ -429,7 +429,7 @@ int oci_summary_image(im_summary_request *request, im_summary_response *response + goto pack_response; + } + +- WARN("Event: {Object: %s, Type: statusing image summary}", resolved_name); ++ INFO("Event: {Object: %s, Type: statusing image summary}", resolved_name); + + image_summary = storage_img_get_summary(resolved_name); + if (image_summary == NULL) { +@@ -442,7 +442,7 @@ int oci_summary_image(im_summary_request *request, im_summary_response *response + response->image_summary = image_summary; + image_summary = NULL; + +- WARN("Event: {Object: %s, Type: statused image summary}", resolved_name); ++ INFO("Event: {Object: %s, Type: statused image summary}", resolved_name); + + pack_response: + free(resolved_name); +@@ -493,7 +493,7 @@ int oci_status_image(im_status_request *request, im_status_response *response) + goto pack_response; + } + +- WARN("Event: {Object: %s, Type: statusing image}", resolved_name); ++ INFO("Event: {Object: %s, Type: statusing image}", resolved_name); + + image_info = storage_img_get(resolved_name); + if (image_info == NULL) { +@@ -506,7 +506,7 @@ int oci_status_image(im_status_request *request, im_status_response *response) + response->image_info->image = image_info; + image_info = NULL; + +- WARN("Event: {Object: %s, Type: statused image}", resolved_name); ++ INFO("Event: {Object: %s, Type: statused image}", resolved_name); + + pack_response: + free(resolved_name); +-- +2.25.1 + diff --git a/0008-fix-isula-cpu-rt-CI.patch b/0008-fix-isula-cpu-rt-CI.patch new file mode 100644 index 0000000..3f2566f --- /dev/null +++ b/0008-fix-isula-cpu-rt-CI.patch @@ -0,0 +1,183 @@ +From edef459d5052dc6d7c29e8a7a48ff4bf1b01bd78 Mon Sep 17 00:00:00 2001 +From: songbuhuang <544824346@qq.com> +Date: Tue, 14 Feb 2023 14:08:01 +0800 +Subject: [PATCH 08/53] fix isula cpu-rt CI + +Signed-off-by: songbuhuang <544824346@qq.com> +--- + CI/test_cases/container_cases/cpu_rt.sh | 102 +++++++++++++++++------- + 1 file changed, 73 insertions(+), 29 deletions(-) + +diff --git a/CI/test_cases/container_cases/cpu_rt.sh b/CI/test_cases/container_cases/cpu_rt.sh +index 3d70c840..353c2d71 100755 +--- a/CI/test_cases/container_cases/cpu_rt.sh ++++ b/CI/test_cases/container_cases/cpu_rt.sh +@@ -21,12 +21,12 @@ + declare -r curr_path=$(dirname $(readlink -f "$0")) + source ../helpers.sh + +-function test_cpu_rt_isulad_spec() ++function test_cpurt_isulad_abnormal() + { + local ret=0 + local test="isulad cpu realtime test => (${FUNCNAME[@]})" + +- msg_info "${test} starting..." ++ msg_info "${test} starting..." + + isulad --cpu-rt-period xx --cpu-rt-runtime 950000 /bin/sh 2>&1 | grep 'Invalid value "xx" for flag --cpu-rt-period: Invalid argument' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Invalid argument for cpu-rt-period" && ((ret++)) +@@ -38,54 +38,64 @@ function test_cpu_rt_isulad_spec() + return ${ret} + } + +-function test_cpu_rt_isula_spec() ++function test_isula_update_normal() + { +- local ret=0 +- local image="busybox" +- local test="container cpu realtime test => (${FUNCNAME[@]})" +- +- msg_info "${test} starting..." +- +- #start isulad without cpu_rt +- start_isulad_without_valgrind ++ #start isulad with cpu_rt ++ isulad --cpu-rt-period 1000000 --cpu-rt-runtime 950000 -l DEBUG > /dev/null 2>&1 & ++ wait_isulad_running ++ ++ c_id=`isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 950000 ${image} sh` ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++)) + +- isula pull ${image} +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to pull image: ${image}" && return ${FAILURE} ++ isula update --cpu-rt-period 900000 --cpu-rt-runtime 800000 $c_id ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to update container cpu-rt-runtime" && ((ret++)) + +- isula images | grep busybox +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${image}" && ((ret++)) ++ isula exec -it $c_id sh -c "cat /sys/fs/cgroup/cpu/cpu.rt_runtime_us" | grep "800000" ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container cpu.rt_runtime_us: 800000" && ((ret++)) + +- test_isula_run_spec ++ isula exec -it $c_id sh -c "cat /sys/fs/cgroup/cpu/cpu.rt_period_us" | grep "900000" ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container cpu.rt_period_us: 900000" && ((ret++)) + +- #start isulad without cpu_rt:isulad cpu.rt_period_us default value is the cpu.rt_period_us of the upper-layer directory,cpu.rt_runtime_us is 0. +- isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 10000 $image /bin/sh 2>&1 | grep "failed to write 10000" | grep "cpu.rt_runtime_us: Invalid argument" +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Invalid argument for cpu-rt-runtime" && ((ret++)) ++ isula rm -f $c_id ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to rm container ${c_id}" && ((ret++)) + + stop_isulad_without_valgrind ++ #set cpu-rt to the initial state ++ isulad --cpu-rt-period 1000000 --cpu-rt-runtime 0 -l DEBUG > /dev/null 2>&1 & ++ wait_isulad_running + ++ msg_info "${test} finished with return ${ret}..." ++ return ${ret} ++} ++ ++function test_isula_update_abnormal() ++{ + #start isulad with cpu_rt + isulad --cpu-rt-period 1000000 --cpu-rt-runtime 950000 -l DEBUG > /dev/null 2>&1 & + wait_isulad_running +- +- test_isula_run_spec + + c_id=`isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 950000 ${image} sh` + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++)) + +- isula update --cpu-rt-runtime 90000 $c_id ++ isula update --cpu-rt-period 800000 --cpu-rt-runtime 900000 $c_id | grep "Invalid --cpu-rt-runtime: rt runtime cannot be higher than rt period" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to update container cpu-rt-runtime" && ((ret++)) + +- isula exec -it $c_id sh -c "cat /sys/fs/cgroup/cpu/cpu.rt_runtime_us" | grep "90000" +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container cpu.rt_runtime_us: 90000" && ((ret++)) ++ isula update --cpu-rt-runtime 1000000 $c_id | grep "updating cgroup cpu.rt_runtime_us to 1000000: Invalid argument" ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to update container cpu-rt-runtime" && ((ret++)) + + isula rm -f $c_id + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to rm container ${c_id}" && ((ret++)) + ++ stop_isulad_without_valgrind ++ #set cpu-rt to the initial state ++ isulad --cpu-rt-period 1000000 --cpu-rt-runtime 0 -l DEBUG > /dev/null 2>&1 & ++ wait_isulad_running ++ + msg_info "${test} finished with return ${ret}..." + return ${ret} + } + +-function test_kernel_without_cpu_rt_spec() ++function test_kernel_without_cpurt() + { + local ret=0 + local image="busybox" +@@ -109,8 +119,24 @@ function test_kernel_without_cpu_rt_spec() + return ${ret} + } + +-function test_isula_run_spec() ++function test_isula_run_abnormal() + { ++ local ret=0 ++ local image="busybox" ++ local test="container cpu realtime test => (${FUNCNAME[@]})" ++ ++ msg_info "${test} starting..." ++ ++ #start isulad without cpu_rt ++ isulad --cpu-rt-period 1000000 --cpu-rt-runtime 950000 -l DEBUG > /dev/null 2>&1 & ++ wait_isulad_running ++ ++ isula pull ${image} ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to pull image: ${image}" && return ${FAILURE} ++ ++ isula images | grep busybox ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${image}" && ((ret++)) ++ + isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime -1 $image /bin/sh 2>&1 | grep "failed to write -1" | grep "cpu.rt_runtime_us: Invalid argument" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Invalid argument for cpu-rt-runtime" && ((ret++)) + +@@ -128,15 +154,33 @@ function test_isula_run_spec() + + isula run -itd --cpu-rt-period 100 --cpu-rt-runtime 10000 $image /bin/sh 2>&1 | grep "Invalid --cpu-rt-runtime: rt runtime cannot be higher than rt period" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cpu-rt-runtime cannot be higher than cpu-rt-period" && ((ret++)) ++ ++ isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 960000 $image /bin/sh 2>&1 | grep "failed to write 960000" | grep "cpu.rt_runtime_us: Invalid argument" ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Invalid argument for cpu-rt-runtime" && ((ret++)) ++ ++ stop_isulad_without_valgrind ++} ++ ++function test_isula_run_normal() ++{ ++ isula run -itd -n box --cpu-rt-period 1000000 --cpu-rt-runtime 900000 $image /bin/sh 2>&1 ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container" && ((ret++)) ++ ++ isula rm -f box ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to rm container ${c_id}" && ((ret++)) ++ + } + + declare -i ans=0 + + if [ -f "/sys/fs/cgroup/cpu/cpu.rt_runtime_us" ];then +- test_cpu_rt_isulad_spec || ((ans++)) +- test_cpu_rt_isula_spec || ((ans++)) ++ test_isula_run_abnormal || ((ans++)) ++ test_isula_run_normal || ((ans++)) ++ test_cpurt_isulad_abnormal || ((ans++)) ++ test_isula_update_normal || ((ans++)) ++ test_isula_update_abnormal || ((ans++)) + else +- test_kernel_without_cpu_rt_spec || ((ans++)) ++ test_kernel_without_cpurt || ((ans++)) + fi + + show_result ${ans} "${curr_path}/${0}" +-- +2.25.1 + diff --git a/0009-fix-cpu-rt-CI.patch b/0009-fix-cpu-rt-CI.patch new file mode 100644 index 0000000..8e0fa06 --- /dev/null +++ b/0009-fix-cpu-rt-CI.patch @@ -0,0 +1,105 @@ +From 0504a907def3efb4c0ad7eabd5921c97090430af Mon Sep 17 00:00:00 2001 +From: songbuhuang <544824346@qq.com> +Date: Tue, 14 Feb 2023 15:55:56 +0800 +Subject: [PATCH 09/53] fix cpu-rt CI + +Signed-off-by: songbuhuang <544824346@qq.com> +--- + CI/test_cases/container_cases/cpu_rt.sh | 32 +++++++++++++++++++------ + 1 file changed, 25 insertions(+), 7 deletions(-) + +diff --git a/CI/test_cases/container_cases/cpu_rt.sh b/CI/test_cases/container_cases/cpu_rt.sh +index 353c2d71..42006bc8 100755 +--- a/CI/test_cases/container_cases/cpu_rt.sh ++++ b/CI/test_cases/container_cases/cpu_rt.sh +@@ -24,14 +24,14 @@ source ../helpers.sh + function test_cpurt_isulad_abnormal() + { + local ret=0 +- local test="isulad cpu realtime test => (${FUNCNAME[@]})" ++ local test="isulad cpu realtime abnormal test => (${FUNCNAME[@]})" + + msg_info "${test} starting..." + +- isulad --cpu-rt-period xx --cpu-rt-runtime 950000 /bin/sh 2>&1 | grep 'Invalid value "xx" for flag --cpu-rt-period: Invalid argument' ++ isulad --cpu-rt-period xx --cpu-rt-runtime 950000 2>&1 | grep 'Invalid value "xx" for flag --cpu-rt-period: Invalid argument' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Invalid argument for cpu-rt-period" && ((ret++)) + +- isulad --cpu-rt-period 1000000 --cpu-rt-runtime xx /bin/sh 2>&1 | grep 'Invalid value "xx" for flag --cpu-rt-runtime: Invalid argument' ++ isulad --cpu-rt-period 1000000 --cpu-rt-runtime xx 2>&1 | grep 'Invalid value "xx" for flag --cpu-rt-runtime: Invalid argument' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Invalid argument for cpu-rt-runtime" && ((ret++)) + + msg_info "${test} finished with return ${ret}..." +@@ -40,6 +40,12 @@ function test_cpurt_isulad_abnormal() + + function test_isula_update_normal() + { ++ local ret=0 ++ local image="busybox" ++ local test="isulad update cpu realtime normal test => (${FUNCNAME[@]})" ++ ++ msg_info "${test} starting..." ++ + #start isulad with cpu_rt + isulad --cpu-rt-period 1000000 --cpu-rt-runtime 950000 -l DEBUG > /dev/null 2>&1 & + wait_isulad_running +@@ -70,6 +76,10 @@ function test_isula_update_normal() + + function test_isula_update_abnormal() + { ++ local ret=0 ++ local image="busybox" ++ local test="isulad update cpu realtime abnormal test => (${FUNCNAME[@]})" ++ + #start isulad with cpu_rt + isulad --cpu-rt-period 1000000 --cpu-rt-runtime 950000 -l DEBUG > /dev/null 2>&1 & + wait_isulad_running +@@ -77,10 +87,10 @@ function test_isula_update_abnormal() + c_id=`isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 950000 ${image} sh` + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++)) + +- isula update --cpu-rt-period 800000 --cpu-rt-runtime 900000 $c_id | grep "Invalid --cpu-rt-runtime: rt runtime cannot be higher than rt period" ++ isula update --cpu-rt-period 800000 --cpu-rt-runtime 900000 $c_id 2>&1 | grep "Invalid --cpu-rt-runtime: rt runtime cannot be higher than rt period" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to update container cpu-rt-runtime" && ((ret++)) + +- isula update --cpu-rt-runtime 1000000 $c_id | grep "updating cgroup cpu.rt_runtime_us to 1000000: Invalid argument" ++ isula update --cpu-rt-runtime 1000000 $c_id 2>&1 | grep "updating cgroup cpu.rt_runtime_us to 1000000: Invalid argument" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to update container cpu-rt-runtime" && ((ret++)) + + isula rm -f $c_id +@@ -158,17 +168,23 @@ function test_isula_run_abnormal() + isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 960000 $image /bin/sh 2>&1 | grep "failed to write 960000" | grep "cpu.rt_runtime_us: Invalid argument" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Invalid argument for cpu-rt-runtime" && ((ret++)) + +- stop_isulad_without_valgrind ++ msg_info "${test} finished with return ${ret}..." ++ return ${ret} + } + + function test_isula_run_normal() + { ++ local ret=0 ++ local image="busybox" ++ + isula run -itd -n box --cpu-rt-period 1000000 --cpu-rt-runtime 900000 $image /bin/sh 2>&1 + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container" && ((ret++)) + + isula rm -f box + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to rm container ${c_id}" && ((ret++)) +- ++ ++ msg_info "${test} finished with return ${ret}..." ++ return ${ret} + } + + declare -i ans=0 +@@ -183,4 +199,6 @@ else + test_kernel_without_cpurt || ((ans++)) + fi + ++isula rm -f $(isula ps -aq) ++ + show_result ${ans} "${curr_path}/${0}" +-- +2.25.1 + diff --git a/0010-fix-cpu-rt-CI.patch b/0010-fix-cpu-rt-CI.patch new file mode 100644 index 0000000..84d7604 --- /dev/null +++ b/0010-fix-cpu-rt-CI.patch @@ -0,0 +1,54 @@ +From 21dc648ef93cd0fb858a408bc843d25a5e20e320 Mon Sep 17 00:00:00 2001 +From: songbuhuang <544824346@qq.com> +Date: Wed, 15 Feb 2023 16:09:38 +0800 +Subject: [PATCH 10/53] fix cpu-rt CI + +Signed-off-by: songbuhuang <544824346@qq.com> +--- + CI/test_cases/container_cases/cpu_rt.sh | 12 ++++++------ + 1 file changed, 6 insertions(+), 6 deletions(-) + +diff --git a/CI/test_cases/container_cases/cpu_rt.sh b/CI/test_cases/container_cases/cpu_rt.sh +index 42006bc8..39c0b427 100755 +--- a/CI/test_cases/container_cases/cpu_rt.sh ++++ b/CI/test_cases/container_cases/cpu_rt.sh +@@ -50,14 +50,14 @@ function test_isula_update_normal() + isulad --cpu-rt-period 1000000 --cpu-rt-runtime 950000 -l DEBUG > /dev/null 2>&1 & + wait_isulad_running + +- c_id=`isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 950000 ${image} sh` ++ c_id=`isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 1000 ${image} sh` + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++)) + +- isula update --cpu-rt-period 900000 --cpu-rt-runtime 800000 $c_id ++ isula update --cpu-rt-period 900000 --cpu-rt-runtime 2000 $c_id + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to update container cpu-rt-runtime" && ((ret++)) + +- isula exec -it $c_id sh -c "cat /sys/fs/cgroup/cpu/cpu.rt_runtime_us" | grep "800000" +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container cpu.rt_runtime_us: 800000" && ((ret++)) ++ isula exec -it $c_id sh -c "cat /sys/fs/cgroup/cpu/cpu.rt_runtime_us" | grep "2000" ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container cpu.rt_runtime_us: 2000" && ((ret++)) + + isula exec -it $c_id sh -c "cat /sys/fs/cgroup/cpu/cpu.rt_period_us" | grep "900000" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container cpu.rt_period_us: 900000" && ((ret++)) +@@ -84,7 +84,7 @@ function test_isula_update_abnormal() + isulad --cpu-rt-period 1000000 --cpu-rt-runtime 950000 -l DEBUG > /dev/null 2>&1 & + wait_isulad_running + +- c_id=`isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 950000 ${image} sh` ++ c_id=`isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 1000 ${image} sh` + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++)) + + isula update --cpu-rt-period 800000 --cpu-rt-runtime 900000 $c_id 2>&1 | grep "Invalid --cpu-rt-runtime: rt runtime cannot be higher than rt period" +@@ -177,7 +177,7 @@ function test_isula_run_normal() + local ret=0 + local image="busybox" + +- isula run -itd -n box --cpu-rt-period 1000000 --cpu-rt-runtime 900000 $image /bin/sh 2>&1 ++ isula run -itd -n box --cpu-rt-period 1000000 --cpu-rt-runtime 1000 $image /bin/sh 2>&1 + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container" && ((ret++)) + + isula rm -f box +-- +2.25.1 + diff --git a/0011-Bugfix-in-config-and-executor.patch b/0011-Bugfix-in-config-and-executor.patch new file mode 100644 index 0000000..e038afd --- /dev/null +++ b/0011-Bugfix-in-config-and-executor.patch @@ -0,0 +1,52 @@ +From 933eceb4545a28dba44c72f183dc7104d0fea714 Mon Sep 17 00:00:00 2001 +From: Xuepeng Xu <xuxuepeng1@huawei.com> +Date: Wed, 15 Feb 2023 12:19:40 +0800 +Subject: [PATCH 11/53] Bugfix in config and executor + +Signed-off-by: Xuepeng Xu <xuxuepeng1@huawei.com> +--- + src/daemon/config/isulad_config.c | 2 +- + src/daemon/executor/container_cb/execution_create.c | 6 +++--- + 2 files changed, 4 insertions(+), 4 deletions(-) + +diff --git a/src/daemon/config/isulad_config.c b/src/daemon/config/isulad_config.c +index 917e3eaa..38bf4bf9 100644 +--- a/src/daemon/config/isulad_config.c ++++ b/src/daemon/config/isulad_config.c +@@ -314,7 +314,7 @@ char *conf_get_routine_rootdir(const char *runtime) + } + + /* path = conf->rootpath + / + engines + / + runtime + /0 */ +- if (strlen(conf->json_confs->graph) > (SIZE_MAX - strlen(ENGINE_ROOTPATH_NAME)) - 3) { ++ if (strlen(conf->json_confs->graph) > (SIZE_MAX - strlen(ENGINE_ROOTPATH_NAME) - strlen(runtime)) - 3) { + ERROR("Graph path is too long"); + goto out; + } +diff --git a/src/daemon/executor/container_cb/execution_create.c b/src/daemon/executor/container_cb/execution_create.c +index cc9ae716..4cc333fd 100644 +--- a/src/daemon/executor/container_cb/execution_create.c ++++ b/src/daemon/executor/container_cb/execution_create.c +@@ -533,7 +533,7 @@ static char *try_generate_id() + int i = 0; + int max_time = 10; + char *id = NULL; +- char *value = NULL; ++ container_t *value = NULL; + + id = util_smart_calloc_s(sizeof(char), (CONTAINER_ID_MAX_LEN + 1)); + if (id == NULL) { +@@ -547,9 +547,9 @@ static char *try_generate_id() + goto err_out; + } + +- value = container_name_index_get(id); ++ value = containers_store_get(id); + if (value != NULL) { +- free(value); ++ container_unref(value); + value = NULL; + continue; + } else { +-- +2.25.1 + diff --git a/0012-fix-cpu-rt-disable-after-reboot-machine.patch b/0012-fix-cpu-rt-disable-after-reboot-machine.patch new file mode 100644 index 0000000..5ae07d1 --- /dev/null +++ b/0012-fix-cpu-rt-disable-after-reboot-machine.patch @@ -0,0 +1,823 @@ +From 166edf2093b2c35fe4e479ca4b6568be8c98f907 Mon Sep 17 00:00:00 2001 +From: haozi007 <liuhao27@huawei.com> +Date: Wed, 15 Feb 2023 17:47:12 +0800 +Subject: [PATCH 12/53] fix cpu-rt disable after reboot machine + +1. ensure parent cgroup cpu-rt of container, should do in start container; +2. current do in create container, will cause failed of start container with cpu-rt after reboot machine + +Signed-off-by: haozi007 <liuhao27@huawei.com> +--- + src/daemon/common/sysinfo.c | 46 +++- + src/daemon/common/sysinfo.h | 2 + + src/daemon/executor/container_cb/execution.c | 158 ++++++++++++- + .../executor/container_cb/execution_create.c | 222 +----------------- + src/daemon/modules/api/specs_api.h | 2 +- + src/daemon/modules/spec/specs.c | 84 ++++--- + test/specs/specs/specs_ut.cc | 40 ++-- + 7 files changed, 280 insertions(+), 274 deletions(-) + +diff --git a/src/daemon/common/sysinfo.c b/src/daemon/common/sysinfo.c +index 38416db4..7559d653 100644 +--- a/src/daemon/common/sysinfo.c ++++ b/src/daemon/common/sysinfo.c +@@ -24,8 +24,10 @@ + #include <linux/magic.h> + #include <sys/stat.h> + ++#include <isula_libutils/auto_cleanup.h> ++#include <isula_libutils/log.h> ++ + #include "err_msg.h" +-#include "isula_libutils/log.h" + #include "utils.h" + #include "utils_array.h" + #include "utils_file.h" +@@ -1627,3 +1629,45 @@ free_out: + } + return minfos; + } ++ ++char *sysinfo_cgroup_controller_cpurt_mnt_path() ++{ ++ int nret = 0; ++ __isula_auto_free char *mnt = NULL; ++ __isula_auto_free char *root = NULL; ++ char fpath[PATH_MAX] = { 0 }; ++ sysinfo_t *sysinfo = NULL; ++ ++ sysinfo = get_sys_info(true); ++ if (sysinfo == NULL) { ++ ERROR("Can not get system info"); ++ return NULL; ++ } ++ ++ if (!(sysinfo->cgcpuinfo.cpu_rt_period)) { ++ ERROR("Daemon-scoped cpu-rt-period and cpu-rt-runtime are not supported by kernel"); ++ isulad_set_error_message("Daemon-scoped cpu-rt-period and cpu-rt-runtime are not supported by kernel"); ++ return NULL; ++ } ++ ++ nret = find_cgroup_mountpoint_and_root("cpu", &mnt, &root); ++ if (nret != 0 || mnt == NULL || root == NULL) { ++ ERROR("Can not find cgroup mnt and root path for subsystem 'cpu'"); ++ isulad_set_error_message("Can not find cgroup mnt and root path for subsystem 'cpu'"); ++ return NULL; ++ } ++ ++ // When iSulad is run inside docker, the root is based of the host cgroup. ++ // Replace root to "/" ++ if (strncmp(root, "/docker/", strlen("/docker/")) == 0) { ++ root[1] = '\0'; ++ } ++ ++ nret = snprintf(fpath, sizeof(fpath), "%s/%s", mnt, root); ++ if (nret < 0 || (size_t)nret >= sizeof(fpath)) { ++ ERROR("Failed to print string"); ++ return NULL; ++ } ++ ++ return util_strdup_s(fpath); ++} +\ No newline at end of file +diff --git a/src/daemon/common/sysinfo.h b/src/daemon/common/sysinfo.h +index 8468e00a..bbb3c6b5 100644 +--- a/src/daemon/common/sysinfo.h ++++ b/src/daemon/common/sysinfo.h +@@ -139,6 +139,8 @@ mountinfo_t *find_mount_info(mountinfo_t **minfos, const char *dir); + + void free_mounts_info(mountinfo_t **minfos); + ++char *sysinfo_cgroup_controller_cpurt_mnt_path(); ++ + #ifdef __cplusplus + } + #endif +diff --git a/src/daemon/executor/container_cb/execution.c b/src/daemon/executor/container_cb/execution.c +index 7b18a8e1..ed70fc14 100644 +--- a/src/daemon/executor/container_cb/execution.c ++++ b/src/daemon/executor/container_cb/execution.c +@@ -18,6 +18,12 @@ + #include <pthread.h> + #include <malloc.h> + #include <sys/eventfd.h> ++#include <stdbool.h> ++#include <stdint.h> ++#include <stdlib.h> ++#include <string.h> ++#include <libgen.h> ++ + #include <isula_libutils/container_config.h> + #include <isula_libutils/container_config_v2.h> + #include <isula_libutils/container_delete_request.h> +@@ -34,13 +40,13 @@ + #include <isula_libutils/container_stop_request.h> + #include <isula_libutils/container_stop_response.h> + #include <isula_libutils/json_common.h> +-#include <stdbool.h> +-#include <stdint.h> +-#include <stdlib.h> +-#include <string.h> ++#include <isula_libutils/auto_cleanup.h> ++#include <isula_libutils/log.h> + +-#include "isula_libutils/log.h" ++#include "isulad_config.h" ++#include "sysinfo.h" + #include "container_api.h" ++#include "specs_api.h" + #include "execution_extend.h" + #include "execution_information.h" + #include "execution_stream.h" +@@ -302,6 +308,135 @@ static void pack_start_response(container_start_response *response, uint32_t cc, + } + } + ++static int do_init_cpurt_cgroups_path(const char *path, int recursive_depth, const char *mnt_root, ++ int64_t cpu_rt_period, int64_t cpu_rt_runtime); ++ ++/* maybe create cpu realtime file */ ++static int maybe_create_cpu_realtime_file(int64_t value, const char *file, const char *path) ++{ ++ int ret; ++ __isula_auto_close int fd = -1; ++ ssize_t nwrite; ++ char fpath[PATH_MAX] = { 0 }; ++ char buf[ISULAD_NUMSTRLEN64] = { 0 }; ++ ++ if (value == 0) { ++ return 0; ++ } ++ ++ ret = util_mkdir_p(path, CONFIG_DIRECTORY_MODE); ++ if (ret != 0) { ++ ERROR("Failed to mkdir: %s", path); ++ return -1; ++ } ++ ++ ret = snprintf(fpath, sizeof(fpath), "%s/%s", path, file); ++ if (ret < 0 || ret >= sizeof(fpath)) { ++ ERROR("Failed to print string"); ++ return -1; ++ } ++ ret = snprintf(buf, sizeof(buf), "%lld", (long long int)value); ++ if (ret < 0 || (size_t)ret >= sizeof(buf)) { ++ ERROR("Failed to print string"); ++ return -1; ++ } ++ ++ fd = util_open(fpath, O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, 0700); ++ if (fd < 0) { ++ ERROR("Failed to open file: %s: %s", fpath, strerror(errno)); ++ isulad_set_error_message("Failed to open file: %s: %s", fpath, strerror(errno)); ++ return -1; ++ } ++ nwrite = util_write_nointr(fd, buf, strlen(buf)); ++ if (nwrite < 0 || nwrite != strlen(buf)) { ++ ERROR("Failed to write %s to %s: %s", buf, fpath, strerror(errno)); ++ isulad_set_error_message("Failed to write '%s' to '%s': %s", buf, fpath, strerror(errno)); ++ return -1; ++ } ++ ++ return 0; ++} ++ ++static int recursively_create_cgroup(const char *path, const char *mnt_root, int recursive_depth, int64_t cpu_rt_period, ++ int64_t cpu_rt_runtime) ++{ ++ int ret = 0; ++ __isula_auto_free char *dup = NULL; ++ char *dirpath = NULL; ++ char fpath[PATH_MAX] = { 0 }; ++ ++ dup = util_strdup_s(path); ++ dirpath = dirname(dup); ++ ret = do_init_cpurt_cgroups_path(dirpath, (recursive_depth + 1), mnt_root, cpu_rt_period, cpu_rt_runtime); ++ if (ret != 0) { ++ return ret; ++ } ++ ++ int nret = snprintf(fpath, sizeof(fpath), "%s/%s", mnt_root, path); ++ if (nret < 0 || (size_t)nret >= sizeof(fpath)) { ++ ERROR("Failed to print string"); ++ return ret; ++ } ++ ++ ret = maybe_create_cpu_realtime_file(cpu_rt_period, "cpu.rt_period_us", fpath); ++ if (ret != 0) { ++ return ret; ++ } ++ ++ return maybe_create_cpu_realtime_file(cpu_rt_runtime, "cpu.rt_runtime_us", fpath); ++} ++ ++/* init cgroups path */ ++static int do_init_cpurt_cgroups_path(const char *path, int recursive_depth, const char *mnt_root, ++ int64_t cpu_rt_period, int64_t cpu_rt_runtime) ++{ ++ if ((recursive_depth + 1) > MAX_PATH_DEPTH) { ++ ERROR("Reach the max cgroup depth:%s", path); ++ return -1; ++ } ++ ++ if (path == NULL || strcmp(path, "/") == 0 || strcmp(path, ".") == 0) { ++ return 0; ++ } ++ ++ // Recursively create cgroup to ensure that the system and all parent cgroups have values set ++ // for the period and runtime as this limits what the children can be set to. ++ return recursively_create_cgroup(path, mnt_root, recursive_depth, cpu_rt_period, cpu_rt_runtime); ++} ++ ++// TODO: maybe we should adapt to cgroup v2 ++static int cpurt_controller_init(const char *id, const host_config *host_spec) ++{ ++ __isula_auto_free char *mnt_root = NULL; ++ __isula_auto_free char *cgroups_path = NULL; ++ char *dirpath = NULL; ++ int64_t cpu_rt_period = 0; ++ int64_t cpu_rt_runtime = 0; ++ ++ cgroups_path = merge_container_cgroups_path(id, host_spec); ++ if (cgroups_path == NULL || strcmp(cgroups_path, "/") == 0 || strcmp(cgroups_path, ".") == 0) { ++ return 0; ++ } ++ ++ if (conf_get_cgroup_cpu_rt(&cpu_rt_period, &cpu_rt_runtime)) { ++ return -1; ++ } ++ ++ if (cpu_rt_period == 0 && cpu_rt_runtime == 0) { ++ return 0; ++ } ++ ++ mnt_root = sysinfo_cgroup_controller_cpurt_mnt_path(); ++ if (mnt_root == NULL) { ++ ERROR("Failed to get cpu rt controller mnt root path"); ++ return -1; ++ } ++ ++ dirpath = dirname(cgroups_path); ++ ++ return do_init_cpurt_cgroups_path(dirpath, 0, mnt_root, cpu_rt_period, cpu_rt_runtime); ++} ++ + static int container_start_prepare(container_t *cont, const container_start_request *request, int stdinfd, + struct io_write_wrapper *stdout_handler, struct io_write_wrapper *stderr_handler, + char **fifopath, char *fifos[], int *sync_fd, pthread_t *thread_id) +@@ -314,6 +449,19 @@ static int container_start_prepare(container_t *cont, const container_start_requ + return -1; + } + ++ // init cgroup path for cpu_rt_runtime and cpu_rt_period ++ // we should do this in start container, not create container ++ // because, in scenarios: ++ // 1. enable cpu-rt of isulad; ++ // 2. then run container with --cpu-rt-runtime ++ // 3. then reboot machine; ++ // 4. finally, start before container, it will failed... ++ // cause of no one to set value into cgroup/isulad/cpu-rt-runtime and cpu-rt-period. ++ if (cpurt_controller_init(id, cont->hostconfig) != 0) { ++ isulad_set_error_message("Failed to init controller of cpu-rt for container \"%s\".", id); ++ return -1; ++ } ++ + if (prepare_start_io(cont, request, fifopath, fifos, stdinfd, stdout_handler, stderr_handler, sync_fd, thread_id) != + 0) { + return -1; +diff --git a/src/daemon/executor/container_cb/execution_create.c b/src/daemon/executor/container_cb/execution_create.c +index 4cc333fd..4abc89c7 100644 +--- a/src/daemon/executor/container_cb/execution_create.c ++++ b/src/daemon/executor/container_cb/execution_create.c +@@ -19,6 +19,14 @@ + #include <errno.h> + #include <sys/stat.h> + #include <malloc.h> ++#include <limits.h> ++#include <stdbool.h> ++#include <stdint.h> ++#include <stdlib.h> ++#include <string.h> ++ ++#include <isula_libutils/log.h> ++#include <isula_libutils/auto_cleanup.h> + #include <isula_libutils/container_config.h> + #include <isula_libutils/container_config_v2.h> + #include <isula_libutils/defs.h> +@@ -26,14 +34,7 @@ + #include <isula_libutils/isulad_daemon_configs.h> + #include <isula_libutils/json_common.h> + #include <isula_libutils/oci_runtime_spec.h> +-#include <limits.h> +-#include <stdbool.h> +-#include <stdint.h> +-#include <stdlib.h> +-#include <string.h> +-#include <libgen.h> + +-#include "isula_libutils/log.h" + #include "isulad_config.h" + #include "specs_api.h" + #include "verify.h" +@@ -58,9 +59,6 @@ + #include "opt_log.h" + #include "runtime_api.h" + +-static int do_init_cpurt_cgroups_path(const char *path, int recursive_depth, const char *mnt_root, +- int64_t cpu_rt_period, int64_t cpu_rt_runtime); +- + static int create_request_check(const container_create_request *request) + { + int ret = 0; +@@ -1323,203 +1321,6 @@ static int save_container_config_before_create(const char *id, const char *runti + return 0; + } + +-/* maybe create cpu realtime file */ +-static int maybe_create_cpu_realtime_file(int64_t value, const char *file, const char *path) +-{ +- int ret; +- int fd = -1; +- ssize_t nwrite; +- char fpath[PATH_MAX] = { 0 }; +- char buf[ISULAD_NUMSTRLEN64] = { 0 }; +- +- if (value == 0) { +- return 0; +- } +- +- ret = util_mkdir_p(path, CONFIG_DIRECTORY_MODE); +- if (ret != 0) { +- ERROR("Failed to mkdir: %s", path); +- return -1; +- } +- +- ret = snprintf(fpath, sizeof(fpath), "%s/%s", path, file); +- if (ret < 0 || ret >= sizeof(fpath)) { +- ERROR("Failed to print string"); +- return -1; +- } +- ret = snprintf(buf, sizeof(buf), "%lld", (long long int)value); +- if (ret < 0 || (size_t)ret >= sizeof(buf)) { +- ERROR("Failed to print string"); +- return -1; +- } +- +- fd = util_open(fpath, O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, 0700); +- if (fd < 0) { +- ERROR("Failed to open file: %s: %s", fpath, strerror(errno)); +- isulad_set_error_message("Failed to open file: %s: %s", fpath, strerror(errno)); +- return -1; +- } +- nwrite = util_write_nointr(fd, buf, strlen(buf)); +- if (nwrite < 0) { +- ERROR("Failed to write %s to %s: %s", buf, fpath, strerror(errno)); +- isulad_set_error_message("Failed to write '%s' to '%s': %s", buf, fpath, strerror(errno)); +- close(fd); +- return -1; +- } +- close(fd); +- +- return 0; +-} +- +-static int recursively_create_cgroup(const char *path, const char *mnt_root, int recursive_depth, int64_t cpu_rt_period, +- int64_t cpu_rt_runtime) +-{ +- int ret = 0; +- char *dup = NULL; +- char *dirpath = NULL; +- char fpath[PATH_MAX] = { 0 }; +- +- dup = util_strdup_s(path); +- dirpath = dirname(dup); +- ret = do_init_cpurt_cgroups_path(dirpath, (recursive_depth + 1), mnt_root, cpu_rt_period, cpu_rt_runtime); +- free(dup); +- if (ret != 0) { +- return ret; +- } +- +- int nret = snprintf(fpath, sizeof(fpath), "%s/%s", mnt_root, path); +- if (nret < 0 || (size_t)nret >= sizeof(fpath)) { +- ERROR("Failed to print string"); +- ret = -1; +- goto out; +- } +- +- ret = maybe_create_cpu_realtime_file(cpu_rt_period, "cpu.rt_period_us", fpath); +- if (ret != 0) { +- goto out; +- } +- +- ret = maybe_create_cpu_realtime_file(cpu_rt_runtime, "cpu.rt_runtime_us", fpath); +- if (ret != 0) { +- goto out; +- } +- +-out: +- return ret; +-} +- +-/* init cgroups path */ +-static int do_init_cpurt_cgroups_path(const char *path, int recursive_depth, const char *mnt_root, +- int64_t cpu_rt_period, int64_t cpu_rt_runtime) +-{ +- if ((recursive_depth + 1) > MAX_PATH_DEPTH) { +- ERROR("Reach the max cgroup depth:%s", path); +- return -1; +- } +- +- if (path == NULL || strcmp(path, "/") == 0 || strcmp(path, ".") == 0) { +- return 0; +- } +- +- // Recursively create cgroup to ensure that the system and all parent cgroups have values set +- // for the period and runtime as this limits what the children can be set to. +- if (recursively_create_cgroup(path, mnt_root, recursive_depth, cpu_rt_period, cpu_rt_runtime)) { +- return -1; +- } +- +- return 0; +-} +- +-static char *get_cpurt_controller_mnt_path() +-{ +- char *res = NULL; +- int nret = 0; +- char *mnt = NULL; +- char *root = NULL; +- char fpath[PATH_MAX] = { 0 }; +- +- nret = find_cgroup_mountpoint_and_root("cpu", &mnt, &root); +- if (nret != 0 || mnt == NULL || root == NULL) { +- ERROR("Can not find cgroup mnt and root path for subsystem 'cpu'"); +- isulad_set_error_message("Can not find cgroup mnt and root path for subsystem 'cpu'"); +- goto out; +- } +- +- // When iSulad is run inside docker, the root is based of the host cgroup. +- // Replace root to "/" +- if (strncmp(root, "/docker/", strlen("/docker/")) == 0) { +- root[1] = '\0'; +- } +- +- nret = snprintf(fpath, sizeof(fpath), "%s/%s", mnt, root); +- if (nret < 0 || (size_t)nret >= sizeof(fpath)) { +- ERROR("Failed to print string"); +- goto out; +- } +- +- res = util_strdup_s(fpath); +- +-out: +- free(mnt); +- free(root); +- return res; +-} +- +-static int cpurt_controller_init(const char *cgroups_path) +-{ +- int ret = 0; +- char *dup = NULL; +- char *dirpath = NULL; +- int64_t cpu_rt_period = 0; +- int64_t cpu_rt_runtime = 0; +- sysinfo_t *sysinfo = NULL; +- char *mnt_root = NULL; +- +- if (cgroups_path == NULL || strcmp(cgroups_path, "/") == 0 || strcmp(cgroups_path, ".") == 0) { +- return 0; +- } +- +- if (conf_get_cgroup_cpu_rt(&cpu_rt_period, &cpu_rt_runtime)) { +- return -1; +- } +- +- if (cpu_rt_period == 0 && cpu_rt_runtime == 0) { +- return 0; +- } +- +- sysinfo = get_sys_info(true); +- if (sysinfo == NULL) { +- ERROR("Can not get system info"); +- ret = -1; +- goto out; +- } +- +- if (!(sysinfo->cgcpuinfo.cpu_rt_period)) { +- ERROR("Daemon-scoped cpu-rt-period and cpu-rt-runtime are not supported by kernel"); +- isulad_set_error_message("Daemon-scoped cpu-rt-period and cpu-rt-runtime are not supported by kernel"); +- ret = -1; +- goto out; +- } +- +- mnt_root = get_cpurt_controller_mnt_path(); +- if (mnt_root == NULL) { +- ERROR("Failed to get cpu rt controller mnt root path"); +- isulad_set_error_message("Failed to get cpu rt controller mnt root path"); +- ret = -1; +- goto out; +- } +- +- dup = util_strdup_s(cgroups_path); +- dirpath = dirname(dup); +- +- ret = do_init_cpurt_cgroups_path(dirpath, 0, mnt_root, cpu_rt_period, cpu_rt_runtime); +- +-out: +- free(mnt_root); +- free(dup); +- return ret; +-} +- + /* + * request -> host_spec + container_spec + * container_spec + image config +@@ -1680,13 +1481,6 @@ int container_create_cb(const container_create_request *request, container_creat + goto umount_channel; + } + +- // init cgroup path for cpu_rt_runtime and cpu_rt_period +- if (cpurt_controller_init(oci_spec->linux->cgroups_path) != 0) { +- ERROR("Unable to init CPU RT controller %s", oci_spec->linux->cgroups_path); +- cc = ISULAD_ERR_EXEC; +- goto umount_channel; +- } +- + if (container_v2_spec_merge_contaner_spec(v2_spec) != 0) { + ERROR("Failed to merge container settings"); + cc = ISULAD_ERR_EXEC; +diff --git a/src/daemon/modules/api/specs_api.h b/src/daemon/modules/api/specs_api.h +index 4c132108..e0a73f55 100644 +--- a/src/daemon/modules/api/specs_api.h ++++ b/src/daemon/modules/api/specs_api.h +@@ -28,7 +28,7 @@ extern "C" { + + int merge_all_specs(host_config *host_spec, const char *real_rootfs, container_config_v2_common_config *v2_spec, + oci_runtime_spec *oci_spec); +-int merge_oci_cgroups_path(const char *id, oci_runtime_spec *oci_spec, const host_config *host_spec); ++char *merge_container_cgroups_path(const char *id, const host_config *host_spec); + int merge_global_config(oci_runtime_spec *oci_spec); + oci_runtime_spec *load_oci_config(const char *rootpath, const char *name); + oci_runtime_spec *default_spec(bool system_container); +diff --git a/src/daemon/modules/spec/specs.c b/src/daemon/modules/spec/specs.c +index 12d9b96d..857fc3dc 100644 +--- a/src/daemon/modules/spec/specs.c ++++ b/src/daemon/modules/spec/specs.c +@@ -26,6 +26,7 @@ + #include <isula_libutils/oci_runtime_hooks.h> + #include <isula_libutils/host_config.h> + #include <isula_libutils/log.h> ++#include <isula_libutils/auto_cleanup.h> + #include <limits.h> + #include <stdint.h> + +@@ -165,36 +166,43 @@ out: + return ret; + } + +-static int make_annotations_cgroup_dir(const container_config *container_spec, const host_config *host_spec) ++static char *do_get_container_cgroup_path(const host_config *host_spec) + { +- int ret = 0; +- char cleaned[PATH_MAX] = { 0 }; +- char *default_cgroup_parent = NULL; + char *path = NULL; + +- default_cgroup_parent = conf_get_isulad_cgroup_parent(); + if (host_spec->cgroup_parent != NULL) { +- path = host_spec->cgroup_parent; +- } else if (default_cgroup_parent != NULL) { +- path = default_cgroup_parent; ++ // first, use user setting ++ path = util_strdup_s(host_spec->cgroup_parent); ++ } else { ++ // second, if user donot set, use setting from daemon config ++ path = conf_get_isulad_cgroup_parent(); + } ++ + if (path == NULL) { +- path = "/isulad"; ++ // third, all faild, just use default '/isulad' ++ path = util_strdup_s("/isulad"); + } ++ ++ return path; ++} ++ ++static int make_annotations_cgroup_dir(const container_config *container_spec, const host_config *host_spec) ++{ ++ char cleaned[PATH_MAX] = { 0 }; ++ __isula_auto_free char *path = NULL; ++ ++ path = do_get_container_cgroup_path(host_spec); + if (util_clean_path(path, cleaned, sizeof(cleaned)) == NULL) { + ERROR("Failed to clean path: %s", path); +- ret = -1; +- goto out; ++ return -1; + } ++ + if (append_json_map_string_string(container_spec->annotations, "cgroup.dir", cleaned)) { + ERROR("Realloc annotations failed"); +- ret = -1; +- goto out; ++ return -1; + } + +-out: +- free(default_cgroup_parent); +- return ret; ++ return 0; + } + + static int make_annotations_oom_score_adj(const container_config *container_spec, const host_config *host_spec) +@@ -2058,42 +2066,40 @@ out: + return ret; + } + +-int merge_oci_cgroups_path(const char *id, oci_runtime_spec *oci_spec, const host_config *host_spec) ++char *merge_container_cgroups_path(const char *id, const host_config *host_spec) + { +- int ret = 0; +- char *default_cgroup_parent = NULL; +- char *path = NULL; ++ __isula_auto_free char *path = NULL; + ++ if (id == NULL || host_spec == NULL) { ++ ERROR("Invalid arguments"); ++ return NULL; ++ } ++ ++ path = do_get_container_cgroup_path(host_spec); ++ ++ return util_path_join(path, id); ++} ++ ++static int merge_oci_cgroups_path(const char *id, oci_runtime_spec *oci_spec, const host_config *host_spec) ++{ + if (id == NULL || oci_spec == NULL || host_spec == NULL) { + ERROR("Invalid arguments"); +- ret = -1; +- goto out; ++ return -1; + } + + if (make_sure_oci_spec_linux(oci_spec) != 0) { + ERROR("Failed to make oci spec linux"); +- ret = -1; +- goto out; ++ return -1; + } + +- default_cgroup_parent = conf_get_isulad_cgroup_parent(); +- path = default_cgroup_parent; +- if (host_spec->cgroup_parent != NULL) { +- path = host_spec->cgroup_parent; +- } ++ free(oci_spec->linux->cgroups_path); ++ oci_spec->linux->cgroups_path = merge_container_cgroups_path(id, host_spec); + +- if (path == NULL) { +- free(oci_spec->linux->cgroups_path); +- oci_spec->linux->cgroups_path = util_path_join("/isulad", id); +- return 0; ++ if (oci_spec->linux->cgroups_path == NULL) { ++ WARN("OCI spec cgroups path is NULL"); + } + +- free(oci_spec->linux->cgroups_path); +- oci_spec->linux->cgroups_path = util_path_join(path, id); +- +-out: +- free(default_cgroup_parent); +- return ret; ++ return 0; + } + + int merge_all_specs(host_config *host_spec, const char *real_rootfs, container_config_v2_common_config *v2_spec, +diff --git a/test/specs/specs/specs_ut.cc b/test/specs/specs/specs_ut.cc +index c4014e2e..96aa1c63 100644 +--- a/test/specs/specs/specs_ut.cc ++++ b/test/specs/specs/specs_ut.cc +@@ -232,15 +232,16 @@ char *invoke_conf_get_isulad_cgroup_parent() + return util_strdup_s("/var/lib/isulad/engines/lcr"); + } + +-TEST_F(SpecsUnitTest, test_merge_oci_cgroups_path_1) ++TEST_F(SpecsUnitTest, test_merge_container_cgroups_path_1) + { +- ASSERT_EQ(merge_oci_cgroups_path(nullptr, nullptr, nullptr), -1); ++ ASSERT_EQ(merge_container_cgroups_path(nullptr, nullptr), nullptr); + } + +-TEST_F(SpecsUnitTest, test_merge_oci_cgroups_path_2) ++TEST_F(SpecsUnitTest, test_merge_container_cgroups_path_2) + { + oci_runtime_spec *oci_spec = nullptr; + host_config *host_spec = nullptr; ++ char *merged_cp = nullptr; + + oci_spec = (oci_runtime_spec *)util_common_calloc_s(sizeof(oci_runtime_spec)); + ASSERT_TRUE(oci_spec != nullptr); +@@ -250,20 +251,23 @@ TEST_F(SpecsUnitTest, test_merge_oci_cgroups_path_2) + + EXPECT_CALL(m_isulad_conf, GetCgroupParent()).WillRepeatedly(Invoke(invoke_conf_get_isulad_cgroup_parent_null)); + +- ASSERT_EQ(merge_oci_cgroups_path("123", oci_spec, host_spec), 0); ++ merged_cp = merge_container_cgroups_path("123", host_spec); ++ ASSERT_NE(merged_cp, nullptr); + +- ASSERT_STREQ(oci_spec->linux->cgroups_path, "/isulad/123"); ++ ASSERT_STREQ(merged_cp, "/isulad/123"); + + free_oci_runtime_spec(oci_spec); + free_host_config(host_spec); ++ free(merged_cp); + + testing::Mock::VerifyAndClearExpectations(&m_isulad_conf); + } + +-TEST_F(SpecsUnitTest, test_merge_oci_cgroups_path_3) ++TEST_F(SpecsUnitTest, test_merge_container_cgroups_path_3) + { + oci_runtime_spec *oci_spec = nullptr; + host_config *host_spec = nullptr; ++ char *merged_cp = nullptr; + + oci_spec = (oci_runtime_spec *)util_common_calloc_s(sizeof(oci_runtime_spec)); + ASSERT_TRUE(oci_spec != nullptr); +@@ -275,20 +279,23 @@ TEST_F(SpecsUnitTest, test_merge_oci_cgroups_path_3) + + EXPECT_CALL(m_isulad_conf, GetCgroupParent()).WillRepeatedly(Invoke(invoke_conf_get_isulad_cgroup_parent_null)); + +- ASSERT_EQ(merge_oci_cgroups_path("123", oci_spec, host_spec), 0); ++ merged_cp = merge_container_cgroups_path("123", host_spec); ++ ASSERT_NE(merged_cp, nullptr); + +- ASSERT_STREQ(oci_spec->linux->cgroups_path, "/test/123"); ++ ASSERT_STREQ(merged_cp, "/test/123"); + + free_oci_runtime_spec(oci_spec); + free_host_config(host_spec); ++ free(merged_cp); + + testing::Mock::VerifyAndClearExpectations(&m_isulad_conf); + } + +-TEST_F(SpecsUnitTest, test_merge_oci_cgroups_path_4) ++TEST_F(SpecsUnitTest, test_merge_container_cgroups_path_4) + { + oci_runtime_spec *oci_spec = nullptr; + host_config *host_spec = nullptr; ++ char *merged_cp = nullptr; + + oci_spec = (oci_runtime_spec *)util_common_calloc_s(sizeof(oci_runtime_spec)); + ASSERT_TRUE(oci_spec != nullptr); +@@ -298,20 +305,23 @@ TEST_F(SpecsUnitTest, test_merge_oci_cgroups_path_4) + + EXPECT_CALL(m_isulad_conf, GetCgroupParent()).WillRepeatedly(Invoke(invoke_conf_get_isulad_cgroup_parent)); + +- ASSERT_EQ(merge_oci_cgroups_path("123", oci_spec, host_spec), 0); ++ merged_cp = merge_container_cgroups_path("123", host_spec); ++ ASSERT_NE(merged_cp, nullptr); + +- ASSERT_STREQ(oci_spec->linux->cgroups_path, "/var/lib/isulad/engines/lcr/123"); ++ ASSERT_STREQ(merged_cp, "/var/lib/isulad/engines/lcr/123"); + + free_oci_runtime_spec(oci_spec); + free_host_config(host_spec); ++ free(merged_cp); + + testing::Mock::VerifyAndClearExpectations(&m_isulad_conf); + } + +-TEST_F(SpecsUnitTest, test_merge_oci_cgroups_path_5) ++TEST_F(SpecsUnitTest, test_merge_container_cgroups_path_5) + { + oci_runtime_spec *oci_spec = nullptr; + host_config *host_spec = nullptr; ++ char *merged_cp = nullptr; + + oci_spec = (oci_runtime_spec *)util_common_calloc_s(sizeof(oci_runtime_spec)); + ASSERT_TRUE(oci_spec != nullptr); +@@ -323,12 +333,14 @@ TEST_F(SpecsUnitTest, test_merge_oci_cgroups_path_5) + + EXPECT_CALL(m_isulad_conf, GetCgroupParent()).WillRepeatedly(Invoke(invoke_conf_get_isulad_cgroup_parent)); + +- ASSERT_EQ(merge_oci_cgroups_path("123", oci_spec, host_spec), 0); ++ merged_cp = merge_container_cgroups_path("123", host_spec); ++ ASSERT_NE(merged_cp, nullptr); + +- ASSERT_STREQ(oci_spec->linux->cgroups_path, "/test/123"); ++ ASSERT_STREQ(merged_cp, "/test/123"); + + free_oci_runtime_spec(oci_spec); + free_host_config(host_spec); ++ free(merged_cp); + + testing::Mock::VerifyAndClearExpectations(&m_isulad_conf); + } +-- +2.25.1 + diff --git a/0013-fix-selinux_label_ut-timeout-and-add-timeout-for-all.patch b/0013-fix-selinux_label_ut-timeout-and-add-timeout-for-all.patch new file mode 100644 index 0000000..38efc68 --- /dev/null +++ b/0013-fix-selinux_label_ut-timeout-and-add-timeout-for-all.patch @@ -0,0 +1,465 @@ +From af8fb9fcf604775f527b58e1b02f220dffd8ff35 Mon Sep 17 00:00:00 2001 +From: zhangxiaoyu <zhangxiaoyu58@huawei.com> +Date: Thu, 16 Feb 2023 15:26:10 +0800 +Subject: [PATCH 13/53] fix selinux_label_ut timeout and add timeout for all ut + +Signed-off-by: zhangxiaoyu <zhangxiaoyu58@huawei.com> +--- + test/cgroup/cpu/CMakeLists.txt | 1 + + test/cmd/isula/extend/pause/CMakeLists.txt | 1 + + test/cmd/isula/extend/resume/CMakeLists.txt | 2 +- + test/cmd/isula/information/info/CMakeLists.txt | 1 + + test/cmd/isula/information/ps/CMakeLists.txt | 1 + + test/cmd/isula/utils/client_show_format/CMakeLists.txt | 1 + + test/cmd/isula/utils/template_string_parse/CMakeLists.txt | 1 + + test/cmd/isulad-shim/CMakeLists.txt | 1 + + test/cutils/mainloop/CMakeLists.txt | 1 + + test/cutils/map/CMakeLists.txt | 1 + + test/cutils/path/CMakeLists.txt | 1 + + test/cutils/util_atomic/CMakeLists.txt | 1 + + test/cutils/utils_aes/CMakeLists.txt | 1 + + test/cutils/utils_array/CMakeLists.txt | 1 + + test/cutils/utils_base64/CMakeLists.txt | 1 + + test/cutils/utils_convert/CMakeLists.txt | 1 + + test/cutils/utils_error/CMakeLists.txt | 1 + + test/cutils/utils_file/CMakeLists.txt | 1 + + test/cutils/utils_filters/CMakeLists.txt | 1 + + test/cutils/utils_fs/CMakeLists.txt | 1 + + test/cutils/utils_mount_spec/CMakeLists.txt | 1 + + test/cutils/utils_namespace/CMakeLists.txt | 1 + + test/cutils/utils_network/CMakeLists.txt | 1 + + test/cutils/utils_pwgr/CMakeLists.txt | 1 + + test/cutils/utils_regex/CMakeLists.txt | 1 + + test/cutils/utils_string/CMakeLists.txt | 1 + + test/cutils/utils_timestamp/CMakeLists.txt | 1 + + test/cutils/utils_utils/CMakeLists.txt | 1 + + test/cutils/utils_verify/CMakeLists.txt | 1 + + test/image/oci/oci_config_merge/CMakeLists.txt | 1 + + test/image/oci/registry/CMakeLists.txt | 1 + + test/image/oci/storage/images/CMakeLists.txt | 1 + + test/image/oci/storage/layers/CMakeLists.txt | 2 ++ + test/image/oci/storage/rootfs/CMakeLists.txt | 1 + + test/network/CMakeLists.txt | 1 + + test/runtime/isula/CMakeLists.txt | 1 + + test/runtime/lcr/CMakeLists.txt | 1 + + .../services/execution/execute/execution_extend/CMakeLists.txt | 1 + + test/services/execution/spec/CMakeLists.txt | 2 ++ + test/services/execution/spec/selinux_label_ut.cc | 2 ++ + test/specs/specs/CMakeLists.txt | 1 + + test/specs/specs_extend/CMakeLists.txt | 1 + + test/volume/CMakeLists.txt | 3 ++- + 43 files changed, 47 insertions(+), 2 deletions(-) + +diff --git a/test/cgroup/cpu/CMakeLists.txt b/test/cgroup/cpu/CMakeLists.txt +index 159b0d85..6a8af719 100644 +--- a/test/cgroup/cpu/CMakeLists.txt ++++ b/test/cgroup/cpu/CMakeLists.txt +@@ -26,3 +26,4 @@ target_include_directories(${EXE} PUBLIC + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cmd/isula/extend/pause/CMakeLists.txt b/test/cmd/isula/extend/pause/CMakeLists.txt +index af48c23f..6eab5172 100644 +--- a/test/cmd/isula/extend/pause/CMakeLists.txt ++++ b/test/cmd/isula/extend/pause/CMakeLists.txt +@@ -47,3 +47,4 @@ target_include_directories(${EXE} PUBLIC + ) + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cmd/isula/extend/resume/CMakeLists.txt b/test/cmd/isula/extend/resume/CMakeLists.txt +index 729c4d3b..0812a66e 100644 +--- a/test/cmd/isula/extend/resume/CMakeLists.txt ++++ b/test/cmd/isula/extend/resume/CMakeLists.txt +@@ -48,4 +48,4 @@ target_include_directories(${EXE} PUBLIC + ) + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) +- ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cmd/isula/information/info/CMakeLists.txt b/test/cmd/isula/information/info/CMakeLists.txt +index 2f134986..d8d8ddb7 100644 +--- a/test/cmd/isula/information/info/CMakeLists.txt ++++ b/test/cmd/isula/information/info/CMakeLists.txt +@@ -49,3 +49,4 @@ target_include_directories(${EXE} PUBLIC + ) + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cmd/isula/information/ps/CMakeLists.txt b/test/cmd/isula/information/ps/CMakeLists.txt +index 81d4202c..436e611b 100644 +--- a/test/cmd/isula/information/ps/CMakeLists.txt ++++ b/test/cmd/isula/information/ps/CMakeLists.txt +@@ -51,3 +51,4 @@ target_include_directories(${EXE} PUBLIC + ) + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cmd/isula/utils/client_show_format/CMakeLists.txt b/test/cmd/isula/utils/client_show_format/CMakeLists.txt +index 60d538ad..e19332cf 100644 +--- a/test/cmd/isula/utils/client_show_format/CMakeLists.txt ++++ b/test/cmd/isula/utils/client_show_format/CMakeLists.txt +@@ -19,3 +19,4 @@ target_include_directories(${EXE} PUBLIC + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cmd/isula/utils/template_string_parse/CMakeLists.txt b/test/cmd/isula/utils/template_string_parse/CMakeLists.txt +index 32256b6b..25efd25c 100644 +--- a/test/cmd/isula/utils/template_string_parse/CMakeLists.txt ++++ b/test/cmd/isula/utils/template_string_parse/CMakeLists.txt +@@ -19,3 +19,4 @@ target_include_directories(${EXE} PUBLIC + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cmd/isulad-shim/CMakeLists.txt b/test/cmd/isulad-shim/CMakeLists.txt +index 1c2de232..dc293f6d 100644 +--- a/test/cmd/isulad-shim/CMakeLists.txt ++++ b/test/cmd/isulad-shim/CMakeLists.txt +@@ -30,3 +30,4 @@ target_include_directories(${EXE} PUBLIC + ) + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/mainloop/CMakeLists.txt b/test/cutils/mainloop/CMakeLists.txt +index 78e3f18d..61ea7ffc 100644 +--- a/test/cutils/mainloop/CMakeLists.txt ++++ b/test/cutils/mainloop/CMakeLists.txt +@@ -25,3 +25,4 @@ target_link_libraries(${EXE} + libutils_ut -lcrypto -lyajl -lz) + + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/map/CMakeLists.txt b/test/cutils/map/CMakeLists.txt +index 4059559f..bd21ee3f 100644 +--- a/test/cutils/map/CMakeLists.txt ++++ b/test/cutils/map/CMakeLists.txt +@@ -14,3 +14,4 @@ target_include_directories(${EXE} PUBLIC + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} libutils_ut -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/path/CMakeLists.txt b/test/cutils/path/CMakeLists.txt +index 745258a1..aa0c6c6e 100644 +--- a/test/cutils/path/CMakeLists.txt ++++ b/test/cutils/path/CMakeLists.txt +@@ -15,3 +15,4 @@ target_include_directories(${EXE} PUBLIC + set_target_properties(${EXE} PROPERTIES LINK_FLAGS "-Wl,--wrap,getcwd -Wl,--wrap,readlink") + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/util_atomic/CMakeLists.txt b/test/cutils/util_atomic/CMakeLists.txt +index 071b2a04..21b05c46 100644 +--- a/test/cutils/util_atomic/CMakeLists.txt ++++ b/test/cutils/util_atomic/CMakeLists.txt +@@ -16,3 +16,4 @@ target_include_directories(${EXE} PUBLIC + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/utils_aes/CMakeLists.txt b/test/cutils/utils_aes/CMakeLists.txt +index f7535bb3..c4d175c8 100644 +--- a/test/cutils/utils_aes/CMakeLists.txt ++++ b/test/cutils/utils_aes/CMakeLists.txt +@@ -14,3 +14,4 @@ target_include_directories(${EXE} PUBLIC + ) + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/utils_array/CMakeLists.txt b/test/cutils/utils_array/CMakeLists.txt +index 71733e31..afe564bf 100644 +--- a/test/cutils/utils_array/CMakeLists.txt ++++ b/test/cutils/utils_array/CMakeLists.txt +@@ -16,3 +16,4 @@ target_include_directories(${EXE} PUBLIC + set_target_properties(${EXE} PROPERTIES LINK_FLAGS "-Wl,--wrap,calloc") + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/utils_base64/CMakeLists.txt b/test/cutils/utils_base64/CMakeLists.txt +index d5b99361..35e6ba04 100644 +--- a/test/cutils/utils_base64/CMakeLists.txt ++++ b/test/cutils/utils_base64/CMakeLists.txt +@@ -15,3 +15,4 @@ target_include_directories(${EXE} PUBLIC + ) + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/utils_convert/CMakeLists.txt b/test/cutils/utils_convert/CMakeLists.txt +index 30068208..fcf70a7c 100644 +--- a/test/cutils/utils_convert/CMakeLists.txt ++++ b/test/cutils/utils_convert/CMakeLists.txt +@@ -15,3 +15,4 @@ target_include_directories(${EXE} PUBLIC + ) + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/utils_error/CMakeLists.txt b/test/cutils/utils_error/CMakeLists.txt +index 28016605..9607aeeb 100644 +--- a/test/cutils/utils_error/CMakeLists.txt ++++ b/test/cutils/utils_error/CMakeLists.txt +@@ -14,3 +14,4 @@ target_include_directories(${EXE} PUBLIC + ) + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/utils_file/CMakeLists.txt b/test/cutils/utils_file/CMakeLists.txt +index 20317e15..01b2ff47 100644 +--- a/test/cutils/utils_file/CMakeLists.txt ++++ b/test/cutils/utils_file/CMakeLists.txt +@@ -16,3 +16,4 @@ target_include_directories(${EXE} PUBLIC + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/utils_filters/CMakeLists.txt b/test/cutils/utils_filters/CMakeLists.txt +index 31d3ac25..561ff4ef 100644 +--- a/test/cutils/utils_filters/CMakeLists.txt ++++ b/test/cutils/utils_filters/CMakeLists.txt +@@ -16,3 +16,4 @@ target_include_directories(${EXE} PUBLIC + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/utils_fs/CMakeLists.txt b/test/cutils/utils_fs/CMakeLists.txt +index 7ff3176a..e909aee6 100644 +--- a/test/cutils/utils_fs/CMakeLists.txt ++++ b/test/cutils/utils_fs/CMakeLists.txt +@@ -14,3 +14,4 @@ target_include_directories(${EXE} PUBLIC + ) + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/utils_mount_spec/CMakeLists.txt b/test/cutils/utils_mount_spec/CMakeLists.txt +index 24fb5add..099805d2 100644 +--- a/test/cutils/utils_mount_spec/CMakeLists.txt ++++ b/test/cutils/utils_mount_spec/CMakeLists.txt +@@ -14,3 +14,4 @@ target_include_directories(${EXE} PUBLIC + ) + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/utils_namespace/CMakeLists.txt b/test/cutils/utils_namespace/CMakeLists.txt +index 8add4a71..fb535e71 100644 +--- a/test/cutils/utils_namespace/CMakeLists.txt ++++ b/test/cutils/utils_namespace/CMakeLists.txt +@@ -14,3 +14,4 @@ target_include_directories(${EXE} PUBLIC + ) + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/utils_network/CMakeLists.txt b/test/cutils/utils_network/CMakeLists.txt +index cf0fc481..22e421ab 100644 +--- a/test/cutils/utils_network/CMakeLists.txt ++++ b/test/cutils/utils_network/CMakeLists.txt +@@ -20,3 +20,4 @@ target_link_libraries(${EXE} + libutils_ut -lcrypto -lyajl -lz) + + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/utils_pwgr/CMakeLists.txt b/test/cutils/utils_pwgr/CMakeLists.txt +index 5938991e..34acb92a 100644 +--- a/test/cutils/utils_pwgr/CMakeLists.txt ++++ b/test/cutils/utils_pwgr/CMakeLists.txt +@@ -15,3 +15,4 @@ target_include_directories(${EXE} PUBLIC + ) + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/utils_regex/CMakeLists.txt b/test/cutils/utils_regex/CMakeLists.txt +index 3f6410b2..45ba2604 100644 +--- a/test/cutils/utils_regex/CMakeLists.txt ++++ b/test/cutils/utils_regex/CMakeLists.txt +@@ -14,3 +14,4 @@ target_include_directories(${EXE} PUBLIC + ) + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/utils_string/CMakeLists.txt b/test/cutils/utils_string/CMakeLists.txt +index 1343f4e6..b9968c43 100644 +--- a/test/cutils/utils_string/CMakeLists.txt ++++ b/test/cutils/utils_string/CMakeLists.txt +@@ -16,3 +16,4 @@ target_include_directories(${EXE} PUBLIC + set_target_properties(${EXE} PROPERTIES LINK_FLAGS "-Wl,--wrap,util_strdup_s -Wl,--wrap,calloc -Wl,--wrap,strcat_s") + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/utils_timestamp/CMakeLists.txt b/test/cutils/utils_timestamp/CMakeLists.txt +index 38aec640..6da3fcc9 100644 +--- a/test/cutils/utils_timestamp/CMakeLists.txt ++++ b/test/cutils/utils_timestamp/CMakeLists.txt +@@ -14,3 +14,4 @@ target_include_directories(${EXE} PUBLIC + ) + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/utils_utils/CMakeLists.txt b/test/cutils/utils_utils/CMakeLists.txt +index 7b3bd546..99a83e7a 100644 +--- a/test/cutils/utils_utils/CMakeLists.txt ++++ b/test/cutils/utils_utils/CMakeLists.txt +@@ -14,3 +14,4 @@ target_include_directories(${EXE} PUBLIC + ) + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/utils_verify/CMakeLists.txt b/test/cutils/utils_verify/CMakeLists.txt +index abf9596f..dd9ef78b 100644 +--- a/test/cutils/utils_verify/CMakeLists.txt ++++ b/test/cutils/utils_verify/CMakeLists.txt +@@ -14,3 +14,4 @@ target_include_directories(${EXE} PUBLIC + ) + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/image/oci/oci_config_merge/CMakeLists.txt b/test/image/oci/oci_config_merge/CMakeLists.txt +index 88047fde..42cd2e78 100644 +--- a/test/image/oci/oci_config_merge/CMakeLists.txt ++++ b/test/image/oci/oci_config_merge/CMakeLists.txt +@@ -73,3 +73,4 @@ target_include_directories(${EXE} PUBLIC + set_target_properties(${EXE} PROPERTIES LINK_FLAGS "-Wl,--wrap,util_common_calloc_s -Wl,--wrap,util_smart_calloc_s -Wl,--wrap,merge_env") + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/image/oci/registry/CMakeLists.txt b/test/image/oci/registry/CMakeLists.txt +index 727a615a..13ed95b2 100644 +--- a/test/image/oci/registry/CMakeLists.txt ++++ b/test/image/oci/registry/CMakeLists.txt +@@ -63,3 +63,4 @@ target_include_directories(${EXE} PUBLIC + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lcrypto -lyajl -lz libhttpclient) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/image/oci/storage/images/CMakeLists.txt b/test/image/oci/storage/images/CMakeLists.txt +index b00c5a0e..3e6b69a4 100644 +--- a/test/image/oci/storage/images/CMakeLists.txt ++++ b/test/image/oci/storage/images/CMakeLists.txt +@@ -45,3 +45,4 @@ target_include_directories(${EXE} PUBLIC + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/image/oci/storage/layers/CMakeLists.txt b/test/image/oci/storage/layers/CMakeLists.txt +index 3fe8ab7c..952e9483 100644 +--- a/test/image/oci/storage/layers/CMakeLists.txt ++++ b/test/image/oci/storage/layers/CMakeLists.txt +@@ -66,6 +66,7 @@ target_link_libraries(${DRIVER_EXE} + -lwebsockets -lcrypto -lyajl -larchive ${SELINUX_LIBRARY} -ldevmapper -lz) + + add_test(NAME ${DRIVER_EXE} COMMAND ${DRIVER_EXE} --gtest_output=xml:${DRIVER_EXE}-Results.xml) ++set_tests_properties(${DRIVER_EXE} PROPERTIES TIMEOUT 120) + + # storage_layers_ut + SET(LAYER_EXE storage_layers_ut) +@@ -141,3 +142,4 @@ target_link_libraries(${LAYER_EXE} + -lwebsockets -lcrypto -lyajl -larchive ${SELINUX_LIBRARY} -ldevmapper -lz) + + add_test(NAME ${LAYER_EXE} COMMAND ${LAYER_EXE} --gtest_output=xml:${LAYER_EXE}-Results.xml) ++set_tests_properties(${LAYER_EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/image/oci/storage/rootfs/CMakeLists.txt b/test/image/oci/storage/rootfs/CMakeLists.txt +index 5bf568f9..4d7d3533 100644 +--- a/test/image/oci/storage/rootfs/CMakeLists.txt ++++ b/test/image/oci/storage/rootfs/CMakeLists.txt +@@ -45,3 +45,4 @@ target_include_directories(${EXE} PUBLIC + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/network/CMakeLists.txt b/test/network/CMakeLists.txt +index e354bebc..be31fd0e 100644 +--- a/test/network/CMakeLists.txt ++++ b/test/network/CMakeLists.txt +@@ -86,3 +86,4 @@ target_include_directories(${EXE} PUBLIC + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/runtime/isula/CMakeLists.txt b/test/runtime/isula/CMakeLists.txt +index 38a454b0..f5821953 100644 +--- a/test/runtime/isula/CMakeLists.txt ++++ b/test/runtime/isula/CMakeLists.txt +@@ -58,3 +58,4 @@ target_include_directories(${EXE} PUBLIC + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lpthread -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/runtime/lcr/CMakeLists.txt b/test/runtime/lcr/CMakeLists.txt +index 6f8f784c..979cbe5a 100644 +--- a/test/runtime/lcr/CMakeLists.txt ++++ b/test/runtime/lcr/CMakeLists.txt +@@ -58,3 +58,4 @@ target_include_directories(${EXE} PUBLIC + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/services/execution/execute/execution_extend/CMakeLists.txt b/test/services/execution/execute/execution_extend/CMakeLists.txt +index aaff39ef..8588fac3 100644 +--- a/test/services/execution/execute/execution_extend/CMakeLists.txt ++++ b/test/services/execution/execute/execution_extend/CMakeLists.txt +@@ -76,3 +76,4 @@ target_include_directories(${EXE} PUBLIC + ) + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/services/execution/spec/CMakeLists.txt b/test/services/execution/spec/CMakeLists.txt +index e1aa680e..d4a9d9b0 100644 +--- a/test/services/execution/spec/CMakeLists.txt ++++ b/test/services/execution/spec/CMakeLists.txt +@@ -74,4 +74,6 @@ target_include_directories(${MOCK_EXE} PUBLIC + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${SELINUX_LIBRARY} ${ISULA_LIBUTILS_LIBRARY} -lcrypto -lyajl -lz) + target_link_libraries(${MOCK_EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${SELINUX_LIBRARY} ${ISULA_LIBUTILS_LIBRARY} -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) + add_test(NAME ${MOCK_EXE} COMMAND ${MOCK_EXE} --gtest_output=xml:${MOCK_EXE}-Results.xml) ++set_tests_properties(${MOCK_EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/services/execution/spec/selinux_label_ut.cc b/test/services/execution/spec/selinux_label_ut.cc +index c0e9ab1c..f9a73cca 100644 +--- a/test/services/execution/spec/selinux_label_ut.cc ++++ b/test/services/execution/spec/selinux_label_ut.cc +@@ -179,11 +179,13 @@ protected: + void SetUp() override + { + CreateTestedObjects(); ++ selinux_state_init(); + } + + void TearDown() override + { + ClearTestedObjects(); ++ selinux_state_free(); + } + + private: +diff --git a/test/specs/specs/CMakeLists.txt b/test/specs/specs/CMakeLists.txt +index b730959c..c4b36c5f 100644 +--- a/test/specs/specs/CMakeLists.txt ++++ b/test/specs/specs/CMakeLists.txt +@@ -83,3 +83,4 @@ target_include_directories(${EXE} PUBLIC + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/specs/specs_extend/CMakeLists.txt b/test/specs/specs_extend/CMakeLists.txt +index 7d05deb4..06f46a37 100644 +--- a/test/specs/specs_extend/CMakeLists.txt ++++ b/test/specs/specs_extend/CMakeLists.txt +@@ -79,3 +79,4 @@ target_include_directories(${EXE} PUBLIC + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/volume/CMakeLists.txt b/test/volume/CMakeLists.txt +index 6eea3f76..e2045b19 100644 +--- a/test/volume/CMakeLists.txt ++++ b/test/volume/CMakeLists.txt +@@ -39,4 +39,5 @@ target_include_directories(${EXE} PUBLIC + ) + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lcrypto -lyajl -lz) +-add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) +\ No newline at end of file ++add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +-- +2.25.1 + diff --git a/0014-add-retry-for-read-write.patch b/0014-add-retry-for-read-write.patch new file mode 100644 index 0000000..c51a570 --- /dev/null +++ b/0014-add-retry-for-read-write.patch @@ -0,0 +1,360 @@ +From 734fca150e1c5da2814a55e0315bde8e828e6e8a Mon Sep 17 00:00:00 2001 +From: zhangxiaoyu <zhangxiaoyu58@huawei.com> +Date: Fri, 17 Feb 2023 16:07:53 +0800 +Subject: [PATCH 14/53] add retry for read/write + +Signed-off-by: zhangxiaoyu <zhangxiaoyu58@huawei.com> +--- + src/cmd/isulad-shim/common.c | 6 +++--- + src/cmd/isulad-shim/process.c | 2 +- + src/cmd/isulad/main.c | 4 ++-- + src/daemon/common/selinux_label.c | 2 +- + src/daemon/entry/connect/grpc/grpc_containers_service.cc | 9 ++++++--- + src/daemon/entry/cri/sysctl_tools.c | 2 +- + src/daemon/executor/container_cb/execution.c | 4 ++-- + .../modules/container/container_gc/containers_gc.c | 3 ++- + src/daemon/modules/events_sender/event_sender.c | 2 +- + src/daemon/modules/image/oci/storage/storage.c | 4 +++- + src/daemon/modules/log/log_gather.c | 6 +++--- + src/daemon/modules/plugin/plugin.c | 2 +- + src/daemon/modules/runtime/isula/isula_rt_ops.c | 4 ++-- + src/daemon/modules/service/io_handler.c | 2 +- + src/daemon/modules/service/service_container.c | 2 +- + src/utils/cutils/utils.c | 2 +- + src/utils/cutils/utils_aes.c | 2 +- + src/utils/cutils/utils_file.c | 2 +- + src/utils/tar/util_archive.c | 4 ++-- + src/utils/tar/util_gzip.c | 2 +- + 20 files changed, 36 insertions(+), 30 deletions(-) + +diff --git a/src/cmd/isulad-shim/common.c b/src/cmd/isulad-shim/common.c +index bb8464bb..0c345187 100644 +--- a/src/cmd/isulad-shim/common.c ++++ b/src/cmd/isulad-shim/common.c +@@ -196,7 +196,7 @@ int generate_random_str(char *id, size_t len) + } + for (i = 0; i < len; i++) { + int nret; +- if (read(fd, &num, sizeof(int)) < 0) { ++ if (read_nointr(fd, &num, sizeof(int)) < 0) { + close(fd); + return SHIM_ERR; + } +@@ -232,8 +232,8 @@ void write_message(int fd, const char *level, const char *fmt, ...) + va_end(arg_list); + + snprintf(msg, MAX_MESSAGE_LEN - 1, "{\"level\": \"%s\", \"msg\": \"%s\"}\n", level, buf); +- nwrite = write(fd, msg, strlen(msg)); +- if (nwrite != strlen(msg)) { ++ nwrite = write_nointr_in_total(fd, msg, strlen(msg)); ++ if (nwrite < 0 || (size_t)nwrite != strlen(msg)) { + return; + } + } +diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c +index 8a0aa142..02ce3c85 100644 +--- a/src/cmd/isulad-shim/process.c ++++ b/src/cmd/isulad-shim/process.c +@@ -283,7 +283,7 @@ static void *do_io_copy(void *data) + break; + } + +- int r_count = read(ioc->fd_from, buf, DEFAULT_IO_COPY_BUF); ++ int r_count = util_read_nointr(ioc->fd_from, buf, DEFAULT_IO_COPY_BUF); + if (r_count == -1) { + if (errno == EAGAIN || errno == EINTR) { + continue; +diff --git a/src/cmd/isulad/main.c b/src/cmd/isulad/main.c +index b17657c5..a75fb189 100644 +--- a/src/cmd/isulad/main.c ++++ b/src/cmd/isulad/main.c +@@ -482,8 +482,8 @@ int check_and_save_pid(const char *fn) + goto out; + } + +- len = (int)write(fd, pidbuf, strlen(pidbuf)); +- if (len < 0) { ++ len = util_write_nointr(fd, pidbuf, strlen(pidbuf)); ++ if (len < 0 || len != strlen(pidbuf)) { + ERROR("Failed to write pid to file:%s: %s", fn, strerror(errno)); + ret = -1; + } +diff --git a/src/daemon/common/selinux_label.c b/src/daemon/common/selinux_label.c +index 24294780..173f3acb 100644 +--- a/src/daemon/common/selinux_label.c ++++ b/src/daemon/common/selinux_label.c +@@ -310,7 +310,7 @@ static int get_random_value(unsigned int range, unsigned int *val) + return -1; + } + +- if (read(fd, &num, sizeof(int)) < 0) { ++ if (util_read_nointr(fd, &num, sizeof(int)) < 0) { + ERROR("Failed to read urandom value\n"); + ret = -1; + goto out; +diff --git a/src/daemon/entry/connect/grpc/grpc_containers_service.cc b/src/daemon/entry/connect/grpc/grpc_containers_service.cc +index c0210ed9..eb79223b 100644 +--- a/src/daemon/entry/connect/grpc/grpc_containers_service.cc ++++ b/src/daemon/entry/connect/grpc/grpc_containers_service.cc +@@ -292,7 +292,8 @@ Status ContainerServiceImpl::RemoteStart(ServerContext *context, + break; + } + const std::string &command = request.stdin(); +- if (write(read_pipe_fd[1], (void *)(command.c_str()), command.length()) < 0) { ++ int nret = util_write_nointr_in_total(read_pipe_fd[1], command.c_str(), command.length()); ++ if (nret < 0 || (size_t)nret != command.length()) { + ERROR("sub write over!"); + break; + } +@@ -407,7 +408,8 @@ public: + } + for (int i = 0; i < request.cmd_size(); i++) { + std::string command = request.cmd(i); +- if (write(m_read_pipe_fd, (void *)(command.c_str()), command.length()) < 0) { ++ int nret = util_write_nointr_in_total(m_read_pipe_fd, command.c_str(), command.length()); ++ if (nret < 0 || (size_t)nret != command.length()) { + ERROR("sub write over!"); + return; + } +@@ -629,7 +631,8 @@ Status ContainerServiceImpl::Attach(ServerContext *context, ServerReaderWriter<A + break; + } + std::string command = request.stdin(); +- if (write(pipefd[1], (void *)(command.c_str()), command.length()) < 0) { ++ int nret = util_write_nointr_in_total(pipefd[1], command.c_str(), command.length()); ++ if (nret < 0 || (size_t)nret != command.length()) { + ERROR("sub write over!"); + break; + } +diff --git a/src/daemon/entry/cri/sysctl_tools.c b/src/daemon/entry/cri/sysctl_tools.c +index 257ccf8f..3c558fa1 100644 +--- a/src/daemon/entry/cri/sysctl_tools.c ++++ b/src/daemon/entry/cri/sysctl_tools.c +@@ -99,7 +99,7 @@ int set_sysctl(const char *sysctl, int new_value, char **err) + goto free_out; + } + rsize = util_write_nointr(fd, buff, strlen(buff)); +- if (rsize <= 0) { ++ if (rsize < 0 || (size_t)rsize != strlen(buff)) { + if (asprintf(err, "Write new value failed: %s", strerror(errno)) < 0) { + *err = util_strdup_s("Out of memory"); + } +diff --git a/src/daemon/executor/container_cb/execution.c b/src/daemon/executor/container_cb/execution.c +index ed70fc14..198052d3 100644 +--- a/src/daemon/executor/container_cb/execution.c ++++ b/src/daemon/executor/container_cb/execution.c +@@ -348,7 +348,7 @@ static int maybe_create_cpu_realtime_file(int64_t value, const char *file, const + return -1; + } + nwrite = util_write_nointr(fd, buf, strlen(buf)); +- if (nwrite < 0 || nwrite != strlen(buf)) { ++ if (nwrite < 0 || (size_t)nwrite != strlen(buf)) { + ERROR("Failed to write %s to %s: %s", buf, fpath, strerror(errno)); + isulad_set_error_message("Failed to write '%s' to '%s': %s", buf, fpath, strerror(errno)); + return -1; +@@ -451,7 +451,7 @@ static int container_start_prepare(container_t *cont, const container_start_requ + + // init cgroup path for cpu_rt_runtime and cpu_rt_period + // we should do this in start container, not create container +- // because, in scenarios: ++ // because, in scenarios: + // 1. enable cpu-rt of isulad; + // 2. then run container with --cpu-rt-runtime + // 3. then reboot machine; +diff --git a/src/daemon/modules/container/container_gc/containers_gc.c b/src/daemon/modules/container/container_gc/containers_gc.c +index 8c858a96..9feb6d3c 100644 +--- a/src/daemon/modules/container/container_gc/containers_gc.c ++++ b/src/daemon/modules/container/container_gc/containers_gc.c +@@ -88,7 +88,8 @@ static int save_gc_config(const char *json_gc_config) + goto out; + } + +- if (write(fd, json_gc_config, strlen(json_gc_config)) == -1) { ++ nret = util_write_nointr(fd, json_gc_config, strlen(json_gc_config)); ++ if (nret < 0 || (size_t)nret != strlen(json_gc_config)) { + ERROR("write %s failed: %s", filename, strerror(errno)); + ret = -1; + } +diff --git a/src/daemon/modules/events_sender/event_sender.c b/src/daemon/modules/events_sender/event_sender.c +index 03dcbbf3..a3903f3e 100644 +--- a/src/daemon/modules/events_sender/event_sender.c ++++ b/src/daemon/modules/events_sender/event_sender.c +@@ -58,7 +58,7 @@ static void isulad_monitor_fifo_send(const struct monitord_msg *msg) + + do { + ret = util_write_nointr(fd, msg, sizeof(struct monitord_msg)); +- if (ret != sizeof(struct monitord_msg)) { ++ if (ret < 0 || (size_t)ret != sizeof(struct monitord_msg)) { + util_usleep_nointerupt(1000); + } + } while (ret != sizeof(struct monitord_msg)); +diff --git a/src/daemon/modules/image/oci/storage/storage.c b/src/daemon/modules/image/oci/storage/storage.c +index 829ea8d0..2f4bdf5f 100644 +--- a/src/daemon/modules/image/oci/storage/storage.c ++++ b/src/daemon/modules/image/oci/storage/storage.c +@@ -1429,6 +1429,7 @@ static int do_add_checked_layer(const char *lid, int fd, map_t *checked_layers) + bool default_value = true; + char buf[PATH_MAX] = { 0 }; + int ret = 0; ++ int nret; + + if (strlen(lid) >= PATH_MAX - 1) { + ERROR("Invalid layer id: %s", lid); +@@ -1438,7 +1439,8 @@ static int do_add_checked_layer(const char *lid, int fd, map_t *checked_layers) + (void)memcpy(buf, lid, strlen(lid)); + buf[strlen(lid)] = '\n'; + // save checked layer ids into file +- if (util_write_nointr(fd, buf, strlen(lid) + 1) < 0) { ++ nret = util_write_nointr(fd, buf, strlen(lid) + 1); ++ if (nret < 0 || (size_t)nret != strlen(lid) + 1) { + ERROR("Write checked layer data failed: %s", strerror(errno)); + ret = -1; + goto out; +diff --git a/src/daemon/modules/log/log_gather.c b/src/daemon/modules/log/log_gather.c +index 49facaa2..414c9ad1 100644 +--- a/src/daemon/modules/log/log_gather.c ++++ b/src/daemon/modules/log/log_gather.c +@@ -183,9 +183,9 @@ static int write_into_file(const void *buf, size_t g_log_size) + return -1; + } + } +- ret = (int)write(g_log_fd, buf, g_log_size); +- if (ret <= 0) { +- return ret; ++ ret = util_write_nointr_in_total(g_log_fd, buf, g_log_size); ++ if (ret < 0 || (size_t)ret != g_log_size) { ++ return -1; + } + + write_size += ret; +diff --git a/src/daemon/modules/plugin/plugin.c b/src/daemon/modules/plugin/plugin.c +index 53afeeaf..1c0af368 100644 +--- a/src/daemon/modules/plugin/plugin.c ++++ b/src/daemon/modules/plugin/plugin.c +@@ -618,7 +618,7 @@ static int process_plugin_events(int inotify_fd, const char *plugin_dir) + struct inotify_event *plugin_event = NULL; + char buffer[8192 + 1] = { 0 }; + int action = 0; +- events_length = read(inotify_fd, buffer, 8192); ++ events_length = util_read_nointr(inotify_fd, buffer, 8192); + + if (events_length <= 0) { + ERROR("Failed to wait events"); +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index 76e3bcb7..5463bb1b 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -1363,8 +1363,8 @@ int rt_isula_exec_resize(const char *id, const char *runtime, const rt_exec_resi + goto out; + } + +- count = write(fd, data, RESIZE_DATA_SIZE); +- if (count <= 0) { ++ count = util_write_nointr(fd, data, strlen(data)); ++ if (count < 0 || (size_t)count != strlen(data)) { + ERROR("write exec resize data error"); + ret = -1; + goto out; +diff --git a/src/daemon/modules/service/io_handler.c b/src/daemon/modules/service/io_handler.c +index 893733bc..98c763a4 100644 +--- a/src/daemon/modules/service/io_handler.c ++++ b/src/daemon/modules/service/io_handler.c +@@ -340,7 +340,7 @@ static ssize_t write_to_fd(void *context, const void *data, size_t len) + { + ssize_t ret; + ret = util_write_nointr(*(int *)context, data, len); +- if ((ret <= 0) || (ret != (ssize_t)len)) { ++ if (ret < 0 || (size_t)ret != len) { + ERROR("Failed to write: %s", strerror(errno)); + return -1; + } +diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c +index eeb035a0..cc777411 100644 +--- a/src/daemon/modules/service/service_container.c ++++ b/src/daemon/modules/service/service_container.c +@@ -345,7 +345,7 @@ static int write_env_content(const char *env_path, const char **env, size_t env_ + goto out; + } + nret = util_write_nointr(fd, env_content, strlen(env_content)); +- if (nret < 0 || nret != len - 1) { ++ if (nret < 0 || (size_t)nret != strlen(env_content)) { + SYSERROR("Write env file failed"); + free(env_content); + ret = -1; +diff --git a/src/utils/cutils/utils.c b/src/utils/cutils/utils.c +index de636bcb..f99b28e4 100644 +--- a/src/utils/cutils/utils.c ++++ b/src/utils/cutils/utils.c +@@ -1251,7 +1251,7 @@ int util_generate_random_str(char *id, size_t len) + } + for (i = 0; i < len; i++) { + int nret; +- if (read(fd, &num, sizeof(int)) < 0) { ++ if (util_read_nointr(fd, &num, sizeof(int)) < 0) { + ERROR("Failed to read urandom value"); + close(fd); + return -1; +diff --git a/src/utils/cutils/utils_aes.c b/src/utils/cutils/utils_aes.c +index 1e25ecd3..055a9538 100644 +--- a/src/utils/cutils/utils_aes.c ++++ b/src/utils/cutils/utils_aes.c +@@ -77,7 +77,7 @@ int util_aes_key(const char *key_file, bool create, unsigned char *aeskey) + goto out; + } + +- if (read(fd, aeskey, AES_256_CFB_KEY_LEN) != AES_256_CFB_KEY_LEN) { ++ if (util_read_nointr(fd, aeskey, AES_256_CFB_KEY_LEN) != AES_256_CFB_KEY_LEN) { + ERROR("read key file %s failed: %s", key_file, strerror(errno)); + ret = -1; + goto out; +diff --git a/src/utils/cutils/utils_file.c b/src/utils/cutils/utils_file.c +index cdd712a7..34c5b060 100644 +--- a/src/utils/cutils/utils_file.c ++++ b/src/utils/cutils/utils_file.c +@@ -998,7 +998,7 @@ int util_file2str(const char *filename, char *buf, size_t len) + if (fd == -1) { + return -1; + } +- num_read = (int)read(fd, buf, len - 1); ++ num_read = (int)util_read_nointr(fd, buf, len - 1); + if (num_read <= 0) { + num_read = -1; + } else { +diff --git a/src/utils/tar/util_archive.c b/src/utils/tar/util_archive.c +index 2d56d8a7..7ace2924 100644 +--- a/src/utils/tar/util_archive.c ++++ b/src/utils/tar/util_archive.c +@@ -662,7 +662,7 @@ child_out: + if (ret != 0) { + ERROR("Wait archive_untar_handler failed with error:%s", strerror(errno)); + fcntl(pipe_stderr[0], F_SETFL, O_NONBLOCK); +- if (read(pipe_stderr[0], errbuf, BUFSIZ) < 0) { ++ if (util_read_nointr(pipe_stderr[0], errbuf, BUFSIZ) < 0) { + ERROR("read error message from child failed"); + } + } +@@ -1057,7 +1057,7 @@ child_out: + if (ret != 0) { + ERROR("tar failed"); + fcntl(pipe_for_read[0], F_SETFL, O_NONBLOCK); +- if (read(pipe_for_read[0], errbuf, BUFSIZ) < 0) { ++ if (util_read_nointr(pipe_for_read[0], errbuf, BUFSIZ) < 0) { + ERROR("read error message from child failed"); + } + } +diff --git a/src/utils/tar/util_gzip.c b/src/utils/tar/util_gzip.c +index 5c34d719..2f4750be 100644 +--- a/src/utils/tar/util_gzip.c ++++ b/src/utils/tar/util_gzip.c +@@ -212,7 +212,7 @@ int gzip(const char *filename, size_t len) + return -1; + } + +- size_read = read(pipefd[0], buffer, BUFSIZ); ++ size_read = util_read_nointr(pipefd[0], buffer, BUFSIZ); + close(pipefd[0]); + + if (size_read) { +-- +2.25.1 + diff --git a/0015-support-pull-image-with-digest.patch b/0015-support-pull-image-with-digest.patch new file mode 100644 index 0000000..5650701 --- /dev/null +++ b/0015-support-pull-image-with-digest.patch @@ -0,0 +1,302 @@ +From aaf8dec80eff5390404d7da66dbb229e44c76b12 Mon Sep 17 00:00:00 2001 +From: haozi007 <liuhao27@huawei.com> +Date: Thu, 16 Feb 2023 18:22:02 +0800 +Subject: [PATCH 15/53] support pull image with digest + +usage: isula pull busybox@sha256:907ca53d7e2947e849b839b1cd258c98fd3916c60f2e6e70c30edbf741ab6754 + +Signed-off-by: haozi007 <liuhao27@huawei.com> +--- + src/daemon/executor/image_cb/image_cb.c | 8 ++++ + src/daemon/modules/image/oci/oci_pull.c | 23 ++++++---- + .../modules/image/oci/registry/registry.c | 2 +- + .../oci/storage/image_store/image_store.c | 7 +++ + src/daemon/modules/image/oci/utils_images.c | 45 +++++++++++++++---- + src/daemon/modules/image/oci/utils_images.h | 2 + + src/utils/cutils/utils_verify.c | 25 ++++++++--- + src/utils/cutils/utils_verify.h | 3 ++ + 8 files changed, 91 insertions(+), 24 deletions(-) + +diff --git a/src/daemon/executor/image_cb/image_cb.c b/src/daemon/executor/image_cb/image_cb.c +index 06de7543..124feb21 100644 +--- a/src/daemon/executor/image_cb/image_cb.c ++++ b/src/daemon/executor/image_cb/image_cb.c +@@ -561,6 +561,14 @@ static int trans_one_image(image_list_images_response *response, size_t image_in + out_image->name = util_strdup_s(im_image->repo_tags[repo_index]); + } + ++ if (out_image->name == NULL && im_image->repo_digests != NULL && im_image->repo_digests_len > 0) { ++ // repo digest must valid, so just get lastest @ ++ char *pod = strrchr(im_image->repo_digests[0], '@'); ++ if (pod != NULL) { ++ out_image->name = util_sub_string(im_image->repo_digests[0], 0, (size_t)(pod - im_image->repo_digests[0])); ++ } ++ } ++ + out_image->target = util_common_calloc_s(sizeof(image_descriptor)); + if (out_image->target == NULL) { + ERROR("Out of memory"); +diff --git a/src/daemon/modules/image/oci/oci_pull.c b/src/daemon/modules/image/oci/oci_pull.c +index 5e774c9e..5b35ca2b 100644 +--- a/src/daemon/modules/image/oci/oci_pull.c ++++ b/src/daemon/modules/image/oci/oci_pull.c +@@ -117,10 +117,19 @@ static int pull_image(const im_pull_request *request, char **name) + options->skip_tls_verify = oci_image_data->insecure_skip_verify_enforce; + insecure_registries = oci_image_data->insecure_registries; + ++ // key of image which save in image-store ++ options->dest_image_name = oci_normalize_image_name(request->image); ++ ++ // add default tag if required ++ with_tag = oci_default_tag(request->image); ++ + host = oci_get_host(request->image); + if (host != NULL) { +- options->image_name = oci_default_tag(request->image); +- options->dest_image_name = oci_normalize_image_name(request->image); ++ // 1. image_name use for split host/tag/name ++ // 2. user for tag of log ++ options->image_name = with_tag; ++ with_tag = NULL; ++ + update_option_insecure_registry(options, insecure_registries, host); + ret = registry_pull(options); + if (ret != 0) { +@@ -141,13 +150,12 @@ static int pull_image(const im_pull_request *request, char **name) + } + host = oci_host_from_mirror(*mirror); + update_option_insecure_registry(options, insecure_registries, host); +- with_tag = oci_default_tag(request->image); ++ // add current mirror to image name ++ free(options->image_name); + options->image_name = oci_add_host(host, with_tag); +- free(with_tag); +- with_tag = NULL; + free(host); + host = NULL; +- options->dest_image_name = oci_normalize_image_name(request->image); ++ + ret = registry_pull(options); + if (ret != 0) { + continue; +@@ -159,10 +167,9 @@ static int pull_image(const im_pull_request *request, char **name) + *name = util_strdup_s(options->dest_image_name); + + out: ++ free(with_tag); + free(host); +- host = NULL; + free_registry_pull_options(options); +- options = NULL; + + return ret; + } +diff --git a/src/daemon/modules/image/oci/registry/registry.c b/src/daemon/modules/image/oci/registry/registry.c +index 143de6e4..62d0c35e 100644 +--- a/src/daemon/modules/image/oci/registry/registry.c ++++ b/src/daemon/modules/image/oci/registry/registry.c +@@ -1861,7 +1861,7 @@ static int prepare_pull_desc(pull_descriptor *desc, registry_pull_options *optio + } + + if (!util_valid_image_name(options->dest_image_name)) { +- ERROR("Invalid dest image name %s", options->image_name); ++ ERROR("Invalid dest image name %s", options->dest_image_name); + isulad_try_set_error_message("Invalid image name"); + return -1; + } +diff --git a/src/daemon/modules/image/oci/storage/image_store/image_store.c b/src/daemon/modules/image/oci/storage/image_store/image_store.c +index 39bda87d..cf1e88ff 100644 +--- a/src/daemon/modules/image/oci/storage/image_store/image_store.c ++++ b/src/daemon/modules/image/oci/storage/image_store/image_store.c +@@ -1979,6 +1979,7 @@ static int resort_image_names(const char **names, size_t names_len, char **first + MAX_IMAGE_NAME_LENGTH - MAX_IMAGE_DIGEST_LENGTH); + } + ++ // TODO: maybe should support other digest + if (prefix != NULL && strcmp(prefix, DIGEST_PREFIX) == 0) { + if (util_array_append(image_digests, names[i]) != 0) { + ERROR("Failed to append image to digest: %s", names[i]); +@@ -2172,6 +2173,7 @@ static int get_image_repo_digests(char ***old_repo_digests, char **image_tags, i + goto out; + } + ++ // get repo digest from images which with tag + if (pack_repo_digest(old_repo_digests, (const char **)image_tags, digest, repo_digests) != 0) { + ERROR("Failed to pack repo digest"); + ret = -1; +@@ -2194,12 +2196,17 @@ static int pack_image_tags_and_repo_digest(image_t *img, imagetool_image *info) + char *image_digest = NULL; + char **repo_digests = NULL; + ++ // get names from image-store names: ++ // 1. image names with tag; ++ // 2. image names with digests; ++ // 3. get first image name, current unused; + if (resort_image_names((const char **)img->simage->names, img->simage->names_len, &name, &tags, &digests) != 0) { + ERROR("Failed to resort image names"); + ret = -1; + goto out; + } + ++ // update repo digests from tags + if (get_image_repo_digests(&digests, tags, img, &image_digest, &repo_digests) != 0) { + ERROR("Failed to get image repo digests"); + ret = -1; +diff --git a/src/daemon/modules/image/oci/utils_images.c b/src/daemon/modules/image/oci/utils_images.c +index 9e7bb16f..ad7fe0f4 100644 +--- a/src/daemon/modules/image/oci/utils_images.c ++++ b/src/daemon/modules/image/oci/utils_images.c +@@ -42,6 +42,26 @@ + // nanos of 2038-01-19T03:14:07, the max valid linux time + #define MAX_NANOS 2147483647000000000 + ++char *oci_image_digest_pos(const char *name) ++{ ++ char *pos = NULL; ++ ++ if (name == NULL) { ++ return NULL; ++ } ++ ++ pos = strrchr(name, '@'); ++ if (pos == NULL) { ++ return NULL; ++ } ++ ++ if (util_reg_match(__DIGESTPattern, pos) != 0) { ++ return NULL; ++ } ++ ++ return pos; ++} ++ + char *get_last_part(char **parts) + { + char *last_part = NULL; +@@ -98,6 +118,7 @@ char *oci_default_tag(const char *name) + } + + last_part = get_last_part(parts); ++ // will pass image name with digest and with tag + if (last_part != NULL && strrchr(last_part, ':') == NULL) { + add_default_tag = DEFAULT_TAG; + } +@@ -181,9 +202,9 @@ char *oci_normalize_image_name(const char *name) + return result; + } + +-int oci_split_image_name(const char *image_name, char **host, char **name, char **tag) ++int oci_split_image_name(const char *image_name, char **host, char **name, char **tag_digest) + { +- char *tag_pos = NULL; ++ char *tag_digest_pos = NULL; + char *name_pos = NULL; + char *tmp_image_name = NULL; + +@@ -193,18 +214,24 @@ int oci_split_image_name(const char *image_name, char **host, char **name, char + } + + tmp_image_name = util_strdup_s(image_name); +- tag_pos = util_tag_pos(tmp_image_name); +- if (tag_pos != NULL) { +- *tag_pos = 0; +- tag_pos++; +- if (tag != NULL) { +- *tag = util_strdup_s(tag_pos); ++ ++ // check digest first ++ tag_digest_pos = oci_image_digest_pos(tmp_image_name); ++ if (tag_digest_pos == NULL) { ++ tag_digest_pos = util_tag_pos(tmp_image_name); ++ } ++ ++ if (tag_digest_pos != NULL) { ++ *tag_digest_pos = '\0'; ++ tag_digest_pos++; ++ if (tag_digest != NULL) { ++ *tag_digest = util_strdup_s(tag_digest_pos); + } + } + + name_pos = strchr(tmp_image_name, '/'); + if (name_pos != NULL) { +- *name_pos = 0; ++ *name_pos = '\0'; + name_pos++; + if (name != NULL) { + *name = util_strdup_s(name_pos); +diff --git a/src/daemon/modules/image/oci/utils_images.h b/src/daemon/modules/image/oci/utils_images.h +index daa8c040..97879e41 100644 +--- a/src/daemon/modules/image/oci/utils_images.h ++++ b/src/daemon/modules/image/oci/utils_images.h +@@ -59,6 +59,8 @@ char *oci_get_isulad_tmpdir(const char *root_dir); + int makesure_isulad_tmpdir_perm_right(const char *root_dir); + char *get_hostname_to_strip(); + ++char *oci_image_digest_pos(const char *name); ++ + #ifdef __cplusplus + } + #endif +diff --git a/src/utils/cutils/utils_verify.c b/src/utils/cutils/utils_verify.c +index d39d8da5..5868e890 100644 +--- a/src/utils/cutils/utils_verify.c ++++ b/src/utils/cutils/utils_verify.c +@@ -359,7 +359,7 @@ cleanup: + bool util_valid_image_name(const char *name) + { + char *copy = NULL; +- char *tag_pos = NULL; ++ char *check_pos = NULL; + bool bret = false; + + if (name == NULL) { +@@ -372,13 +372,26 @@ bool util_valid_image_name(const char *name) + } + + copy = util_strdup_s(name); +- tag_pos = util_tag_pos(copy); +- if (tag_pos != NULL) { +- if (util_reg_match(__TagPattern, tag_pos)) { ++ ++ // 1. first, check digest or not ++ check_pos = strrchr(copy, '@'); ++ if (check_pos != NULL) { ++ // image name with digest ++ if (util_reg_match(__DIGESTPattern, check_pos)) { + goto cleanup; + } +- +- *tag_pos = '\0'; ++ *check_pos = '\0'; ++ } else { ++ // image name without digest ++ // 2. check tag or not ++ check_pos = util_tag_pos(copy); ++ if (check_pos != NULL) { ++ if (util_reg_match(__TagPattern, check_pos)) { ++ goto cleanup; ++ } ++ ++ *check_pos = '\0'; ++ } + } + + if (util_reg_match(__NamePattern, copy)) { +diff --git a/src/utils/cutils/utils_verify.h b/src/utils/cutils/utils_verify.h +index a885250f..ad4466ef 100644 +--- a/src/utils/cutils/utils_verify.h ++++ b/src/utils/cutils/utils_verify.h +@@ -33,6 +33,9 @@ extern "C" { + "^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])" \ + "((\\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9]))+)?(:[0-9]+)?/)?[a-z0-9]" \ + "+((([._]|__|[-]*)[a-z0-9]+)+)?((/[a-z0-9]+((([._]|__|[-]*)[a-z0-9]+)+)?)+)?$" ++ ++#define __DIGESTPattern "@[a-z0-9]+:[a-z0-9]{32,}" ++ + #define VALID_VOLUME_NAME "[a-zA-Z0-9][a-zA-Z0-9_.-]{1,63}" + + extern const char *g_all_caps[]; +-- +2.25.1 + diff --git a/0016-isulad-shim-support-execSync-with-timeout.patch b/0016-isulad-shim-support-execSync-with-timeout.patch new file mode 100644 index 0000000..a4e8c98 --- /dev/null +++ b/0016-isulad-shim-support-execSync-with-timeout.patch @@ -0,0 +1,186 @@ +From 32dbf764fd5b7f6941c49750b49dbba253bd3234 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Mon, 13 Feb 2023 15:36:58 +0800 +Subject: [PATCH 16/53] isulad-shim support execSync with timeout + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + .../executor/container_cb/execution_stream.c | 2 +- + .../modules/runtime/isula/isula_rt_ops.c | 37 +++++++++++++++--- + src/utils/cutils/utils.c | 39 +++++++++++++++++++ + src/utils/cutils/utils.h | 5 +++ + 4 files changed, 77 insertions(+), 6 deletions(-) + +diff --git a/src/daemon/executor/container_cb/execution_stream.c b/src/daemon/executor/container_cb/execution_stream.c +index fde0335e..1a7353b5 100644 +--- a/src/daemon/executor/container_cb/execution_stream.c ++++ b/src/daemon/executor/container_cb/execution_stream.c +@@ -161,7 +161,7 @@ static int container_exec_cb(const container_exec_request *request, container_ex + + if (exec_container(cont, request, *response, stdinfd, stdout_handler, stderr_handler) != 0) { + ret = -1; +- goto out; ++ goto pack_err_response; + } + + goto out; +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index 5463bb1b..6f2b4f7d 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -205,6 +205,10 @@ static void show_shim_runtime_errlog(const char *workdir) + char buf1[SHIM_LOG_SIZE] = { 0 }; + char buf2[SHIM_LOG_SIZE] = { 0 }; + ++ if (g_isulad_errmsg != NULL) { ++ return; ++ } ++ + get_err_message(buf1, sizeof(buf1), workdir, "shim-log.json"); + get_err_message(buf2, sizeof(buf2), workdir, "log.json"); + ERROR("shim-log: %s", buf1); +@@ -686,8 +690,29 @@ static int status_to_exit_code(int status) + return exit_code; + } + ++static int try_wait_pid(pid_t pid) ++{ ++ if (waitpid(pid, NULL, WNOHANG) == pid) { ++ return 0; ++ } ++ ++ return 1; ++} ++ ++static void kill_and_show_err(pid_t pid) ++{ ++ int nret = 0; ++ kill(pid, SIGKILL); ++ // wait atmost 0.5 seconds ++ DO_RETRY_CALL(5, 100000, nret, try_wait_pid, pid); ++ if (nret != 0) { ++ WARN("Fail to wait isulad-shim"); ++ } ++ isulad_set_error_message("Exec container error;exec timeout"); ++} ++ + static int shim_create(bool fg, const char *id, const char *workdir, const char *bundle, const char *runtime_cmd, +- int *exit_code) ++ int *exit_code, const int64_t timeout) + { + pid_t pid = 0; + int exec_fd[2] = { -1, -1 }; +@@ -778,7 +803,7 @@ realexec: + goto out; + } + +- status = util_wait_for_pid_status(pid); ++ status = util_waitpid_with_timeout(pid, timeout, kill_and_show_err); + if (status < 0) { + ERROR("failed wait shim-parent %d exit %s", pid, strerror(errno)); + ret = -1; +@@ -792,7 +817,9 @@ realexec: + out: + if (ret != 0) { + show_shim_runtime_errlog(workdir); +- kill(pid, SIGKILL); /* can kill other process? */ ++ if (timeout <= 0) { ++ kill(pid, SIGKILL); /* can kill other process? */ ++ } + } + + return ret; +@@ -901,7 +928,7 @@ int rt_isula_create(const char *id, const char *runtime, const rt_create_params_ + } + + get_runtime_cmd(runtime, &cmd); +- ret = shim_create(false, id, workdir, params->bundle, cmd, NULL); ++ ret = shim_create(false, id, workdir, params->bundle, cmd, NULL, -1); + if (ret != 0) { + runtime_call_delete_force(workdir, runtime, id); + ERROR("%s: failed create shim process", id); +@@ -1173,7 +1200,7 @@ int rt_isula_exec(const char *id, const char *runtime, const rt_exec_params_t *p + } + + get_runtime_cmd(runtime, &cmd); +- ret = shim_create(fg_exec(params), id, workdir, bundle, cmd, exit_code); ++ ret = shim_create(fg_exec(params), id, workdir, bundle, cmd, exit_code, params->timeout); + if (ret != 0) { + ERROR("%s: failed create shim process for exec %s", id, exec_id); + goto errlog_out; +diff --git a/src/utils/cutils/utils.c b/src/utils/cutils/utils.c +index f99b28e4..2c4c01e4 100644 +--- a/src/utils/cutils/utils.c ++++ b/src/utils/cutils/utils.c +@@ -311,6 +311,45 @@ rep: + return 0; + } + ++/* ++ * If timeout <= 0, blocking wait pid. ++ * If timeout > 0, non-blocking wait pid with timeout. ++ * When waitpid timeout, calling handle_timeout_callback_t. ++ */ ++int util_waitpid_with_timeout(pid_t pid, const int64_t timeout, handle_timeout_callback_t cb) ++{ ++ int nret = 0; ++ time_t start_time = time(NULL); ++ time_t end_time; ++ double interval; ++ ++ if (timeout <= 0) { ++ return util_wait_for_pid_status(pid); ++ } ++ ++ for (;;) { ++ nret = waitpid(pid, NULL, WNOHANG); ++ if (nret == pid) { ++ break; ++ } ++ if (nret == -1 && errno != EINTR) { ++ return -1; ++ } ++ end_time = time(NULL); ++ interval = difftime(end_time, start_time); ++ if (nret == 0 && interval >= timeout) { ++ INFO("Wait %d timeout", pid); ++ if (cb != NULL) { ++ cb(pid); ++ } ++ return -1; ++ } ++ // sleep some time instead to avoid cpu full running and then retry. ++ sleep(0.1); ++ } ++ return 0; ++} ++ + int util_wait_for_pid_status(pid_t pid) + { + int st; +diff --git a/src/utils/cutils/utils.h b/src/utils/cutils/utils.h +index 6261dc05..01107605 100644 +--- a/src/utils/cutils/utils.h ++++ b/src/utils/cutils/utils.h +@@ -302,6 +302,9 @@ typedef struct _proc_t { + processor; /* current (or most recent?) CPU */ + } proc_t; + ++// handle waitpid timeout. ++typedef void(*handle_timeout_callback_t)(pid_t pid); ++ + struct signame { + int num; + const char *name; +@@ -329,6 +332,8 @@ char *util_strdup_s(const char *src); + + int util_wait_for_pid(pid_t pid); + ++int util_waitpid_with_timeout(pid_t pid, const int64_t timeout, handle_timeout_callback_t cb); ++ + void util_contain_errmsg(const char *errmsg, int *exit_code); + + char *util_short_digest(const char *digest); +-- +2.25.1 + diff --git a/0017-Refine-the-commit-info.patch b/0017-Refine-the-commit-info.patch new file mode 100644 index 0000000..2b46810 --- /dev/null +++ b/0017-Refine-the-commit-info.patch @@ -0,0 +1,30 @@ +From a1c06194fea99d1011551fd84b1fb1f28b974170 Mon Sep 17 00:00:00 2001 +From: sailorvii <chenw66@chinaunicom.cn> +Date: Tue, 21 Feb 2023 02:40:50 +0000 +Subject: [PATCH 17/53] Refine the commit info. + +--- + docs/design/detailed/Network/native_network_design_zh.md | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/docs/design/detailed/Network/native_network_design_zh.md b/docs/design/detailed/Network/native_network_design_zh.md +index 0ca0f850..27c10c3e 100644 +--- a/docs/design/detailed/Network/native_network_design_zh.md ++++ b/docs/design/detailed/Network/native_network_design_zh.md +@@ -299,10 +299,10 @@ cache:::unFinish + + ## 4.2 adaptor模块 + +-1. 查看 CRI adapter 模块的设计文档: [CRI_adapter_design](./cni_operator_design_zh.md) 。 ++1. 查看 CRI adapter 模块的设计文档: [CRI_adapter_design](./CRI_adapter_design_zh.md) 。 + + 2. 查看 native network adapter 模块的设计文档: [native_network_adapter_design](./native_network_adapter_design_zh.md) 。 + + ## 4.3 cni-operator模块 + +-- 查看 cni operator 模块的设计文档: [cni_operator_design](./cni_operator_design_zh.md) 。 +\ No newline at end of file ++- 查看 cni operator 模块的设计文档: [cni_operator_design](./cni_operator_design_zh.md) 。 +-- +2.25.1 + diff --git a/0018-Refine-typo-of-word-container.patch b/0018-Refine-typo-of-word-container.patch new file mode 100644 index 0000000..34f8397 --- /dev/null +++ b/0018-Refine-typo-of-word-container.patch @@ -0,0 +1,85 @@ +From 53ec87b8c5224b1069bef50d09403c53fb48640f Mon Sep 17 00:00:00 2001 +From: sailorvii <chenw66@chinaunicom.cn> +Date: Tue, 21 Feb 2023 06:50:21 +0000 +Subject: [PATCH 18/53] Refine typo of word "container". + +--- + src/daemon/executor/container_cb/execution_create.c | 2 +- + src/daemon/modules/api/container_api.h | 2 +- + src/daemon/modules/container/container_unix.c | 2 +- + src/daemon/modules/service/service_network.c | 6 +++--- + 4 files changed, 6 insertions(+), 6 deletions(-) + +diff --git a/src/daemon/executor/container_cb/execution_create.c b/src/daemon/executor/container_cb/execution_create.c +index 4abc89c7..e8f74f1b 100644 +--- a/src/daemon/executor/container_cb/execution_create.c ++++ b/src/daemon/executor/container_cb/execution_create.c +@@ -1481,7 +1481,7 @@ int container_create_cb(const container_create_request *request, container_creat + goto umount_channel; + } + +- if (container_v2_spec_merge_contaner_spec(v2_spec) != 0) { ++ if (container_v2_spec_merge_container_spec(v2_spec) != 0) { + ERROR("Failed to merge container settings"); + cc = ISULAD_ERR_EXEC; + goto umount_channel; +diff --git a/src/daemon/modules/api/container_api.h b/src/daemon/modules/api/container_api.h +index 270d6da6..1511db78 100644 +--- a/src/daemon/modules/api/container_api.h ++++ b/src/daemon/modules/api/container_api.h +@@ -175,7 +175,7 @@ void container_unlock(container_t *cont); + + char *container_get_env_nolock(const container_t *cont, const char *key); + +-int container_v2_spec_merge_contaner_spec(container_config_v2_common_config *v2_spec); ++int container_v2_spec_merge_container_spec(container_config_v2_common_config *v2_spec); + + char *container_get_command(const container_t *cont); + +diff --git a/src/daemon/modules/container/container_unix.c b/src/daemon/modules/container/container_unix.c +index adc11be7..1a252b92 100644 +--- a/src/daemon/modules/container/container_unix.c ++++ b/src/daemon/modules/container/container_unix.c +@@ -470,7 +470,7 @@ out: + } + + /* container merge basic v2 spec info */ +-int container_v2_spec_merge_contaner_spec(container_config_v2_common_config *v2_spec) ++int container_v2_spec_merge_container_spec(container_config_v2_common_config *v2_spec) + { + int ret = 0; + int i = 0; +diff --git a/src/daemon/modules/service/service_network.c b/src/daemon/modules/service/service_network.c +index 2e7fa28c..2d5f2f6e 100644 +--- a/src/daemon/modules/service/service_network.c ++++ b/src/daemon/modules/service/service_network.c +@@ -569,7 +569,7 @@ err_out: + return NULL; + } + +-static container_network_settings *dup_contaner_network_settings(const container_network_settings *settings) ++static container_network_settings *dup_container_network_settings(const container_network_settings *settings) + { + char *jstr = NULL; + container_network_settings *res = NULL; +@@ -1278,7 +1278,7 @@ static int update_container_network_settings(container_t *cont, const cni_anno_p + bool to_disk = false; + container_network_settings *backup = NULL; + +- backup = dup_contaner_network_settings(cont->network_settings); ++ backup = dup_container_network_settings(cont->network_settings); + if (backup == NULL) { + ERROR("Failed to dup container network settings"); + return -1; +@@ -1509,7 +1509,7 @@ static int drop_container_network_settings(container_t *cont) + return -1; + } + +- backup = dup_contaner_network_settings(cont->network_settings); ++ backup = dup_container_network_settings(cont->network_settings); + if (backup == NULL) { + ERROR("Failed to dup container network settings"); + return -1; +-- +2.25.1 + diff --git a/0019-cleancode-for-read-write.patch b/0019-cleancode-for-read-write.patch new file mode 100644 index 0000000..3fb9e1a --- /dev/null +++ b/0019-cleancode-for-read-write.patch @@ -0,0 +1,54 @@ +From f9224d47ddc4193678f7ffe501be144fedff0102 Mon Sep 17 00:00:00 2001 +From: zhangxiaoyu <zhangxiaoyu58@huawei.com> +Date: Mon, 20 Feb 2023 17:28:33 +0800 +Subject: [PATCH 19/53] cleancode for read/write + +Signed-off-by: zhangxiaoyu <zhangxiaoyu58@huawei.com> +--- + src/cmd/isulad-shim/process.c | 2 +- + src/cmd/isulad/main.c | 2 +- + src/daemon/entry/connect/grpc/grpc_containers_service.cc | 2 +- + 3 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c +index 02ce3c85..8a0aa142 100644 +--- a/src/cmd/isulad-shim/process.c ++++ b/src/cmd/isulad-shim/process.c +@@ -283,7 +283,7 @@ static void *do_io_copy(void *data) + break; + } + +- int r_count = util_read_nointr(ioc->fd_from, buf, DEFAULT_IO_COPY_BUF); ++ int r_count = read(ioc->fd_from, buf, DEFAULT_IO_COPY_BUF); + if (r_count == -1) { + if (errno == EAGAIN || errno == EINTR) { + continue; +diff --git a/src/cmd/isulad/main.c b/src/cmd/isulad/main.c +index a75fb189..0cdbfb53 100644 +--- a/src/cmd/isulad/main.c ++++ b/src/cmd/isulad/main.c +@@ -483,7 +483,7 @@ int check_and_save_pid(const char *fn) + } + + len = util_write_nointr(fd, pidbuf, strlen(pidbuf)); +- if (len < 0 || len != strlen(pidbuf)) { ++ if (len < 0 || (size_t)len != strlen(pidbuf)) { + ERROR("Failed to write pid to file:%s: %s", fn, strerror(errno)); + ret = -1; + } +diff --git a/src/daemon/entry/connect/grpc/grpc_containers_service.cc b/src/daemon/entry/connect/grpc/grpc_containers_service.cc +index eb79223b..7340c3ed 100644 +--- a/src/daemon/entry/connect/grpc/grpc_containers_service.cc ++++ b/src/daemon/entry/connect/grpc/grpc_containers_service.cc +@@ -409,7 +409,7 @@ public: + for (int i = 0; i < request.cmd_size(); i++) { + std::string command = request.cmd(i); + int nret = util_write_nointr_in_total(m_read_pipe_fd, command.c_str(), command.length()); +- if (nret < 0 || (size_t)nret != command.length()) { ++ if (nret < 0 || (size_t)nret != command.length()) { + ERROR("sub write over!"); + return; + } +-- +2.25.1 + diff --git a/0020-add-crictl-timeout-and-sync-for-CI.patch b/0020-add-crictl-timeout-and-sync-for-CI.patch new file mode 100644 index 0000000..dbab1fd --- /dev/null +++ b/0020-add-crictl-timeout-and-sync-for-CI.patch @@ -0,0 +1,127 @@ +From 7941e0fcd8d7b8edb303a1661233fd9688c46819 Mon Sep 17 00:00:00 2001 +From: zhangxiaoyu <zhangxiaoyu58@huawei.com> +Date: Mon, 20 Feb 2023 15:42:40 +0800 +Subject: [PATCH 20/53] add crictl timeout and sync for CI + +Signed-off-by: zhangxiaoyu <zhangxiaoyu58@huawei.com> +--- + CI/test_cases/container_cases/bigdata_stream.sh | 7 +++++++ + CI/test_cases/container_cases/bigdata_stream_runc.sh | 4 ++++ + CI/test_cases/helpers.sh | 5 ++++- + 3 files changed, 15 insertions(+), 1 deletion(-) + +diff --git a/CI/test_cases/container_cases/bigdata_stream.sh b/CI/test_cases/container_cases/bigdata_stream.sh +index 7e74d700..3bfc2d50 100755 +--- a/CI/test_cases/container_cases/bigdata_stream.sh ++++ b/CI/test_cases/container_cases/bigdata_stream.sh +@@ -124,6 +124,7 @@ function test_concurrent_bigdata_stream() + pids[${#pids[@]}]=$! + done + wait ${pids[*]// /|} ++ sync && sync + + for index in $(seq 1 5); do + ls -l /home/iocopy_stream_data_500M_$index +@@ -151,6 +152,7 @@ function test_concurrent_bigdata_stream_without_pty() + pids[${#pids[@]}]=$! + done + wait ${pids[*]// /|} ++ sync && sync + + for index in $(seq 1 5); do + ls -l /home/iocopy_stream_data_500M_$index +@@ -209,6 +211,7 @@ function test_stream_with_stop_client() + kill -18 $pid + + wait $pid ++ sync && sync + + ls -l /home/iocopy_stream_data_500M + total_size=$(stat -c"%s" /home/iocopy_stream_data_500M) +@@ -254,6 +257,7 @@ function test_stream_with_stop_attach() + kill -18 $pid + + wait $exec_pid ++ sync && sync + + ls -l /home/iocopy_stream_data_500M + total_size=$(stat -c"%s" /home/iocopy_stream_data_500M) +@@ -299,6 +303,7 @@ function test_stream_with_stop_lxc_monitor() + kill -18 $pid + + wait $exec_pid ++ sync && sync + + ls -l /home/iocopy_stream_data_500M + total_size=$(stat -c"%s" /home/iocopy_stream_data_500M) +@@ -347,6 +352,7 @@ function test_stream_with_stop_isulad() + kill -18 $(cat /var/run/isulad.pid) + + wait $pid ++ sync && sync + + ls -l /home/iocopy_stream_data_500M + total_size=$(stat -c"%s" /home/iocopy_stream_data_500M) +@@ -397,6 +403,7 @@ function test_stream_with_runc() + isula exec -it $RUNCID cat test_500M > /home/iocopy_stream_data_500M + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to cat bigdata" && ((ret++)) + ++ sync && sync + total_size=$(stat -c"%s" /home/iocopy_stream_data_500M) + [[ $total_size -ne 524288000 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stream iocopy loss data" && ((ret++)) + +diff --git a/CI/test_cases/container_cases/bigdata_stream_runc.sh b/CI/test_cases/container_cases/bigdata_stream_runc.sh +index 6933bef0..3a384cd8 100755 +--- a/CI/test_cases/container_cases/bigdata_stream_runc.sh ++++ b/CI/test_cases/container_cases/bigdata_stream_runc.sh +@@ -117,6 +117,7 @@ function test_cat_bigdata() + pids[${#pids[@]}]=$! + done + wait ${pids[*]// /|} ++ sync && sync + + for index in $(seq 1 5); do + ls -l /home/iocopy_stream_data_500M_$index +@@ -144,6 +145,7 @@ function test_cat_bigdata_without_pty() + pids[${#pids[@]}]=$! + done + wait ${pids[*]// /|} ++ sync && sync + + for index in $(seq 1 5); do + ls -l /home/iocopy_stream_data_500M_$index +@@ -173,6 +175,7 @@ function test_stream_with_stop_client() + kill -18 $pid + + wait $pid ++ sync && sync + + ls -l /home/iocopy_stream_data_500M + total_size=$(stat -c"%s" /home/iocopy_stream_data_500M) +@@ -217,6 +220,7 @@ function test_stream_with_stop_isulad() + kill -18 $(cat /var/run/isulad.pid) + + wait $pid ++ sync && sync + + ls -l /home/iocopy_stream_data_500M + total_size=$(stat -c"%s" /home/iocopy_stream_data_500M) +diff --git a/CI/test_cases/helpers.sh b/CI/test_cases/helpers.sh +index bba4e7e3..5ea4ff94 100755 +--- a/CI/test_cases/helpers.sh ++++ b/CI/test_cases/helpers.sh +@@ -69,7 +69,10 @@ function testcontainer() { + + function crictl() { + CRICTL=$(which crictl) +- "$CRICTL" -i unix:///var/run/isulad.sock -r unix:///var/run/isulad.sock "$@" ++ # Default timeout is 2s. ++ # In some high IO testcase, isulad handle CRI request time maybe more than 2s. ++ # And the crictl will print error message "context deadline exceeded" ++ "$CRICTL" -i unix:///var/run/isulad.sock -r unix:///var/run/isulad.sock --timeout 5s "$@" + } + + function msg_ok() +-- +2.25.1 + diff --git a/0021-unlock-m_podsLock-if-new-failed.patch b/0021-unlock-m_podsLock-if-new-failed.patch new file mode 100644 index 0000000..c0dc084 --- /dev/null +++ b/0021-unlock-m_podsLock-if-new-failed.patch @@ -0,0 +1,27 @@ +From f6243bb672bca8fd2e32752480aa92dc8f97adc9 Mon Sep 17 00:00:00 2001 +From: zhangxiaoyu <zhangxiaoyu58@huawei.com> +Date: Wed, 22 Feb 2023 10:43:52 +0800 +Subject: [PATCH 21/53] unlock m_podsLock if new failed + +Signed-off-by: zhangxiaoyu <zhangxiaoyu58@huawei.com> +--- + src/daemon/entry/cri/network_plugin.cc | 3 +++ + 1 file changed, 3 insertions(+) + +diff --git a/src/daemon/entry/cri/network_plugin.cc b/src/daemon/entry/cri/network_plugin.cc +index f6a155c3..4a119d6b 100644 +--- a/src/daemon/entry/cri/network_plugin.cc ++++ b/src/daemon/entry/cri/network_plugin.cc +@@ -345,6 +345,9 @@ void PluginManager::Lock(const std::string &fullPodName, Errors &error) + auto tmpLock = std::unique_ptr<PodLock>(new (std::nothrow) PodLock()); + if (tmpLock == nullptr) { + error.SetError("Out of memory"); ++ if (pthread_mutex_unlock(&m_podsLock) != 0) { ++ error.SetError("plugin manager unlock failed"); ++ } + return; + } + lock = tmpLock.get(); +-- +2.25.1 + diff --git a/0022-Update-CRI.patch b/0022-Update-CRI.patch new file mode 100644 index 0000000..713338e --- /dev/null +++ b/0022-Update-CRI.patch @@ -0,0 +1,73 @@ +From 65c3b3c803128f92113f9f21bf41da1ad56017c8 Mon Sep 17 00:00:00 2001 +From: shijiaqi1 <jiaqi@isrc.iscas.ac.cn> +Date: Wed, 8 Feb 2023 13:31:36 +0800 +Subject: [PATCH 22/53] Update-CRI + +--- + .../cri/cri_container_manager_service.cc | 19 +++++++++++++++++++ + src/daemon/entry/cri/cri_helpers.cc | 19 +++++++++++++++++++ + 2 files changed, 38 insertions(+) + +diff --git a/src/daemon/entry/cri/cri_container_manager_service.cc b/src/daemon/entry/cri/cri_container_manager_service.cc +index 710556a3..b02367c8 100644 +--- a/src/daemon/entry/cri/cri_container_manager_service.cc ++++ b/src/daemon/entry/cri/cri_container_manager_service.cc +@@ -1179,6 +1179,25 @@ void ContainerManagerService::UpdateContainerResources(const std::string &contai + if (!resources.cpuset_mems().empty()) { + hostconfig->cpuset_mems = util_strdup_s(resources.cpuset_mems().c_str()); + } ++ if (resources.hugepage_limits_size() != 0) { ++ hostconfig->hugetlbs = (host_config_hugetlbs_element **)util_smart_calloc_s( ++ sizeof(host_config_hugetlbs_element *), resources.hugepage_limits_size()); ++ if (hostconfig->hugetlbs == nullptr) { ++ error.SetError("Out of memory"); ++ return; ++ } ++ for (int i = 0; i < resources.hugepage_limits_size(); i++) { ++ hostconfig->hugetlbs[i] = ++ (host_config_hugetlbs_element *)util_common_calloc_s(sizeof(host_config_hugetlbs_element)); ++ if (hostconfig->hugetlbs[i] == nullptr) { ++ error.SetError("Out of memory"); ++ goto cleanup; ++ } ++ hostconfig->hugetlbs[i]->page_size = util_strdup_s(resources.hugepage_limits(i).page_size().c_str()); ++ hostconfig->hugetlbs[i]->limit = resources.hugepage_limits(i).limit(); ++ hostconfig->hugetlbs_len++; ++ } ++ } + + request->host_config = host_config_generate_json(hostconfig, &ctx, &perror); + if (request->host_config == nullptr) { +diff --git a/src/daemon/entry/cri/cri_helpers.cc b/src/daemon/entry/cri/cri_helpers.cc +index 2f6dcf78..6d59ec11 100644 +--- a/src/daemon/entry/cri/cri_helpers.cc ++++ b/src/daemon/entry/cri/cri_helpers.cc +@@ -461,6 +461,25 @@ void UpdateCreateConfig(container_config *createConfig, host_config *hc, + } + } + hc->unified = unified; ++ if (rOpts.hugepage_limits_size() != 0) { ++ hc->hugetlbs = (host_config_hugetlbs_element **)util_smart_calloc_s(sizeof(host_config_hugetlbs_element *), ++ rOpts.hugepage_limits_size()); ++ if (hc->hugetlbs == nullptr) { ++ error.SetError("Out of memory"); ++ return; ++ } ++ for (int i = 0; i < rOpts.hugepage_limits_size(); i++) { ++ hc->hugetlbs[i] = ++ (host_config_hugetlbs_element *)util_common_calloc_s(sizeof(host_config_hugetlbs_element)); ++ if (hc->hugetlbs[i] == nullptr) { ++ error.SetError("Out of memory"); ++ return; ++ } ++ hc->hugetlbs[i]->page_size = util_strdup_s(rOpts.hugepage_limits(i).page_size().c_str()); ++ hc->hugetlbs[i]->limit = rOpts.hugepage_limits(i).limit(); ++ hc->hugetlbs_len++; ++ } ++ } + } + createConfig->open_stdin = config.stdin(); + createConfig->tty = config.tty(); +-- +2.25.1 + diff --git a/0023-add-cgroup-cpu-ut.patch b/0023-add-cgroup-cpu-ut.patch new file mode 100644 index 0000000..9cafd62 --- /dev/null +++ b/0023-add-cgroup-cpu-ut.patch @@ -0,0 +1,70 @@ +From dd238f80fd754b135c7b0e6a6535d45cf57c0b82 Mon Sep 17 00:00:00 2001 +From: songbuhuang <544824346@qq.com> +Date: Wed, 22 Feb 2023 16:46:45 +0800 +Subject: [PATCH 23/53] add cgroup cpu ut + +Signed-off-by: songbuhuang <544824346@qq.com> +--- + test/cgroup/cpu/CMakeLists.txt | 2 ++ + test/cgroup/cpu/cgroup_cpu_ut.cc | 21 +++++++++++++++++++++ + 2 files changed, 23 insertions(+) + +diff --git a/test/cgroup/cpu/CMakeLists.txt b/test/cgroup/cpu/CMakeLists.txt +index 6a8af719..b619fb59 100644 +--- a/test/cgroup/cpu/CMakeLists.txt ++++ b/test/cgroup/cpu/CMakeLists.txt +@@ -22,8 +22,10 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/config + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../mocks + ) + ++set_target_properties(${EXE} PROPERTIES LINK_FLAGS "-Wl,--wrap,util_common_calloc_s") + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) + set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cgroup/cpu/cgroup_cpu_ut.cc b/test/cgroup/cpu/cgroup_cpu_ut.cc +index 032ad656..7d23d014 100644 +--- a/test/cgroup/cpu/cgroup_cpu_ut.cc ++++ b/test/cgroup/cpu/cgroup_cpu_ut.cc +@@ -16,10 +16,18 @@ + #include <stdlib.h> + #include <stdio.h> + #include <gtest/gtest.h> ++#include <gmock/gmock.h> + #include "daemon_arguments.h" + #include "isulad_config.h" ++#include "mock.h" ++#include "sysinfo.h" + #include "utils.h" + ++extern "C" { ++ DECLARE_WRAPPER(util_common_calloc_s, void *, (size_t size)); ++ DEFINE_WRAPPER(util_common_calloc_s, void *, (size_t size), (size)); ++} ++ + struct service_arguments *new_args(int64_t cpu_rt_period, int64_t cpu_rt_runtime) + { + struct service_arguments *args = (struct service_arguments *)util_common_calloc_s(sizeof(struct service_arguments)); +@@ -57,4 +65,17 @@ TEST(CgroupCpuUnitTest, test_conf_get_cgroup_cpu_rt) + ASSERT_EQ(cpu_rt_runtime, 0); + } + ++TEST(CgroupCpuUnitTest, test_find_cgroup_mountpoint_and_root) ++{ ++ char *mnt = NULL; ++ char *root = NULL; ++ ASSERT_EQ(find_cgroup_mountpoint_and_root(nullptr, &mnt, &root), -1); ++} + ++TEST(CgroupCpuUnitTest, test_sysinfo_cgroup_controller_cpurt_mnt_path) ++{ ++ MOCK_SET(util_common_calloc_s, nullptr); ++ ASSERT_EQ(get_sys_info(true), nullptr); ++ ASSERT_EQ(sysinfo_cgroup_controller_cpurt_mnt_path(), nullptr); ++ MOCK_CLEAR(util_common_calloc_s); ++} +-- +2.25.1 + diff --git a/0024-remove-temp-variables.patch b/0024-remove-temp-variables.patch new file mode 100644 index 0000000..3faeba9 --- /dev/null +++ b/0024-remove-temp-variables.patch @@ -0,0 +1,179 @@ +From e20767fc709bb945eb8e076a57f07d5b70b5f3ab Mon Sep 17 00:00:00 2001 +From: "ilya.kuksenok" <ilya.kuksenok@huawei.com> +Date: Wed, 22 Feb 2023 16:17:34 +0300 +Subject: [PATCH 24/53] remove temp variables + +--- + src/api/services/containers/container.proto | 5 +++++ + .../connect/grpc/grpc_containers_client.cc | 5 +++++ + src/client/connect/protocol_type.h | 5 +++++ + .../connect/rest/rest_containers_client.c | 5 +++++ + .../connect/grpc/container/stats_service.cc | 6 ++++++ + .../entry/cri/cri_container_manager_service.cc | 18 ++++++++++++++++-- + .../executor/container_cb/execution_extend.c | 5 +++++ + src/daemon/modules/api/runtime_api.h | 5 +++++ + .../modules/runtime/engines/lcr/lcr_engine.c | 6 ++++++ + 9 files changed, 58 insertions(+), 2 deletions(-) + +diff --git a/src/api/services/containers/container.proto b/src/api/services/containers/container.proto +index 13d1367b..230d18b3 100644 +--- a/src/api/services/containers/container.proto ++++ b/src/api/services/containers/container.proto +@@ -86,6 +86,11 @@ message Container_info { + uint64 cache = 16; + uint64 cache_total = 17; + uint64 inactive_file_total = 18; ++ uint64 avaliable_bytes = 19; ++ uint64 usage_bytes = 20; ++ uint64 rss_bytes = 21; ++ uint64 page_faults = 22; ++ uint64 major_page_faults = 23; + } + + message Event { +diff --git a/src/client/connect/grpc/grpc_containers_client.cc b/src/client/connect/grpc/grpc_containers_client.cc +index 1528b2ee..20766049 100644 +--- a/src/client/connect/grpc/grpc_containers_client.cc ++++ b/src/client/connect/grpc/grpc_containers_client.cc +@@ -1690,6 +1690,11 @@ public: + response->container_stats[i].blkio_read = gresponse->containers(i).blkio_read(); + response->container_stats[i].blkio_write = gresponse->containers(i).blkio_write(); + response->container_stats[i].mem_used = gresponse->containers(i).mem_used(); ++ response->container_stats[i].avaliable_bytes = gresponse->containers(i).avaliable_bytes(); ++ response->container_stats[i].usage_bytes = gresponse->containers(i).usage_bytes(); ++ response->container_stats[i].rss_bytes = gresponse->containers(i).rss_bytes(); ++ response->container_stats[i].page_faults = gresponse->containers(i).page_faults(); ++ response->container_stats[i].major_page_faults = gresponse->containers(i).major_page_faults(); + response->container_stats[i].mem_limit = gresponse->containers(i).mem_limit(); + response->container_stats[i].kmem_used = gresponse->containers(i).kmem_used(); + response->container_stats[i].kmem_limit = gresponse->containers(i).kmem_limit(); +diff --git a/src/client/connect/protocol_type.h b/src/client/connect/protocol_type.h +index 3831c7c0..a2d474a6 100644 +--- a/src/client/connect/protocol_type.h ++++ b/src/client/connect/protocol_type.h +@@ -165,6 +165,11 @@ struct isula_container_info { + // Memory usage + uint64_t mem_used; + uint64_t mem_limit; ++ uint64_t avaliable_bytes; ++ uint64_t usage_bytes; ++ uint64_t rss_bytes; ++ uint64_t page_faults; ++ uint64_t major_page_faults; + // Kernel Memory usage + uint64_t kmem_used; + uint64_t kmem_limit; +diff --git a/src/client/connect/rest/rest_containers_client.c b/src/client/connect/rest/rest_containers_client.c +index 0a735a64..9eee9d40 100644 +--- a/src/client/connect/rest/rest_containers_client.c ++++ b/src/client/connect/rest/rest_containers_client.c +@@ -2195,6 +2195,11 @@ static int unpack_container_info_for_stats_response(const container_stats_respon + // memory usage + infos[i].mem_used = con_info->mem_used; + infos[i].mem_limit = con_info->mem_limit; ++ infos[i].avaliable_bytes = con_info->avaliable_bytes(); ++ infos[i].usage_bytes = con_info->usage_bytes(); ++ infos[i].rss_bytes = con_info->rss_bytes(); ++ infos[i].page_faults = con_info->page_faults(); ++ infos[i].major_page_faults = con_info->major_page_faults(); + // kernel memory usage + infos[i].kmem_used = con_info->kmem_used; + infos[i].kmem_limit = con_info->kmem_limit; +diff --git a/src/daemon/entry/connect/grpc/container/stats_service.cc b/src/daemon/entry/connect/grpc/container/stats_service.cc +index 6e116b4c..7315fe75 100644 +--- a/src/daemon/entry/connect/grpc/container/stats_service.cc ++++ b/src/daemon/entry/connect/grpc/container/stats_service.cc +@@ -87,6 +87,12 @@ void ContainerStatsService::FillResponseTogRPC(void *containerRes, StatsResponse + stats->set_mem_limit(response->container_stats[i]->mem_limit); + stats->set_kmem_used(response->container_stats[i]->kmem_used); + stats->set_kmem_limit(response->container_stats[i]->kmem_limit); ++ stats->set_avaliable_bytes(response->container_stats[i]->avaliable_bytes); ++ stats->set_usage_bytes(response->container_stats[i]->usage_bytes); ++ stats->set_mem_used(response->container_stats[i]->mem_used); ++ stats->set_rss_bytes(response->container_stats[i]->rss_bytes); ++ stats->set_page_faults(response->container_stats[i]->page_faults); ++ stats->set_major_page_faults(response->container_stats[i]->major_page_faults); + if (response->container_stats[i]->name != nullptr) { + stats->set_name(response->container_stats[i]->name); + } +diff --git a/src/daemon/entry/cri/cri_container_manager_service.cc b/src/daemon/entry/cri/cri_container_manager_service.cc +index b02367c8..57284593 100644 +--- a/src/daemon/entry/cri/cri_container_manager_service.cc ++++ b/src/daemon/entry/cri/cri_container_manager_service.cc +@@ -842,8 +842,22 @@ void ContainerManagerService::ContainerStatsToGRPC( + response->container_stats[i]->cpu_use_nanos); + container->mutable_cpu()->set_timestamp(timestamp); + } +- +- containerstats->push_back(move(container)); ++ if (response->container_stats[i]->avaliable_bytes != 0u) { ++ container->mutable_memory()->mutable_available_bytes()->set_value(response->container_stats[i]->avaliable_bytes); ++ } ++ if (response->container_stats[i]->usage_bytes != 0u) { ++ container->mutable_memory()->mutable_usage_bytes()->set_value(response->container_stats[i]->usage_bytes); ++ } ++ if (response->container_stats[i]->rss_bytes != 0u) { ++ container->mutable_memory()->mutable_rss_bytes()->set_value(response->container_stats[i]->rss_bytes); ++ } ++ if (response->container_stats[i]->page_faults != 0u) { ++ container->mutable_memory()->mutable_page_faults()->set_value(response->container_stats[i]->page_faults); ++ } ++ if (response->container_stats[i]->major_page_faults != 0u) { ++ container->mutable_memory()->mutable_major_page_faults()->set_value(response->container_stats[i]->major_page_faults); ++ } ++ containerstats->push_back(std::move(container)); + } + } + +diff --git a/src/daemon/executor/container_cb/execution_extend.c b/src/daemon/executor/container_cb/execution_extend.c +index b0da705e..67d0845a 100644 +--- a/src/daemon/executor/container_cb/execution_extend.c ++++ b/src/daemon/executor/container_cb/execution_extend.c +@@ -243,6 +243,11 @@ static container_info *get_container_stats(const container_t *cont, + info->blkio_write = einfo->blkio_write; + info->mem_used = einfo->mem_used; + info->mem_limit = einfo->mem_limit; ++ info->avaliable_bytes = einfo->avaliable_bytes; ++ info->usage_bytes = einfo->usage_bytes; ++ info->rss_bytes = einfo->rss_bytes; ++ info->page_faults = einfo->page_faults; ++ info->major_page_faults = einfo->major_page_faults; + info->kmem_used = einfo->kmem_used; + info->kmem_limit = einfo->kmem_limit; + +diff --git a/src/daemon/modules/api/runtime_api.h b/src/daemon/modules/api/runtime_api.h +index de4136a3..b245ebf9 100644 +--- a/src/daemon/modules/api/runtime_api.h ++++ b/src/daemon/modules/api/runtime_api.h +@@ -55,6 +55,11 @@ struct runtime_container_resources_stats_info { + /* Memory usage */ + uint64_t mem_used; + uint64_t mem_limit; ++ uint64_t avaliable_bytes; ++ uint64_t usage_bytes; ++ uint64_t rss_bytes; ++ uint64_t page_faults; ++ uint64_t major_page_faults; + /* Kernel Memory usage */ + uint64_t kmem_used; + uint64_t kmem_limit; +diff --git a/src/daemon/modules/runtime/engines/lcr/lcr_engine.c b/src/daemon/modules/runtime/engines/lcr/lcr_engine.c +index 2ca12545..2f7f73f8 100644 +--- a/src/daemon/modules/runtime/engines/lcr/lcr_engine.c ++++ b/src/daemon/modules/runtime/engines/lcr/lcr_engine.c +@@ -162,6 +162,12 @@ static void copy_container_resources_stats(const struct lcr_container_state *lcs + rs_stats->mem_limit = lcs->mem_limit; + rs_stats->kmem_used = lcs->kmem_used; + rs_stats->kmem_limit = lcs->kmem_limit; ++ rs_stats->avaliable_bytes = lcs->avaliable_bytes; ++ rs_stats->usage_bytes = lcs->usage_bytes; ++ rs_stats->mem_used = lcs->mem_used; ++ rs_stats->rss_bytes = lcs->rss_bytes; ++ rs_stats->page_faults = lcs->page_faults; ++ rs_stats->major_page_faults = lcs->major_page_faults; + rs_stats->cache = lcs->cache; + rs_stats->cache_total = lcs->cache_total; + rs_stats->inactive_file_total = lcs->inactive_file_total; +-- +2.25.1 + diff --git a/0025-fix-read-member-error-from-struct.patch b/0025-fix-read-member-error-from-struct.patch new file mode 100644 index 0000000..041cd12 --- /dev/null +++ b/0025-fix-read-member-error-from-struct.patch @@ -0,0 +1,34 @@ +From 3e71fe5910a1b0c9dbb899e06614103a2cebf0fc Mon Sep 17 00:00:00 2001 +From: haozi007 <liuhao27@huawei.com> +Date: Fri, 24 Feb 2023 10:25:46 +0800 +Subject: [PATCH 25/53] fix read member error from struct + +Signed-off-by: haozi007 <liuhao27@huawei.com> +--- + src/client/connect/rest/rest_containers_client.c | 10 +++++----- + 1 file changed, 5 insertions(+), 5 deletions(-) + +diff --git a/src/client/connect/rest/rest_containers_client.c b/src/client/connect/rest/rest_containers_client.c +index 9eee9d40..0ae9a4f6 100644 +--- a/src/client/connect/rest/rest_containers_client.c ++++ b/src/client/connect/rest/rest_containers_client.c +@@ -2195,11 +2195,11 @@ static int unpack_container_info_for_stats_response(const container_stats_respon + // memory usage + infos[i].mem_used = con_info->mem_used; + infos[i].mem_limit = con_info->mem_limit; +- infos[i].avaliable_bytes = con_info->avaliable_bytes(); +- infos[i].usage_bytes = con_info->usage_bytes(); +- infos[i].rss_bytes = con_info->rss_bytes(); +- infos[i].page_faults = con_info->page_faults(); +- infos[i].major_page_faults = con_info->major_page_faults(); ++ infos[i].avaliable_bytes = con_info->avaliable_bytes; ++ infos[i].usage_bytes = con_info->usage_bytes; ++ infos[i].rss_bytes = con_info->rss_bytes; ++ infos[i].page_faults = con_info->page_faults; ++ infos[i].major_page_faults = con_info->major_page_faults; + // kernel memory usage + infos[i].kmem_used = con_info->kmem_used; + infos[i].kmem_limit = con_info->kmem_limit; +-- +2.25.1 + diff --git a/0026-Fix-PR-runc.patch b/0026-Fix-PR-runc.patch new file mode 100644 index 0000000..0467a7c --- /dev/null +++ b/0026-Fix-PR-runc.patch @@ -0,0 +1,108 @@ +From 39d7fd140e8b590a925e5cdf8ace20b0161328c8 Mon Sep 17 00:00:00 2001 +From: sailorvii <chenw66@chinaunicom.cn> +Date: Mon, 27 Feb 2023 02:33:46 +0000 +Subject: [PATCH 26/53] =?UTF-8?q?Fix=20PR=20runc=E8=BF=90=E8=A1=8C?= + =?UTF-8?q?=E5=A4=B1=E8=B4=A5.?= +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +The environment NOTIFY_SOCKET is used when runc start but not runc create. +As the source code: https://github.com/opencontainers/runc/blob/main/start.go#L35. + +So move the related code to the right location. +--- + .../modules/runtime/isula/isula_rt_ops.c | 27 +++++++++---------- + 1 file changed, 13 insertions(+), 14 deletions(-) + +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index 6f2b4f7d..60742d42 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -760,12 +760,6 @@ static int shim_create(bool fg, const char *id, const char *workdir, const char + goto realexec; + } + +- // clear NOTIFY_SOCKET from the env to adapt runc create +- if (unsetenv("NOTIFY_SOCKET") != 0) { +- (void)dprintf(exec_fd[1], "%s: unset env NOTIFY_SOCKET failed %s", id, strerror(errno)); +- exit(EXIT_FAILURE); +- } +- + pid = fork(); + if (pid < 0) { + (void)dprintf(exec_fd[1], "%s: fork shim-process failed %s", id, strerror(errno)); +@@ -945,7 +939,7 @@ int rt_isula_start(const char *id, const char *runtime, const rt_start_params_t + char shim_pid_file_name[PATH_MAX] = { 0 }; + pid_t pid = 0; + pid_t shim_pid = -1; +- int ret = 0; ++ int ret = -1; + int splice_ret = 0; + proc_t *proc = NULL; + proc_t *p_proc = NULL; +@@ -967,28 +961,24 @@ int rt_isula_start(const char *id, const char *runtime, const rt_start_params_t + + pid = get_container_process_pid(workdir); + if (pid < 0) { +- ret = -1; + ERROR("%s: failed wait init pid", id); + goto out; + } + + file_read_int(shim_pid_file_name, &shim_pid); + if (shim_pid < 0) { +- ret = -1; + ERROR("%s: failed to read isulad shim pid", id); + goto out; + } + + proc = util_get_process_proc_info(pid); + if (proc == NULL) { +- ret = -1; + ERROR("%s: failed to read pidinfo", id); + goto out; + } + + p_proc = util_get_process_proc_info(shim_pid); + if (p_proc == NULL) { +- ret = -1; + ERROR("%s: failed to read isulad shim pidinfo", id); + goto out; + } +@@ -998,20 +988,29 @@ int rt_isula_start(const char *id, const char *runtime, const rt_start_params_t + pid_info->ppid = shim_pid; + pid_info->pstart_time = p_proc->start_time; + ++ // clear NOTIFY_SOCKET from the env to adapt runc start ++ if (unsetenv("NOTIFY_SOCKET") != 0) { ++ ERROR("%s: unset env NOTIFY_SOCKET failed %s", id); ++ } ++ + if (runtime_call_simple(workdir, runtime, "start", NULL, 0, id, NULL) != 0) { + ERROR("call runtime start id failed"); +- ret = -1; + goto out; + } + ++ ret = 0; + out: + if (ret != 0) { + show_shim_runtime_errlog(workdir); + shim_kill_force(workdir); + } + +- free(proc); +- free(p_proc); ++ if (proc != NULL) { ++ free(proc); ++ } ++ if (p_proc != NULL) { ++ free(p_proc); ++ } + + return ret; + } +-- +2.25.1 + diff --git a/0027-allow-the-paused-container-to-be-stopped.patch b/0027-allow-the-paused-container-to-be-stopped.patch new file mode 100644 index 0000000..8fe9aa1 --- /dev/null +++ b/0027-allow-the-paused-container-to-be-stopped.patch @@ -0,0 +1,68 @@ +From 70173556a3825870b44cc344e83cdf0fd6d577c5 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Thu, 23 Feb 2023 16:08:48 +0800 +Subject: [PATCH 27/53] allow the paused container to be stopped + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/daemon/entry/cri/cri_container_manager_service.cc | 6 +++--- + src/daemon/entry/cri/cri_helpers.cc | 2 +- + src/daemon/modules/service/service_container.c | 7 +++---- + 3 files changed, 7 insertions(+), 8 deletions(-) + +diff --git a/src/daemon/entry/cri/cri_container_manager_service.cc b/src/daemon/entry/cri/cri_container_manager_service.cc +index 57284593..1592c0a6 100644 +--- a/src/daemon/entry/cri/cri_container_manager_service.cc ++++ b/src/daemon/entry/cri/cri_container_manager_service.cc +@@ -1195,14 +1195,14 @@ void ContainerManagerService::UpdateContainerResources(const std::string &contai + } + if (resources.hugepage_limits_size() != 0) { + hostconfig->hugetlbs = (host_config_hugetlbs_element **)util_smart_calloc_s( +- sizeof(host_config_hugetlbs_element *), resources.hugepage_limits_size()); ++ sizeof(host_config_hugetlbs_element *), resources.hugepage_limits_size()); + if (hostconfig->hugetlbs == nullptr) { + error.SetError("Out of memory"); + return; + } +- for (int i = 0; i < resources.hugepage_limits_size(); i++) { ++ for (int i = 0; i < resources.hugepage_limits_size(); i++) { + hostconfig->hugetlbs[i] = +- (host_config_hugetlbs_element *)util_common_calloc_s(sizeof(host_config_hugetlbs_element)); ++ (host_config_hugetlbs_element *)util_common_calloc_s(sizeof(host_config_hugetlbs_element)); + if (hostconfig->hugetlbs[i] == nullptr) { + error.SetError("Out of memory"); + goto cleanup; +diff --git a/src/daemon/entry/cri/cri_helpers.cc b/src/daemon/entry/cri/cri_helpers.cc +index 6d59ec11..c24c8b73 100644 +--- a/src/daemon/entry/cri/cri_helpers.cc ++++ b/src/daemon/entry/cri/cri_helpers.cc +@@ -470,7 +470,7 @@ void UpdateCreateConfig(container_config *createConfig, host_config *hc, + } + for (int i = 0; i < rOpts.hugepage_limits_size(); i++) { + hc->hugetlbs[i] = +- (host_config_hugetlbs_element *)util_common_calloc_s(sizeof(host_config_hugetlbs_element)); ++ (host_config_hugetlbs_element *)util_common_calloc_s(sizeof(host_config_hugetlbs_element)); + if (hc->hugetlbs[i] == nullptr) { + error.SetError("Out of memory"); + return; +diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c +index cc777411..d69ee757 100644 +--- a/src/daemon/modules/service/service_container.c ++++ b/src/daemon/modules/service/service_container.c +@@ -1495,10 +1495,9 @@ int stop_container(container_t *cont, int timeout, bool force, bool restart) + + container_lock(cont); + +- if (container_is_paused(cont->state)) { +- ERROR("Container %s is paused. Unpause the container before stopping or killing", id); +- isulad_set_error_message("Container %s is paused. Unpause the container before stopping or killing", id); +- ret = -1; ++ if (!container_is_running(cont->state)) { ++ INFO("Container %s is already stopped", id); ++ ret = 0; + goto out; + } + +-- +2.25.1 + diff --git a/0028-Refine.patch b/0028-Refine.patch new file mode 100644 index 0000000..b8fb739 --- /dev/null +++ b/0028-Refine.patch @@ -0,0 +1,29 @@ +From 376677f2f38b3c27b14b7a21aa021ea683a2f0e1 Mon Sep 17 00:00:00 2001 +From: sailorvii <chenw66@chinaunicom.cn> +Date: Tue, 28 Feb 2023 06:09:48 +0000 +Subject: [PATCH 28/53] Refine. + +--- + src/daemon/modules/runtime/isula/isula_rt_ops.c | 6 ++++++ + 1 file changed, 6 insertions(+) + +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index 60742d42..5d622515 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -760,6 +760,12 @@ static int shim_create(bool fg, const char *id, const char *workdir, const char + goto realexec; + } + ++ // clear NOTIFY_SOCKET from the env to adapt runc create ++ if (unsetenv("NOTIFY_SOCKET") != 0) { ++ (void)dprintf(exec_fd[1], "%s: unset env NOTIFY_SOCKET failed %s", id, strerror(errno)); ++ exit(EXIT_FAILURE); ++ } ++ + pid = fork(); + if (pid < 0) { + (void)dprintf(exec_fd[1], "%s: fork shim-process failed %s", id, strerror(errno)); +-- +2.25.1 + diff --git a/0029-support-isula-update-when-runtime-is-runc.patch b/0029-support-isula-update-when-runtime-is-runc.patch new file mode 100644 index 0000000..aad54f0 --- /dev/null +++ b/0029-support-isula-update-when-runtime-is-runc.patch @@ -0,0 +1,291 @@ +From 7f00006ea65378e7b27049ff3f0eb3fa70e69b09 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Thu, 16 Feb 2023 20:20:54 +0800 +Subject: [PATCH 29/53] support isula update when runtime is runc + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + .../executor/container_cb/execution_extend.c | 1 + + src/daemon/modules/api/runtime_api.h | 1 + + .../modules/runtime/isula/isula_rt_ops.c | 180 ++++++++++++++++-- + 3 files changed, 161 insertions(+), 21 deletions(-) + +diff --git a/src/daemon/executor/container_cb/execution_extend.c b/src/daemon/executor/container_cb/execution_extend.c +index 67d0845a..2f565d78 100644 +--- a/src/daemon/executor/container_cb/execution_extend.c ++++ b/src/daemon/executor/container_cb/execution_extend.c +@@ -1134,6 +1134,7 @@ static int do_update_resources(const container_update_request *request, containe + if (container_is_running(cont->state)) { + params.rootpath = cont->root_path; + params.hostconfig = hostconfig; ++ params.state = cont->state_path; + if (runtime_update(id, cont->runtime, ¶ms)) { + ERROR("Update container %s failed", id); + ret = -1; +diff --git a/src/daemon/modules/api/runtime_api.h b/src/daemon/modules/api/runtime_api.h +index b245ebf9..199c9f4b 100644 +--- a/src/daemon/modules/api/runtime_api.h ++++ b/src/daemon/modules/api/runtime_api.h +@@ -169,6 +169,7 @@ typedef struct _rt_attach_params_t { + typedef struct _rt_update_params_t { + const char *rootpath; + const host_config *hostconfig; ++ const char *state; + } rt_update_params_t; + + typedef struct _rt_listpids_params_t { +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index 6f2b4f7d..41791388 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -21,6 +21,7 @@ + #include <limits.h> + #include <errno.h> + #include <fcntl.h> ++#include <isula_libutils/auto_cleanup.h> + #include <isula_libutils/defs.h> + #include <isula_libutils/isulad_daemon_configs.h> + #include <isula_libutils/json_common.h> +@@ -39,6 +40,7 @@ + #include "constants.h" + #include "isula_libutils/shim_client_process_state.h" + #include "isula_libutils/shim_client_runtime_stats.h" ++#include "isula_libutils/shim_client_cgroup_resources.h" + #include "isula_libutils/oci_runtime_state.h" + #include "isulad_config.h" + #include "utils_string.h" +@@ -54,6 +56,9 @@ + #define RESIZE_DATA_SIZE 100 + #define PID_WAIT_TIME 120 + ++// file name formats of cgroup resources json ++#define RESOURCE_FNAME_FORMATS "%s/resources.json" ++ + // handle string from stderr output. + typedef int(*handle_output_callback_t)(const char *output); + +@@ -229,34 +234,27 @@ bool rt_isula_detect(const char *runtime) + static int create_process_json_file(const char *workdir, const shim_client_process_state *p) + { + struct parser_context ctx = { OPT_GEN_SIMPLIFY, 0 }; +- parser_error perr = NULL; +- char *data = NULL; ++ __isula_auto_free parser_error perr = NULL; ++ __isula_auto_free char *data = NULL; + char fname[PATH_MAX] = { 0 }; +- int retcode = 0; + + if (snprintf(fname, sizeof(fname), "%s/process.json", workdir) < 0) { +- ERROR("failed make process.json full path"); ++ ERROR("Failed make process.json full path"); + return -1; + } + + data = shim_client_process_state_generate_json(p, &ctx, &perr); + if (data == NULL) { +- retcode = -1; +- ERROR("failed generate json for process.json error=%s", perr); +- goto out; ++ ERROR("Failed generate json for process.json error=%s", perr); ++ return -1; + } + + if (util_write_file(fname, data, strlen(data), DEFAULT_SECURE_FILE_MODE) != 0) { +- retcode = -1; +- ERROR("failed write process.json"); +- goto out; ++ ERROR("Failed write process.json"); ++ return -1; + } + +-out: +- UTIL_FREE_AND_SET_NULL(perr); +- UTIL_FREE_AND_SET_NULL(data); +- +- return retcode; ++ return 0; + } + + static void get_runtime_cmd(const char *runtime, const char **cmd) +@@ -733,18 +731,18 @@ static int shim_create(bool fg, const char *id, const char *workdir, const char + runtime_exec_param_dump(params); + + if (snprintf(fpid, sizeof(fpid), "%s/shim-pid", workdir) < 0) { +- ERROR("failed make shim-pid full path"); ++ ERROR("Failed make shim-pid full path"); + return -1; + } + + if (pipe2(exec_fd, O_CLOEXEC) != 0) { +- ERROR("failed to create pipe for shim create"); ++ ERROR("Failed to create pipe for shim create"); + return -1; + } + + pid = fork(); + if (pid < 0) { +- ERROR("failed fork for shim parent %s", strerror(errno)); ++ ERROR("Failed fork for shim parent %s", strerror(errno)); + close(exec_fd[0]); + close(exec_fd[1]); + return -1; +@@ -1264,13 +1262,153 @@ int rt_isula_attach(const char *id, const char *runtime, const rt_attach_params_ + return -1; + } + +-int rt_isula_update(const char *id, const char *runtime, const rt_update_params_t *params) ++static int to_engine_resources(const host_config *hostconfig, shim_client_cgroup_resources *cr) ++{ ++ uint64_t period = 0; ++ int64_t quota = 0; ++ ++ if (hostconfig == NULL || cr == NULL) { ++ return -1; ++ } ++ ++ cr->block_io = util_common_calloc_s(sizeof(shim_client_cgroup_resources_block_io)); ++ if (cr->block_io == NULL) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ ++ cr->cpu = util_common_calloc_s(sizeof(shim_client_cgroup_resources_cpu)); ++ if (cr->cpu == NULL) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ ++ cr->memory = util_common_calloc_s(sizeof(shim_client_cgroup_resources_memory)); ++ if (cr->memory == NULL) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ ++ cr->block_io->weight = hostconfig->blkio_weight; ++ cr->cpu->shares = (uint64_t)hostconfig->cpu_shares; ++ cr->cpu->period = (uint64_t)hostconfig->cpu_period; ++ cr->cpu->quota = hostconfig->cpu_quota; ++ cr->cpu->cpus = util_strdup_s(hostconfig->cpuset_cpus); ++ cr->cpu->mems = util_strdup_s(hostconfig->cpuset_mems); ++ cr->memory->limit = (uint64_t)hostconfig->memory; ++ cr->memory->swap = (uint64_t)hostconfig->memory_swap; ++ cr->memory->reservation = (uint64_t)hostconfig->memory_reservation; ++ cr->memory->kernel = (uint64_t)hostconfig->kernel_memory; ++ cr->cpu->realtime_period = hostconfig->cpu_realtime_period; ++ cr->cpu->realtime_runtime = hostconfig->cpu_realtime_runtime; ++ ++ // when --cpus=n is set, nano_cpus = n * 1e9. ++ if (hostconfig->nano_cpus > 0) { ++ // in the case, period will be set to the default value of 100000(0.1s). ++ period = (uint64_t)(100 * Time_Milli / Time_Micro); ++ // set quota = period * n, in order to let container process fully occupy n cpus. ++ if ((hostconfig->nano_cpus / 1e9) > (INT64_MAX / (int64_t)period)) { ++ ERROR("Overflow of quota"); ++ return -1; ++ } ++ quota = hostconfig->nano_cpus / 1e9 * (int64_t)period; ++ cr->cpu->period = period; ++ cr->cpu->quota = quota; ++ } ++ ++ return 0; ++} ++ ++static int create_resources_json_file(const char *workdir, const shim_client_cgroup_resources *cr, char *fname, ++ size_t fname_size) + { +- ERROR("isula update not support on isulad-shim"); +- isulad_set_error_message("isula update not support on isulad-shim"); ++ struct parser_context ctx = { OPT_GEN_SIMPLIFY, 0 }; ++ __isula_auto_free parser_error perr = NULL; ++ __isula_auto_free char *data = NULL; ++ ++ if (snprintf(fname, fname_size, RESOURCE_FNAME_FORMATS, workdir) < 0) { ++ ERROR("Failed make resources.json full path"); ++ return -1; ++ } ++ ++ data = shim_client_cgroup_resources_generate_json(cr, &ctx, &perr); ++ if (data == NULL) { ++ return -1; ++ } ++ ++ if (util_write_file(fname, data, strlen(data), DEFAULT_SECURE_FILE_MODE) != 0) { ++ return -1; ++ } ++ ++ return 0; ++} ++ ++// show std error msg, always return -1. ++static int show_stderr(const char *err) ++{ ++ isulad_set_error_message(err); + return -1; + } + ++int rt_isula_update(const char *id, const char *runtime, const rt_update_params_t *params) ++{ ++ int ret = 0; ++ char workdir[PATH_MAX] = { 0 }; ++ char resources_fname[PATH_MAX] = { 0 }; ++ const char *opts[2] = { 0 }; ++ shim_client_cgroup_resources *cr = NULL; ++ ++ if (id == NULL || runtime == NULL || params == NULL) { ++ ERROR("Nullptr arguments not allowed"); ++ return -1; ++ } ++ ++ ret = snprintf(workdir, sizeof(workdir), "%s/%s/update", params->state, id); ++ if (ret < 0) { ++ ERROR("Failed join update full path"); ++ return ret; ++ } ++ ++ ret = util_mkdir_p(workdir, DEFAULT_SECURE_DIRECTORY_MODE); ++ if (ret < 0) { ++ ERROR("Failed mkdir update workdir %s", workdir); ++ return ret; ++ } ++ ++ cr = util_common_calloc_s(sizeof(shim_client_cgroup_resources)); ++ if (cr == NULL) { ++ ERROR("Out of memory"); ++ goto del_out; ++ } ++ ++ ret = to_engine_resources(params->hostconfig, cr); ++ if (ret < 0) { ++ ERROR("Failed to get resources for update"); ++ goto del_out; ++ } ++ ++ ret = create_resources_json_file(workdir, cr, resources_fname, sizeof(resources_fname)); ++ if (ret != 0) { ++ ERROR("%s: failed create update json file", id); ++ goto del_out; ++ } ++ ++ opts[0] = "--resources"; ++ opts[1] = resources_fname; ++ ++ if (runtime_call_simple(workdir, runtime, "update", opts, 2, id, show_stderr) != 0) { ++ ERROR("Call runtime update id failed"); ++ ret = -1; ++ } ++ ++del_out: ++ if (util_recursive_rmdir(workdir, 0)) { ++ ERROR("Rmdir %s failed", workdir); ++ } ++ free_shim_client_cgroup_resources(cr); ++ return ret; ++} ++ + int rt_isula_pause(const char *id, const char *runtime, const rt_pause_params_t *params) + { + char workdir[PATH_MAX] = { 0 }; +-- +2.25.1 + diff --git a/0030-Refine-as-others-feedback.patch b/0030-Refine-as-others-feedback.patch new file mode 100644 index 0000000..39691ca --- /dev/null +++ b/0030-Refine-as-others-feedback.patch @@ -0,0 +1,41 @@ +From 4900996ca52e46795eb25b05ce50519c7c71ae38 Mon Sep 17 00:00:00 2001 +From: sailorvii <chenw66@chinaunicom.cn> +Date: Thu, 2 Mar 2023 09:04:57 +0000 +Subject: [PATCH 30/53] Refine as others' feedback. + +--- + src/daemon/modules/runtime/isula/isula_rt_ops.c | 11 ++++++----- + 1 file changed, 6 insertions(+), 5 deletions(-) + +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index 5d622515..7ec3fc49 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -473,6 +473,12 @@ static void runtime_exec_func(void *arg) + _exit(EXIT_FAILURE); + } + ++ // clear NOTIFY_SOCKET from the env to adapt runc start ++ if (strcmp(rei->subcmd, "start") == 0 && unsetenv("NOTIFY_SOCKET") != 0) { ++ dprintf(STDERR_FILENO, "unset env NOTIFY_SOCKET failed %s", strerror(errno)); ++ _exit(EXIT_FAILURE); ++ } ++ + execvp(rei->cmd, rei->params); + dprintf(STDERR_FILENO, "exec %s %s %s failed", rei->cmd, rei->subcmd, rei->id); + _exit(EXIT_FAILURE); +@@ -994,11 +1000,6 @@ int rt_isula_start(const char *id, const char *runtime, const rt_start_params_t + pid_info->ppid = shim_pid; + pid_info->pstart_time = p_proc->start_time; + +- // clear NOTIFY_SOCKET from the env to adapt runc start +- if (unsetenv("NOTIFY_SOCKET") != 0) { +- ERROR("%s: unset env NOTIFY_SOCKET failed %s", id); +- } +- + if (runtime_call_simple(workdir, runtime, "start", NULL, 0, id, NULL) != 0) { + ERROR("call runtime start id failed"); + goto out; +-- +2.25.1 + diff --git a/0031-fix-CRI-SetupPod-and-TearDownPod-deadlock.patch b/0031-fix-CRI-SetupPod-and-TearDownPod-deadlock.patch new file mode 100644 index 0000000..c5bd888 --- /dev/null +++ b/0031-fix-CRI-SetupPod-and-TearDownPod-deadlock.patch @@ -0,0 +1,134 @@ +From a87e8aeea252e5aec9e1dea0daf99562eb86b092 Mon Sep 17 00:00:00 2001 +From: zhangxiaoyu <zhangxiaoyu58@huawei.com> +Date: Thu, 2 Mar 2023 20:05:49 +0800 +Subject: [PATCH 31/53] fix CRI SetupPod and TearDownPod deadlock + +Signed-off-by: zhangxiaoyu <zhangxiaoyu58@huawei.com> +--- + src/daemon/entry/cri/cni_network_plugin.cc | 14 +++++++---- + .../cri/cri_pod_sandbox_manager_service.cc | 2 ++ + src/daemon/entry/cri/network_plugin.cc | 25 +++++++++++++------ + 3 files changed, 29 insertions(+), 12 deletions(-) + +diff --git a/src/daemon/entry/cri/cni_network_plugin.cc b/src/daemon/entry/cri/cni_network_plugin.cc +index 1bce13f6..0fe095b6 100644 +--- a/src/daemon/entry/cri/cni_network_plugin.cc ++++ b/src/daemon/entry/cri/cni_network_plugin.cc +@@ -120,8 +120,10 @@ auto CniNetworkPlugin::Name() const -> const std::string & + + void CniNetworkPlugin::CheckInitialized(Errors &err) + { +- RLockNetworkMap(err); +- if (err.NotEmpty()) { ++ Errors tmpErr; ++ RLockNetworkMap(tmpErr); ++ if (tmpErr.NotEmpty()) { ++ err.AppendError(tmpErr.GetCMessage()); + return; + } + +@@ -129,9 +131,10 @@ void CniNetworkPlugin::CheckInitialized(Errors &err) + err.SetError("cni config uninitialized"); + } + +- UnlockNetworkMap(err); +- if (err.NotEmpty()) { +- WARN("Unable to update cni config: %s", err.GetCMessage()); ++ UnlockNetworkMap(tmpErr); ++ if (tmpErr.NotEmpty()) { ++ WARN("Unable to update cni config: %s", tmpErr.GetCMessage()); ++ err.AppendError(tmpErr.GetCMessage()); + } + } + +@@ -623,6 +626,7 @@ void CniNetworkPlugin::SetUpPod(const std::string &ns, const std::string &name, + return; + } + ++ err.Clear(); + RLockNetworkMap(err); + if (err.NotEmpty()) { + ERROR("%s", err.GetCMessage()); +diff --git a/src/daemon/entry/cri/cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/cri_pod_sandbox_manager_service.cc +index 3d183861..ee523b9c 100644 +--- a/src/daemon/entry/cri/cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/cri_pod_sandbox_manager_service.cc +@@ -705,6 +705,7 @@ auto PodSandboxManagerService::GetRealSandboxIDToStop(const std::string &podSand + if (status->linux().namespaces().has_options()) { + hostNetwork = (status->linux().namespaces().options().network() == runtime::v1alpha2::NamespaceMode::NODE); + } ++ // if metadata is invalid, don't return -1 and continue stopping pod + if (status->has_metadata()) { + name = status->metadata().name(); + ns = status->metadata().namespace_(); +@@ -831,6 +832,7 @@ auto PodSandboxManagerService::ClearCniNetwork(const std::string &realSandboxID, + } + + stdAnnos.insert(std::pair<std::string, std::string>(CRIHelpers::Constants::POD_SANDBOX_KEY, netnsPath)); ++ pluginErr.Clear(); + m_pluginManager->TearDownPod(ns, name, Network::DEFAULT_NETWORK_INTERFACE_NAME, realSandboxID, stdAnnos, + pluginErr); + if (pluginErr.NotEmpty()) { +diff --git a/src/daemon/entry/cri/network_plugin.cc b/src/daemon/entry/cri/network_plugin.cc +index 4a119d6b..e55db4ea 100644 +--- a/src/daemon/entry/cri/network_plugin.cc ++++ b/src/daemon/entry/cri/network_plugin.cc +@@ -425,20 +425,26 @@ void PluginManager::SetUpPod(const std::string &ns, const std::string &name, con + return; + } + ++ Errors tmpErr; + std::string fullName = name + "_" + ns; +- Lock(fullName, error); +- if (error.NotEmpty()) { ++ Lock(fullName, tmpErr); ++ if (tmpErr.NotEmpty()) { ++ error.AppendError(tmpErr.GetCMessage()); + return; + } + INFO("Calling network plugin %s to set up pod %s", m_plugin->Name().c_str(), fullName.c_str()); + +- Errors tmpErr; + m_plugin->SetUpPod(ns, name, interfaceName, podSandboxID, annotations, options, network_settings_json, tmpErr); + if (tmpErr.NotEmpty()) { + error.Errorf("NetworkPlugin %s failed to set up pod %s network: %s", m_plugin->Name().c_str(), fullName.c_str(), + tmpErr.GetCMessage()); + } +- Unlock(fullName, error); ++ ++ tmpErr.Clear(); ++ Unlock(fullName, tmpErr); ++ if (tmpErr.NotEmpty()) { ++ error.AppendError(tmpErr.GetCMessage()); ++ } + } + + void PluginManager::TearDownPod(const std::string &ns, const std::string &name, const std::string &interfaceName, +@@ -447,8 +453,9 @@ void PluginManager::TearDownPod(const std::string &ns, const std::string &name, + { + Errors tmpErr; + std::string fullName = name + "_" + ns; +- Lock(fullName, error); +- if (error.NotEmpty()) { ++ Lock(fullName, tmpErr); ++ if (tmpErr.NotEmpty()) { ++ error.AppendError(tmpErr.GetCMessage()); + return; + } + if (m_plugin == nullptr) { +@@ -462,7 +469,11 @@ void PluginManager::TearDownPod(const std::string &ns, const std::string &name, + fullName.c_str(), tmpErr.GetCMessage()); + } + unlock: +- Unlock(fullName, error); ++ tmpErr.Clear(); ++ Unlock(fullName, tmpErr); ++ if (tmpErr.NotEmpty()) { ++ error.AppendError(tmpErr.GetCMessage()); ++ } + } + + void NoopNetworkPlugin::Init(const std::string &hairpinMode, const std::string &nonMasqueradeCIDR, int mtu, +-- +2.25.1 + diff --git a/0032-remote-layer-store-demo.patch b/0032-remote-layer-store-demo.patch new file mode 100644 index 0000000..a20a04d --- /dev/null +++ b/0032-remote-layer-store-demo.patch @@ -0,0 +1,2296 @@ +From 9072007e7f61f7658baf7c4101126040dc341d0b Mon Sep 17 00:00:00 2001 +From: "Neil.wrz" <wangrunze13@huawei.com> +Date: Mon, 9 Jan 2023 17:12:53 -0800 +Subject: [PATCH 32/53] remote layer store demo + +Signed-off-by: Neil <wangrunze13@huawei.com> +--- + cmake/options.cmake | 6 + + src/daemon/config/isulad_config.c | 1 + + src/daemon/modules/image/oci/oci_image.c | 6 +- + .../modules/image/oci/storage/CMakeLists.txt | 6 + + .../storage/image_store/image_remote_impl.c | 173 +++++++++ + .../oci/storage/image_store/image_store.c | 76 +++- + .../oci/storage/image_store/image_store.h | 11 + + .../storage/layer_store/graphdriver/driver.c | 4 +- + .../storage/layer_store/graphdriver/driver.h | 3 + + .../graphdriver/overlay2/driver_overlay2.c | 158 +++++++- + .../graphdriver/overlay2/driver_overlay2.h | 8 + + .../overlay2/overlay_remote_impl.c | 282 ++++++++++++++ + .../storage/layer_store/layer_remote_impl.c | 219 +++++++++++ + .../oci/storage/layer_store/layer_store.c | 199 +++++++++- + .../oci/storage/layer_store/layer_store.h | 11 + + .../remote_layer_support/CMakeLists.txt | 12 + + .../remote_layer_support/remote_support.c | 122 ++++++ + .../remote_layer_support/remote_support.h | 58 +++ + .../ro_symlink_maintain.c | 347 ++++++++++++++++++ + .../ro_symlink_maintain.h | 52 +++ + .../modules/image/oci/storage/storage.c | 11 +- + .../modules/image/oci/storage/storage.h | 3 + + 22 files changed, 1750 insertions(+), 18 deletions(-) + create mode 100644 src/daemon/modules/image/oci/storage/image_store/image_remote_impl.c + create mode 100644 src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/overlay_remote_impl.c + create mode 100644 src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c + create mode 100644 src/daemon/modules/image/oci/storage/remote_layer_support/CMakeLists.txt + create mode 100644 src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c + create mode 100644 src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h + create mode 100644 src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c + create mode 100644 src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.h + +diff --git a/cmake/options.cmake b/cmake/options.cmake +index a357a504..44ccf672 100644 +--- a/cmake/options.cmake ++++ b/cmake/options.cmake +@@ -148,6 +148,12 @@ if (DISABLE_CLEANUP STREQUAL "ON") + message("${Green}-- Disable cleanup module") + endif() + ++option(ENABLE_REMOTE_LAYER_STORE "enable remote layer store" OFF) ++if (ENABLE_REMOTE_LAYER_STORE STREQUAL "ON") ++ add_definitions(-DENABLE_REMOTE_LAYER_STORE) ++ message("${Green}-- Enable remote layer store") ++endif() ++ + option(MUSL "available for musl" OFF) + if (MUSL) + add_definitions(-D__MUSL__) +diff --git a/src/daemon/config/isulad_config.c b/src/daemon/config/isulad_config.c +index 38bf4bf9..b7bfe2df 100644 +--- a/src/daemon/config/isulad_config.c ++++ b/src/daemon/config/isulad_config.c +@@ -1555,6 +1555,7 @@ static int merge_authorization_conf_into_global(struct service_arguments *args, + static int merge_storage_conf_into_global(struct service_arguments *args, isulad_daemon_configs *tmp_json_confs) + { + override_string_value(&args->json_confs->storage_driver, &tmp_json_confs->storage_driver); ++ args->json_confs->storage_enable_remote_layer = tmp_json_confs->storage_enable_remote_layer; + + if (string_array_append(tmp_json_confs->storage_opts, tmp_json_confs->storage_opts_len, + &(args->json_confs->storage_opts_len), &(args->json_confs->storage_opts)) != 0) { +diff --git a/src/daemon/modules/image/oci/oci_image.c b/src/daemon/modules/image/oci/oci_image.c +index 50d13cec..fa92a861 100644 +--- a/src/daemon/modules/image/oci/oci_image.c ++++ b/src/daemon/modules/image/oci/oci_image.c +@@ -218,6 +218,10 @@ static int storage_module_init_helper(const isulad_daemon_configs *args) + goto out; + } + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ storage_opts->enable_remote_layer = args->storage_enable_remote_layer; ++#endif ++ + if (util_dup_array_of_strings((const char **)args->storage_opts, args->storage_opts_len, &storage_opts->driver_opts, + &storage_opts->driver_opts_len) != 0) { + ERROR("Failed to get storage storage opts"); +@@ -763,4 +767,4 @@ int oci_search(const im_search_request *request, imagetool_search_result **resul + + return ret; + } +-#endif +\ No newline at end of file ++#endif +diff --git a/src/daemon/modules/image/oci/storage/CMakeLists.txt b/src/daemon/modules/image/oci/storage/CMakeLists.txt +index 06c2f378..24b71450 100644 +--- a/src/daemon/modules/image/oci/storage/CMakeLists.txt ++++ b/src/daemon/modules/image/oci/storage/CMakeLists.txt +@@ -3,12 +3,17 @@ aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_storage_srcs) + add_subdirectory(image_store) + add_subdirectory(layer_store) + add_subdirectory(rootfs_store) ++IF (ENABLE_REMOTE_LAYER_STORE) ++add_subdirectory(remote_layer_support) ++ENDIF() ++ + + set(STORAGE_SRCS + ${local_storage_srcs} + ${IMAGE_STORE_SRCS} + ${LAYER_STORE_SRCS} + ${ROOTFS_STORE_SRCS} ++ ${REMOTE_LAYER_SUPPORT_SRCS} + PARENT_SCOPE + ) + +@@ -17,5 +22,6 @@ set(STORAGE_INCS + ${IMAGE_STORE_INCS} + ${LAYER_STORE_INCS} + ${ROOTFS_STORE_INCS} ++ ${REMOTE_LAYER_SUPPORT_INCS} + PARENT_SCOPE + ) +diff --git a/src/daemon/modules/image/oci/storage/image_store/image_remote_impl.c b/src/daemon/modules/image/oci/storage/image_store/image_remote_impl.c +new file mode 100644 +index 00000000..20da8116 +--- /dev/null ++++ b/src/daemon/modules/image/oci/storage/image_store/image_remote_impl.c +@@ -0,0 +1,173 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2020-2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: wangrunze ++ * Create: 2023-03-03 ++ * Description: provide remote image store functions ++ ******************************************************************************/ ++#define _GNU_SOURCE ++#include "image_store.h" ++ ++#include <isula_libutils/log.h> ++#include <stdio.h> ++ ++#include "remote_support.h" ++#include "ro_symlink_maintain.h" ++#include "map.h" ++#include "utils_file.h" ++#include "utils.h" ++#include "layer_store.h" ++#include "utils_array.h" ++ ++struct remote_image_data { ++ const char *image_home; ++}; ++ ++static map_t *image_byid_old = NULL; ++static map_t *image_byid_new = NULL; ++ ++static void *remote_support_create(const char *remote_home, const char *remote_ro) ++{ ++ struct remote_image_data *data = util_common_calloc_s(sizeof(struct remote_image_data)); ++ if (data == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ data->image_home = remote_home; ++ image_byid_old = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); ++ image_byid_new = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); ++ return data; ++} ++ ++static void remote_support_destroy(void *data) ++{ ++ if (data == NULL) { ++ return; ++ } ++ ++ map_free(image_byid_old); ++ map_free(image_byid_new); ++ ++ free(data); ++ return; ++} ++ ++static int remote_support_scan(void *data) ++{ ++ int ret = 0; ++ int nret; ++ char **image_dirs = NULL; ++ size_t image_dirs_num = 0; ++ size_t i; ++ char *id_patten = "^[a-f0-9]{64}$"; ++ char image_path[PATH_MAX] = { 0x00 }; ++ bool exist = true; ++ struct remote_image_data *img_data = (struct remote_image_data *)data; ++ ++ ret = util_list_all_subdir(img_data->image_home, &image_dirs); ++ if (ret != 0) { ++ ERROR("Failed to get images directory"); ++ goto out; ++ } ++ image_dirs_num = util_array_len((const char **)image_dirs); ++ ++ for (i = 0; i < image_dirs_num; i++) { ++ bool valid_v1_image = false; ++ ++ if (util_reg_match(id_patten, image_dirs[i]) != 0) { ++ DEBUG("Image's json is placed inside image's data directory, so skip any other file or directory: %s", ++ image_dirs[i]); ++ continue; ++ } ++ ++ nret = snprintf(image_path, sizeof(image_path), "%s/%s", img_data->image_home, image_dirs[i]); ++ if (nret < 0 || (size_t)nret >= sizeof(image_path)) { ++ ERROR("Failed to get image path"); ++ continue; ++ } ++ ++ if (validate_manifest_schema_version_1(image_path, &valid_v1_image) != 0) { ++ ERROR("Failed to validate manifest schema version 1 format"); ++ continue; ++ } ++ ++ if (!valid_v1_image) { ++ map_insert(image_byid_new, util_strdup_s(image_dirs[i]), (void *)&exist); ++ } ++ } ++ ++out: ++ util_free_array(image_dirs); ++ return ret; ++} ++ ++static int remote_support_add(void *data) ++{ ++ char **array_added = NULL; ++ char **array_deleted = NULL; ++ char *top_layer = NULL; ++ map_t *tmp_map = NULL; ++ int i = 0; ++ int ret = 0; ++ ++ if (data == NULL) { ++ return -1; ++ } ++ ++ array_added = added_layers(image_byid_old, image_byid_new); ++ array_deleted = deleted_layers(image_byid_old, image_byid_new); ++ ++ for (i = 0; i < util_array_len((const char **)array_added); i++) { ++ top_layer = get_top_layer_from_json(array_added[i]); ++ if (top_layer != NULL && !layer_remote_layer_valid(top_layer)) { ++ ERROR("ERROR not find valid under layer, remoet image:%s not added", array_added[i]); ++ map_remove(image_byid_new, (void *)array_added[i]); ++ continue; ++ } ++ ++ if (append_image_by_directory_with_lock(array_added[i]) != 0) { ++ ERROR("Failed to load image into memrory: %s", array_added[i]); ++ ret = -1; ++ } ++ } ++ ++ for (i = 0; i < util_array_len((const char **)array_deleted); i++) { ++ if (remove_image_from_memory_with_lock(array_deleted[i]) != 0) { ++ ERROR("Failed to remove remote memory store"); ++ ret = -1; ++ } ++ } ++ ++ tmp_map = image_byid_old; ++ image_byid_old = image_byid_new; ++ image_byid_new = tmp_map; ++ empty_map(image_byid_new); ++ ++ util_free_array(array_added); ++ util_free_array(array_deleted); ++ free(top_layer); ++ ++ return ret; ++} ++ ++remote_support *image_store_impl_remote_support(void) ++{ ++ remote_support *rs = util_common_calloc_s(sizeof(remote_support)); ++ if (rs == NULL) { ++ return NULL; ++ } ++ ++ rs->create = remote_support_create; ++ rs->destroy = remote_support_destroy; ++ rs->scan_remote_dir = remote_support_scan; ++ rs->load_item = remote_support_add; ++ ++ return rs; ++} +diff --git a/src/daemon/modules/image/oci/storage/image_store/image_store.c b/src/daemon/modules/image/oci/storage/image_store/image_store.c +index cf1e88ff..caff3705 100644 +--- a/src/daemon/modules/image/oci/storage/image_store/image_store.c ++++ b/src/daemon/modules/image/oci/storage/image_store/image_store.c +@@ -49,6 +49,9 @@ + #include "image_type.h" + #include "linked_list.h" + #include "utils_verify.h" ++#ifdef ENABLE_REMOTE_LAYER_STORE ++#include "ro_symlink_maintain.h" ++#endif + + // the name of the big data item whose contents we consider useful for computing a "digest" of the + // image, by which we can locate the image later. +@@ -3096,7 +3099,7 @@ out: + return ret; + } + +-static int validate_manifest_schema_version_1(const char *path, bool *valid) ++int validate_manifest_schema_version_1(const char *path, bool *valid) + { + int ret = 0; + int nret; +@@ -3640,6 +3643,10 @@ int image_store_init(struct storage_module_init_options *opts) + goto out; + } + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ remote_image_init(g_image_store->dir); ++#endif ++ + out: + if (ret != 0) { + free_image_store(g_image_store); +@@ -3648,3 +3655,70 @@ out: + free(root_dir); + return ret; + } ++ ++#ifdef ENABLE_REMOTE_LAYER_STORE ++int append_image_by_directory_with_lock(const char *id) ++{ ++ int ret = 0; ++ int nret = 0; ++ char image_path[PATH_MAX] = { 0x00 }; ++ ++ if (!image_store_lock(EXCLUSIVE)) { ++ ERROR("Failed to lock remote image store when handle: %s", id); ++ return -1; ++ } ++ ++ nret = snprintf(image_path, sizeof(image_path), "%s/%s", g_image_store->dir, id); ++ if (nret < 0 || (size_t)nret >= sizeof(image_path)) { ++ ERROR("Failed to get image path"); ++ return -1; ++ } ++ ++ ret = append_image_by_directory(image_path); ++ image_store_unlock(); ++ ++ return ret; ++} ++ ++int remove_image_from_memory_with_lock(const char *id) ++{ ++ int ret = 0; ++ ++ if (!image_store_lock(EXCLUSIVE)) { ++ ERROR("Failed to lock remote image store when handle: %s", id); ++ return -1; ++ } ++ ++ ret = remove_image_from_memory(id); ++ image_store_unlock(); ++ ++ return ret; ++} ++ ++char *get_top_layer_from_json(const char *img_id) ++{ ++ ++ char *ret = NULL; ++ int nret = 0; ++ char image_path[PATH_MAX] = { 0x00 }; ++ storage_image *im = NULL; ++ parser_error err = NULL; ++ ++ nret = snprintf(image_path, sizeof(image_path), "%s/%s/%s", g_image_store->dir, img_id, IMAGE_JSON); ++ if (nret < 0 || (size_t)nret >= sizeof(image_path)) { ++ ERROR("Failed to get image path"); ++ return NULL; ++ } ++ ++ im = storage_image_parse_file(image_path, NULL, &err); ++ if (im == NULL) { ++ ERROR("Failed to parse images path: %s", err); ++ return NULL; ++ } ++ ++ ret = util_strdup_s(im->layer); ++ free_storage_image(im); ++ ++ return ret; ++} ++#endif +diff --git a/src/daemon/modules/image/oci/storage/image_store/image_store.h b/src/daemon/modules/image/oci/storage/image_store/image_store.h +index edd4fa2c..c3cb50e3 100644 +--- a/src/daemon/modules/image/oci/storage/image_store/image_store.h ++++ b/src/daemon/modules/image/oci/storage/image_store/image_store.h +@@ -28,6 +28,9 @@ + #include "isula_libutils/imagetool_image.h" + #include "isula_libutils/imagetool_images_list.h" + #include "isula_libutils/imagetool_image_summary.h" ++#ifdef ENABLE_REMOTE_LAYER_STORE ++#include "remote_support.h" ++#endif + + struct storage_module_init_options; + +@@ -108,6 +111,14 @@ void image_store_free(); + + imagetool_image_summary *image_store_get_image_summary(const char *id); + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++remote_support *image_store_impl_remote_support(); ++int validate_manifest_schema_version_1(const char *path, bool *valid); ++int append_image_by_directory_with_lock(const char *image_dir); ++int remove_image_from_memory_with_lock(const char *id); ++char *get_top_layer_from_json(const char *img_id); /* return top layer id */ ++#endif ++ + #ifdef __cplusplus + } + #endif +diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.c b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.c +index d3b5209a..b83c63b1 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.c ++++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.c +@@ -141,7 +141,9 @@ int graphdriver_init(const struct storage_module_init_options *opts) + ret = -1; + goto out; + } +- ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ g_drivers[i].enable_remote_layer = opts->enable_remote_layer; ++#endif + if (g_drivers[i].ops->init(&g_drivers[i], driver_home, (const char **)opts->driver_opts, + opts->driver_opts_len) != 0) { + ret = -1; +diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.h b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.h +index 7faf70c8..acd847cc 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.h ++++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.h +@@ -90,6 +90,9 @@ struct graphdriver { + bool support_dtype; + + bool support_quota; ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ bool enable_remote_layer; ++#endif + struct pquota_control *quota_ctrl; + + // options for overlay2 +diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.c b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.c +index 330c230a..eedbeef2 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.c ++++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.c +@@ -12,6 +12,7 @@ + * Create: 2020-04-02 + * Description: provide overlay2 function definition + ******************************************************************************/ ++#define _GNU_SOURCE + #include "driver_overlay2.h" + + #include <string.h> +@@ -44,6 +45,9 @@ + #include "utils_timestamp.h" + #include "selinux_label.h" + #include "err_msg.h" ++#ifdef ENABLE_REMOTE_LAYER_STORE ++#include "ro_symlink_maintain.h" ++#endif + + struct io_read_wrapper; + +@@ -343,6 +347,13 @@ int overlay2_init(struct graphdriver *driver, const char *driver_home, const cha + return -1; + } + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ if (driver->enable_remote_layer && remote_overlay_init(driver_home) != 0) { ++ ERROR("Failed to init overlay remote"); ++ return -1; ++ } ++#endif ++ + driver->home = util_strdup_s(driver_home); + + root_dir = util_path_dir(driver_home); +@@ -423,7 +434,7 @@ static int mk_diff_directory(const char *layer_dir) + int ret = 0; + char *diff_dir = NULL; + #ifdef ENABLE_USERNS_REMAP +- char* userns_remap = conf_get_isulad_userns_remap(); ++ char *userns_remap = conf_get_isulad_userns_remap(); + #endif + + diff_dir = util_path_join(layer_dir, OVERLAY_LAYER_DIFF); +@@ -538,7 +549,7 @@ static int mk_work_directory(const char *layer_dir) + int ret = 0; + char *work_dir = NULL; + #ifdef ENABLE_USERNS_REMAP +- char* userns_remap = conf_get_isulad_userns_remap(); ++ char *userns_remap = conf_get_isulad_userns_remap(); + #endif + + work_dir = util_path_join(layer_dir, OVERLAY_LAYER_WORK); +@@ -575,7 +586,7 @@ static int mk_merged_directory(const char *layer_dir) + int ret = 0; + char *merged_dir = NULL; + #ifdef ENABLE_USERNS_REMAP +- char* userns_remap = conf_get_isulad_userns_remap(); ++ char *userns_remap = conf_get_isulad_userns_remap(); + #endif + + merged_dir = util_path_join(layer_dir, OVERLAY_LAYER_MERGED); +@@ -852,13 +863,115 @@ out: + return ret; + } + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++static int do_create_remote_ro(const char *id, const char *parent, const struct graphdriver *driver, ++ const struct driver_create_opts *create_opts) ++{ ++ int ret = 0; ++ int get_err = 0; ++ char *ro_symlink = NULL; ++ char *ro_home = NULL; ++ char *layer_dir = NULL; ++#ifdef ENABLE_USERNS_REMAP ++ char *userns_remap = conf_get_isulad_userns_remap(); ++#endif ++ ++ ro_home = util_path_join(driver->home, OVERLAY_RO_DIR); ++ if (ro_home == NULL) { ++ ERROR("Failed to join ro_home"); ++ ret = -1; ++ goto out; ++ } ++ ++ layer_dir = util_path_join(ro_home, id); ++ if (layer_dir == NULL) { ++ ERROR("Failed to join layer_dir"); ++ ret = -1; ++ goto out; ++ } ++ ++ ro_symlink = util_path_join(driver->home, id); ++ if (ro_symlink == NULL) { ++ ERROR("Failed to join ro_symlink"); ++ ret = -1; ++ goto out; ++ } ++ ++ if (layer_dir == NULL) { ++ ERROR("Failed to join layer dir:%s", id); ++ ret = -1; ++ goto out; ++ } ++ ++ if (check_parent_valid(parent, driver) != 0) { ++ ret = -1; ++ goto out; ++ } ++ ++ if (util_mkdir_p(layer_dir, 0700) != 0) { ++ ERROR("Unable to create layer directory %s.", layer_dir); ++ ret = -1; ++ goto out; ++ } ++ ++ // mk symbol link ++ if (symlink(layer_dir, ro_symlink) != 0) { ++ SYSERROR("Unable to create symbol link to layer directory %s", layer_dir); ++ ret = -1; ++ goto err_out; ++ } ++ ++#ifdef ENABLE_USERNS_REMAP ++ if (set_file_owner_for_userns_remap(layer_dir, userns_remap) != 0) { ++ ERROR("Unable to change directory %s owner for user remap.", layer_dir); ++ ret = -1; ++ goto out; ++ } ++#endif ++ ++ if (create_opts->storage_opt != NULL && create_opts->storage_opt->len != 0) { ++ if (set_layer_quota(layer_dir, create_opts->storage_opt, driver) != 0) { ++ ERROR("Unable to set layer quota %s", layer_dir); ++ ret = -1; ++ goto err_out; ++ } ++ } ++ ++ if (mk_sub_directories(id, parent, layer_dir, driver->home) != 0) { ++ ret = -1; ++ goto err_out; ++ } ++ ++ goto out; ++ ++err_out: ++ if (util_recursive_rmdir(layer_dir, 0)) { ++ ERROR("Failed to delete layer path: %s", layer_dir); ++ } ++ ++ // to remove a file ++ if (util_fileself_exists(ro_symlink) && !util_force_remove_file(ro_symlink, &get_err)) { ++ ERROR("Failed to remove symbol link %s", ro_symlink); ++ } ++ ++out: ++ free(layer_dir); ++ free(ro_home); ++ free(ro_symlink); ++#ifdef ENABLE_USERNS_REMAP ++ free(userns_remap); ++#endif ++ return ret; ++} ++#endif ++ + static int do_create(const char *id, const char *parent, const struct graphdriver *driver, + const struct driver_create_opts *create_opts) + { + int ret = 0; + char *layer_dir = NULL; + #ifdef ENABLE_USERNS_REMAP +- char* userns_remap = conf_get_isulad_userns_remap(); ++ char *userns_remap = conf_get_isulad_userns_remap(); + #endif + + layer_dir = util_path_join(driver->home, id); +@@ -1002,7 +1115,15 @@ int overlay2_create_ro(const char *id, const char *parent, const struct graphdri + return -1; + } + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ if (driver->enable_remote_layer) { ++ return do_create_remote_ro(id, parent, driver, create_opts); ++ } else { ++ return do_create(id, parent, driver, create_opts); ++ } ++#else + return do_create(id, parent, driver, create_opts); ++#endif + } + + static char *read_layer_link_file(const char *layer_dir) +@@ -1047,6 +1168,9 @@ int overlay2_rm_layer(const char *id, const struct graphdriver *driver) + char *link_id = NULL; + char link_path[PATH_MAX] = { 0 }; + char clean_path[PATH_MAX] = { 0 }; ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ struct stat stat_buf; ++#endif + + if (id == NULL || driver == NULL) { + ERROR("Invalid input arguments"); +@@ -1079,11 +1203,34 @@ int overlay2_rm_layer(const char *id, const struct graphdriver *driver) + } + } + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ if (lstat(layer_dir, &stat_buf) < 0) { ++ SYSERROR("Failed to lstat path: %s", layer_dir); ++ ret = -1; ++ goto out; ++ } ++ ++ if (driver->enable_remote_layer && S_ISLNK(stat_buf.st_mode)) { ++ // jusdge if the dir is symlink? ++ if (remote_overlay_remove_ro_dir(id) != 0) { ++ ERROR("Failed to delete symlink to layer dir: %s", layer_dir); ++ ret = -1; ++ goto out; ++ } ++ } else { ++ if (util_recursive_rmdir(layer_dir, 0) != 0) { ++ SYSERROR("Failed to remove layer directory %s", layer_dir); ++ ret = -1; ++ goto out; ++ } ++ } ++#else + if (util_recursive_rmdir(layer_dir, 0) != 0) { + SYSERROR("Failed to remove layer directory %s", layer_dir); + ret = -1; + goto out; + } ++#endif + + out: + free(layer_dir); +@@ -1747,7 +1894,7 @@ int overlay2_apply_diff(const char *id, const struct graphdriver *driver, const + int ret = 0; + #ifdef ENABLE_USERNS_REMAP + unsigned int size = 0; +- char* userns_remap = conf_get_isulad_userns_remap(); ++ char *userns_remap = conf_get_isulad_userns_remap(); + #endif + char *layer_dir = NULL; + char *layer_diff = NULL; +@@ -2166,3 +2313,4 @@ out: + free(layer_diff); + return ret; + } ++ +diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.h b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.h +index e14271b1..5c1d93fb 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.h ++++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.h +@@ -22,6 +22,9 @@ + #include <stdint.h> + + #include "driver.h" ++#ifdef ENABLE_REMOTE_LAYER_STORE ++#include "remote_support.h" ++#endif + + struct driver_create_opts; + struct driver_mount_opts; +@@ -68,6 +71,11 @@ int overlay2_repair_lowers(const char *id, const char *parent, const struct grap + + int overlay2_get_layer_fs_info(const char *id, const struct graphdriver *driver, imagetool_fs_info *fs_info); + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++remote_support *overlay_driver_impl_remote_support(void); ++bool overlay_remote_layer_valid(const char *layer_id); ++#endif ++ + #ifdef __cplusplus + } + #endif +diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/overlay_remote_impl.c b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/overlay_remote_impl.c +new file mode 100644 +index 00000000..a674a00f +--- /dev/null ++++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/overlay_remote_impl.c +@@ -0,0 +1,282 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: wangrunze ++ * Create: 2023-02-27 ++ * Description: provide remote implementation for driver overlay ++ ******************************************************************************/ ++#define _GNU_SOURCE ++#include "driver_overlay2.h" ++ ++#include <stdio.h> ++ ++#include "map.h" ++#include "remote_support.h" ++#include "ro_symlink_maintain.h" ++#include "isula_libutils/log.h" ++#include "utils.h" ++#include "utils_array.h" ++#include "utils_file.h" ++#include "path.h" ++ ++#define OVERLAY_LINK_DIR "l" ++#define OVERLAY_LAYER_LINK "link" ++ ++struct remote_overlay_data { ++ const char *overlay_home; ++ const char *overlay_ro; ++}; ++ ++static map_t *overlay_byid_old = NULL; ++static map_t *overlay_byid_new = NULL; ++ ++static void *remote_support_create(const char *remote_home, const char *remote_ro) ++{ ++ struct remote_overlay_data *data = util_common_calloc_s(sizeof(struct remote_overlay_data)); ++ if (data == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ data->overlay_home = remote_home; ++ data->overlay_ro = remote_ro; ++ overlay_byid_old = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); ++ overlay_byid_new = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); ++ ++ return data; ++} ++ ++static void remote_support_destroy(void *data) ++{ ++ if (data == NULL) { ++ return; ++ } ++ ++ map_free(overlay_byid_old); ++ map_free(overlay_byid_new); ++ free(data); ++} ++ ++static bool overlay_walk_dir_cb(const char *path_name, const struct dirent *sub_dir, void *context) ++{ ++ bool exist = true; ++ if (!map_insert(overlay_byid_new, util_strdup_s(sub_dir->d_name), (void *)&exist)) { ++ ERROR("can't insert remote layer into map"); ++ return false; ++ } ++ ++ return true; ++} ++ ++static int remote_support_scan(void *data) ++{ ++ struct remote_overlay_data *remote_data = data; ++ return util_scan_subdirs(remote_data->overlay_ro, overlay_walk_dir_cb, data); ++} ++ ++static int do_diff_symlink(const char *id, char *link_id, const char *driver_home) ++{ ++ int ret = 0; ++ int nret = 0; ++ char target_path[PATH_MAX] = { 0 }; ++ char link_path[PATH_MAX] = { 0 }; ++ char clean_path[PATH_MAX] = { 0 }; ++ ++ nret = snprintf(target_path, PATH_MAX, "../%s/diff", id); ++ if (nret < 0 || nret >= PATH_MAX) { ++ ERROR("Failed to get target path %s", id); ++ ret = -1; ++ goto out; ++ } ++ ++ nret = snprintf(link_path, PATH_MAX, "%s/%s/%s", driver_home, OVERLAY_LINK_DIR, link_id); ++ if (nret < 0 || nret >= PATH_MAX) { ++ ERROR("Failed to get link path %s", link_id); ++ ret = -1; ++ goto out; ++ } ++ ++ if (util_clean_path(link_path, clean_path, sizeof(clean_path)) == NULL) { ++ ERROR("failed to get clean path %s", link_path); ++ ret = -1; ++ goto out; ++ } ++ ++ if (util_fileself_exists(clean_path) && util_path_remove(clean_path) != 0) { ++ ERROR("failed to remove old symbol link"); ++ ret = -1; ++ goto out; ++ } ++ ++ nret = symlink(target_path, clean_path); ++ if (nret < 0) { ++ SYSERROR("Failed to create symlink from \"%s\" to \"%s\"", clean_path, target_path); ++ ret = -1; ++ goto out; ++ } ++ ++out: ++ return ret; ++} ++ ++static int remove_one_remote_overlay_layer(struct remote_overlay_data *data, const char *overlay_id) ++{ ++ char *ro_symlink = NULL; ++ char clean_path[PATH_MAX] = { 0 }; ++ int nret = 0; ++ int ret = 0; ++ ++ nret = asprintf(&ro_symlink, "%s/%s", data->overlay_home, overlay_id); ++ if (nret < 0 || nret > PATH_MAX) { ++ SYSERROR("Create layer symbol link path failed"); ++ ret = -1; ++ goto out; ++ } ++ ++ if (util_clean_path(ro_symlink, clean_path, sizeof(clean_path)) == NULL) { ++ ERROR("Failed to clean path: %s", ro_symlink); ++ ret = -1; ++ goto out; ++ } ++ ++ if (util_path_remove(clean_path) != 0) { ++ SYSERROR("Failed to remove link path %s", clean_path); ++ } ++ ++out: ++ free(ro_symlink); ++ return ret; ++} ++ ++static int add_one_remote_overlay_layer(struct remote_overlay_data *data, const char *overlay_id) ++{ ++ char *ro_symlink = NULL; ++ char *layer_dir = NULL; ++ char *link_file = NULL; ++ char *diff_symlink = NULL; ++ int ret = 0; ++ ++ ro_symlink = util_path_join(data->overlay_home, overlay_id); ++ if (ro_symlink == NULL) { ++ ERROR("Failed to join ro symlink path: %s", overlay_id); ++ ret = -1; ++ goto free_out; ++ } ++ ++ layer_dir = util_path_join(data->overlay_ro, overlay_id); ++ if (layer_dir == NULL) { ++ ERROR("Failed to join ro layer dir: %s", overlay_id); ++ ret = -1; ++ goto free_out; ++ } ++ ++ // add RO symbol link first ++ if (!util_fileself_exists(ro_symlink) && symlink(layer_dir, ro_symlink) != 0) { ++ SYSERROR("Unable to create symbol link to layer directory: %s", layer_dir); ++ ret = -1; ++ goto free_out; ++ } ++ ++ // maintain link ++ // try read link file in layer_dir ++ // mk symlink between ro_symlink ++ link_file = util_path_join(layer_dir, OVERLAY_LAYER_LINK); ++ if (link_file == NULL) { ++ ERROR("Failed to get layer link file %s", layer_dir); ++ ret = -1; ++ goto free_out; ++ } ++ ++ if (!util_fileself_exists(link_file)) { ++ ERROR("link file for layer %s not exist", layer_dir); ++ ret = -1; ++ goto free_out; ++ } ++ ++ diff_symlink = util_read_content_from_file(link_file); ++ if (link_file == NULL) { ++ ERROR("Failed to read content from link file of layer %s", layer_dir); ++ ret = -1; ++ goto free_out; ++ } ++ ++ if (do_diff_symlink(overlay_id, diff_symlink, data->overlay_home) != 0) { ++ ERROR("Failed to add diff link for layer %s", overlay_id); ++ ret = -1; ++ } ++ ++free_out: ++ free(ro_symlink); ++ free(layer_dir); ++ free(link_file); ++ free(diff_symlink); ++ ++ return ret; ++} ++ ++static int remote_support_add(void *data) ++{ ++ int ret = 0; ++ char **array_added = NULL; ++ char **array_deleted = NULL; ++ map_t *tmp_map = NULL; ++ int i = 0; ++ ++ if (data == NULL) { ++ return -1; ++ } ++ ++ array_added = added_layers(overlay_byid_old, overlay_byid_new); ++ array_deleted = deleted_layers(overlay_byid_old, overlay_byid_new); ++ ++ for (i = 0; i < util_array_len((const char **)array_added); i++) { ++ if (add_one_remote_overlay_layer(data, array_added[i]) != 0) { ++ ERROR("Failed to add remote overlay layer: %s", array_added[i]); ++ ret = -1; ++ } ++ } ++ ++ for (i = 0; i < util_array_len((const char **)array_deleted); i++) { ++ if (remove_one_remote_overlay_layer(data, array_deleted[i]) != 0) { ++ ERROR("Failed to delete remote overlay layer: %s", array_deleted[i]); ++ ret = -1; ++ } ++ } ++ ++ tmp_map = overlay_byid_old; ++ overlay_byid_old = overlay_byid_new; ++ overlay_byid_new = tmp_map; ++ empty_map(overlay_byid_new); ++ ++ util_free_array(array_added); ++ util_free_array(array_deleted); ++ ++ return ret; ++} ++ ++remote_support *overlay_driver_impl_remote_support(void) ++{ ++ remote_support *rs = util_common_calloc_s(sizeof(remote_support)); ++ if (rs == NULL) { ++ ERROR("Failed to calloc overlay supporter"); ++ return NULL; ++ } ++ ++ rs->create = remote_support_create; ++ rs->destroy = remote_support_destroy; ++ rs->scan_remote_dir = remote_support_scan; ++ rs->load_item = remote_support_add; ++ ++ return rs; ++} ++ ++bool overlay_remote_layer_valid(const char *layer_id) ++{ ++ return map_search(overlay_byid_old, (void *)layer_id) != NULL; ++} +diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c b/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c +new file mode 100644 +index 00000000..d03fc20b +--- /dev/null ++++ b/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c +@@ -0,0 +1,219 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: wangrunze ++ * Create: 2023-02-27 ++ * Description: remote layer store implementation ++ ******************************************************************************/ ++#define _GNU_SOURCE ++#include "layer_store.h" ++ ++#include <pthread.h> ++#include <isula_libutils/log.h> ++#include <stdio.h> ++ ++#include "map.h" ++#include "utils.h" ++#include "remote_support.h" ++#include "ro_symlink_maintain.h" ++#include "path.h" ++#include "driver_overlay2.h" ++ ++struct remote_layer_data { ++ const char *layer_home; ++ const char *layer_ro; ++}; ++ ++static map_t *layer_byid_old = NULL; ++static map_t *layer_byid_new = NULL; ++ ++static void *remote_support_create(const char *layer_home, const char *layer_ro) ++{ ++ struct remote_layer_data *data = util_common_calloc_s(sizeof(struct remote_layer_data)); ++ if (data == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ data->layer_home = util_strdup_s(layer_home); ++ data->layer_ro = util_strdup_s(layer_ro); ++ layer_byid_old = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); ++ layer_byid_new = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); ++ ++ return data; ++}; ++ ++static void remote_support_destroy(void *data) ++{ ++ if (data == NULL) { ++ return; ++ } ++ ++ map_free(layer_byid_old); ++ map_free(layer_byid_new); ++ free(data); ++} ++ ++static bool layer_walk_dir_cb(const char *path_name, const struct dirent *sub_dir, void *context) ++{ ++ bool exist = true; ++ ++ if (!map_insert(layer_byid_new, util_strdup_s(sub_dir->d_name), (void *)&exist)) { ++ ERROR("can't insert remote layer into map"); ++ return false; ++ } ++ ++ return true; ++} ++ ++static int remote_support_scan(void *data) ++{ ++ struct remote_layer_data *remote_data = data; ++ return util_scan_subdirs(remote_data->layer_ro, layer_walk_dir_cb, data); ++} ++ ++static int remove_one_remote_layer(struct remote_layer_data *data, char *layer_id) ++{ ++ char *ro_symlink = NULL; ++ char clean_path[PATH_MAX] = { 0 }; ++ int nret = 0; ++ int ret = 0; ++ ++ nret = asprintf(&ro_symlink, "%s/%s", data->layer_home, layer_id); ++ if (nret < 0 || nret > PATH_MAX) { ++ SYSERROR("Create layer symbol link path failed"); ++ ret = -1; ++ goto out; ++ } ++ ++ if (util_clean_path(ro_symlink, clean_path, sizeof(clean_path)) == NULL) { ++ ERROR("Failed to clean path: %s", ro_symlink); ++ ret = -1; ++ goto out; ++ } ++ ++ if (util_path_remove(clean_path) != 0) { ++ SYSERROR("Failed to remove link path %s", clean_path); ++ } ++ ++ if (remove_memory_stores_with_lock(layer_id) != 0) { ++ ERROR("Failed to remove remote layer store memory"); ++ ret = -1; ++ } ++ ++out: ++ free(ro_symlink); ++ return ret; ++ ++} ++ ++static int add_one_remote_layer(struct remote_layer_data *data, char *layer_id) ++{ ++ char *ro_symlink = NULL; ++ char *layer_dir = NULL; ++ int ret = 0; ++ ++ ro_symlink = util_path_join(data->layer_home, layer_id); ++ layer_dir = util_path_join(data->layer_ro, layer_id); ++ ++ if (ro_symlink == NULL) { ++ ERROR("Failed to join ro symlink path: %s", layer_id); ++ ret = -1; ++ goto free_out; ++ } ++ ++ if (layer_dir == NULL) { ++ ERROR("Failed to join ro layer dir: %s", layer_id); ++ ret = -1; ++ goto free_out; ++ } ++ // add symbol link first ++ if (!util_fileself_exists(ro_symlink) && symlink(layer_dir, ro_symlink) != 0) { ++ SYSERROR("Unable to create symbol link to layer directory: %s", layer_dir); ++ ret = -1; ++ goto free_out; ++ } ++ // insert layer into memory ++ if (load_one_layer(layer_id) != 0) { ++ ERROR("Failed to load new layer: %s into memory", layer_id); ++ ret = -1; ++ } ++ ++free_out: ++ free(ro_symlink); ++ free(layer_dir); ++ ++ return ret; ++} ++ ++static int remote_support_add(void *data) ++{ ++ int ret = 0; ++ char **array_added = NULL; ++ char **array_deleted = NULL; ++ map_t *tmp_map = NULL; ++ int i = 0; ++ ++ if (data == NULL) { ++ return -1; ++ } ++ ++ array_added = added_layers(layer_byid_old, layer_byid_new); ++ array_deleted = deleted_layers(layer_byid_old, layer_byid_new); ++ ++ for (i = 0; i < util_array_len((const char **)array_added); i++) { ++ if (!overlay_remote_layer_valid(array_added[i]) != 0) { ++ map_remove(layer_byid_new, (void *)array_added[i]); ++ ERROR("remote overlay layer current not valid: %s", array_added[i]); ++ continue; ++ } ++ ++ if (add_one_remote_layer(data, array_added[i]) != 0) { ++ ERROR("Failed to add remote overlay layer: %s", array_added[i]); ++ ret = -1; ++ } ++ } ++ ++ for (i = 0; i < util_array_len((const char **)array_deleted); i++) { ++ if (remove_one_remote_layer(data, array_deleted[i]) != 0) { ++ ERROR("Failed to delete remote overlay layer: %s", array_deleted[i]); ++ ret = -1; ++ } ++ } ++ ++ tmp_map = layer_byid_old; ++ layer_byid_old = layer_byid_new; ++ layer_byid_new = tmp_map; ++ empty_map(layer_byid_new); ++ ++ util_free_array(array_added); ++ util_free_array(array_deleted); ++ ++ return ret; ++} ++ ++remote_support *layer_store_impl_remote_support() ++{ ++ remote_support *rs = util_common_calloc_s(sizeof(remote_support)); ++ if (rs == NULL) { ++ return NULL; ++ } ++ ++ rs->create = remote_support_create; ++ rs->destroy = remote_support_destroy; ++ rs->scan_remote_dir = remote_support_scan; ++ rs->load_item = remote_support_add; ++ ++ return rs; ++} ++ ++bool layer_remote_layer_valid(const char *layer_id) ++{ ++ return map_search(layer_byid_old, (void *)layer_id) != NULL; ++} +diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c +index 89b4f58c..c00c3356 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c ++++ b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c +@@ -30,7 +30,6 @@ + #include <stdlib.h> + #include <string.h> + #include <sys/stat.h> +- + #include <archive.h> + #include <archive_entry.h> + +@@ -48,6 +47,10 @@ + #include "http.h" + #include "utils_base64.h" + #include "constants.h" ++#include "path.h" ++#ifdef ENABLE_REMOTE_LAYER_STORE ++#include "ro_symlink_maintain.h" ++#endif + + #define PAYLOAD_CRC_LEN 12 + +@@ -74,6 +77,9 @@ typedef struct { + static layer_store_metadata g_metadata; + static char *g_root_dir; + static char *g_run_dir; ++#ifdef ENABLE_REMOTE_LAYER_STORE ++static bool g_enable_remote_layer; ++#endif + + static inline char *tar_split_path(const char *id); + static inline char *mountpoint_json_path(const char *id); +@@ -127,7 +133,7 @@ void layer_store_cleanup() + map_free(g_metadata.by_uncompress_digest); + g_metadata.by_uncompress_digest = NULL; + +- linked_list_for_each_safe(item, &(g_metadata.layers_list), next) { ++ linked_list_for_each_safe (item, &(g_metadata.layers_list), next) { + linked_list_del(item); + layer_ref_dec((layer_t *)item->elem); + free(item); +@@ -158,7 +164,7 @@ static void free_digest_layer_t(digest_layer_t *ptr) + return; + } + +- linked_list_for_each_safe(item, &(ptr->layer_list), next) { ++ linked_list_for_each_safe (item, &(ptr->layer_list), next) { + linked_list_del(item); + free(item->elem); + item->elem = NULL; +@@ -272,6 +278,10 @@ static bool init_from_conf(const struct storage_module_init_options *conf) + g_root_dir = tmp_path; + tmp_path = NULL; + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ g_enable_remote_layer = conf->enable_remote_layer; ++#endif ++ + return true; + free_out: + free(g_run_dir); +@@ -609,7 +619,7 @@ static int delete_digest_from_map(map_t *by_digest, const char *digest, const ch + return 0; + } + +- linked_list_for_each_safe(item, &(old_list->layer_list), next) { ++ linked_list_for_each_safe (item, &(old_list->layer_list), next) { + char *t_id = (char *)item->elem; + if (strcmp(t_id, id) == 0) { + linked_list_del(item); +@@ -722,7 +732,7 @@ static int remove_memory_stores(const char *id) + } + } + +- linked_list_for_each_safe(item, &(g_metadata.layers_list), next) { ++ linked_list_for_each_safe (item, &(g_metadata.layers_list), next) { + layer_t *tl = (layer_t *)item->elem; + if (strcmp(tl->slayer->id, id) != 0) { + continue; +@@ -1115,10 +1125,25 @@ static int new_layer_by_opts(const char *id, const struct layer_opts *opts) + ret = -1; + goto out; + } ++ ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ if (g_enable_remote_layer && !opts->writable) { ++ if (remote_layer_build_ro_dir(id) != 0) { ++ ret = -1; ++ goto out; ++ } ++ } else { ++ if (!build_layer_dir(id)) { ++ ret = -1; ++ goto out; ++ } ++ } ++#else + if (!build_layer_dir(id)) { + ret = -1; + goto out; + } ++#endif + + ret = update_layer_datas(id, opts, l); + if (ret != 0) { +@@ -1300,7 +1325,15 @@ clear_memory: + driver_remove: + if (ret != 0) { + (void)graphdriver_rm_layer(lid); ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ if (g_enable_remote_layer) { ++ (void)remote_layer_remove_ro_dir(lid); ++ } else { ++ (void)layer_store_remove_layer(lid); ++ } ++#else + (void)layer_store_remove_layer(lid); ++#endif + } + free_out: + layer_store_unlock(); +@@ -1375,7 +1408,15 @@ static int do_delete_layer(const char *id) + goto free_out; + } + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ if (l->slayer->writable) { ++ ret = layer_store_remove_layer(l->slayer->id); ++ } else { ++ ret = remote_layer_remove_ro_dir(l->slayer->id); ++ } ++#else + ret = layer_store_remove_layer(l->slayer->id); ++#endif + + free_out: + free(tspath); +@@ -1457,7 +1498,7 @@ int layer_store_list(struct layer_list *resp) + goto unlock; + } + +- linked_list_for_each_safe(item, &(g_metadata.layers_list), next) { ++ linked_list_for_each_safe (item, &(g_metadata.layers_list), next) { + layer_t *l = (layer_t *)item->elem; + resp->layers[i] = util_common_calloc_s(sizeof(struct layer)); + if (resp->layers[i] == NULL) { +@@ -1500,7 +1541,7 @@ static int layers_by_digest_map(map_t *m, const char *digest, struct layer_list + goto free_out; + } + +- linked_list_for_each_safe(item, &(id_list->layer_list), next) { ++ linked_list_for_each_safe (item, &(id_list->layer_list), next) { + layer_t *l = NULL; + resp->layers[i] = util_common_calloc_s(sizeof(struct layer)); + if (resp->layers[i] == NULL) { +@@ -1744,6 +1785,114 @@ out: + return ret; + } + ++static layer_t *load_one_layer_from_json(const char *id) ++{ ++ int nret = 0; ++ char *mount_point_path = NULL; ++ char tmpdir[PATH_MAX] = { 0 }; ++ char *rpath = NULL; ++ layer_t *l = NULL; ++ bool layer_valid = false; ++ ++ nret = snprintf(tmpdir, PATH_MAX, "%s/%s", g_root_dir, id); ++ if (nret < 0 || nret >= PATH_MAX) { ++ ERROR("Sprintf: %s failed", id); ++ goto free_out; ++ } ++ ++ mount_point_path = mountpoint_json_path(id); ++ if (mount_point_path == NULL) { ++ ERROR("Out of Memory"); ++ goto free_out; ++ } ++ ++ rpath = layer_json_path(id); ++ if (rpath == NULL) { ++ ERROR("%s is invalid layer", id); ++ goto free_out; ++ } ++ ++ l = load_layer(rpath, mount_point_path); ++ if (l == NULL) { ++ ERROR("load layer: %s failed, remove it", id); ++ goto free_out; ++ } ++ ++ if (do_validate_image_layer(tmpdir, l) != 0) { ++ ERROR("%s is invalid image layer", id); ++ goto free_out; ++ } ++ ++ if (do_validate_rootfs_layer(l) != 0) { ++ ERROR("%s is invalid rootfs layer", id); ++ goto free_out; ++ } ++ ++ layer_valid = true; ++ ++free_out: ++ free(rpath); ++ free(mount_point_path); ++ if (!layer_valid) { ++ free_layer_t(l); ++ l = NULL; ++ } ++ // always return true; ++ // if load layer failed, just remove it ++ return l; ++} ++ ++int load_one_layer(const char *id) ++{ ++ int ret = 0; ++ layer_t *tl = NULL; ++ int i = 0; ++ ++ if (!layer_store_lock(true)) { ++ return -1; ++ } ++ ++ tl = load_one_layer_from_json(id); ++ if (tl == NULL) { ++ ret = -1; ++ goto unlock_out; ++ } ++ ++ if (!map_insert(g_metadata.by_id, (void *)tl->slayer->id, (void *)tl)) { ++ ERROR("Insert id: %s for layer failed", tl->slayer->id); ++ ret = -1; ++ goto unlock_out; ++ } ++ ++ for (; i < tl->slayer->names_len; i++) { ++ // this should be done by master isulad ++ // if (remove_name(tl->slayer->names[i])) { ++ // should_save = true; ++ // } ++ if (!map_insert(g_metadata.by_name, (void *)tl->slayer->names[i], (void *)tl)) { ++ ret = -1; ++ ERROR("Insert name: %s for layer failed", tl->slayer->names[i]); ++ goto unlock_out; ++ } ++ } ++ ret = insert_digest_into_map(g_metadata.by_compress_digest, tl->slayer->compressed_diff_digest, tl->slayer->id); ++ if (ret != 0) { ++ ERROR("update layer: %s compress failed", tl->slayer->id); ++ goto unlock_out; ++ } ++ ++ ret = insert_digest_into_map(g_metadata.by_uncompress_digest, tl->slayer->diff_digest, tl->slayer->id); ++ if (ret != 0) { ++ ERROR("update layer: %s uncompress failed", tl->slayer->id); ++ goto unlock_out; ++ } ++ ++ ret = 0; ++unlock_out: ++ layer_store_unlock(); ++ return ret; ++} ++ + static bool load_layer_json_cb(const char *path_name, const struct dirent *sub_dir, void *context) + { + #define LAYER_NAME_LEN 64 +@@ -1760,6 +1909,14 @@ static bool load_layer_json_cb(const char *path_name, const struct dirent *sub_d + goto free_out; + } + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ // skip RO dir ++ // otherwise, RO dir will be treat as invalid layer dir ++ if (strcmp(sub_dir->d_name, REMOTE_RO_LAYER_DIR) == 0) { ++ goto free_out; ++ } ++#endif ++ + if (!util_dir_exists(tmpdir)) { + // ignore non-dir + DEBUG("%s is not directory", sub_dir->d_name); +@@ -1846,7 +2003,7 @@ static int load_layers_from_json_files() + goto unlock_out; + } + +- linked_list_for_each_safe(item, &(g_metadata.layers_list), next) { ++ linked_list_for_each_safe (item, &(g_metadata.layers_list), next) { + layer_t *tl = (layer_t *)item->elem; + size_t i = 0; + +@@ -1953,6 +2110,13 @@ int layer_store_init(const struct storage_module_init_options *conf) + goto free_out; + } + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ if (g_enable_remote_layer && remote_layer_init(g_root_dir) != 0) { ++ ERROR("Failed to init layer remote"); ++ goto free_out; ++ } ++#endif ++ + if (load_layers_from_json_files() != 0) { + goto free_out; + } +@@ -2125,7 +2289,7 @@ static tar_split *new_tar_split(layer_t *l, const char *tspath) + int ret = 0; + int nret = 0; + tar_split *ts = NULL; +- char path[PATH_MAX] = {0}; ++ char path[PATH_MAX] = { 0 }; + + ts = util_common_calloc_s(sizeof(tar_split)); + if (ts == NULL) { +@@ -2308,3 +2472,20 @@ container_inspect_graph_driver *layer_store_get_metadata_by_layer_id(const char + { + return graphdriver_get_metadata(id); + } ++ ++#ifdef ENABLE_REMOTE_LAYER_STORE ++int remove_memory_stores_with_lock(const char *id) ++{ ++ int ret = 0; ++ ++ if (!layer_store_lock(true)) { ++ ERROR("Failed to lock layer store when handle: %s", id); ++ return -1; ++ } ++ ++ ret = remove_memory_stores(id); ++ layer_store_unlock(); ++ ++ return ret; ++} ++#endif +diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_store.h b/src/daemon/modules/image/oci/storage/layer_store/layer_store.h +index 94d4bf04..44bd297e 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/layer_store.h ++++ b/src/daemon/modules/image/oci/storage/layer_store/layer_store.h +@@ -23,6 +23,10 @@ + + #include "storage.h" + #include "io_wrapper.h" ++#include "map.h" ++#ifdef ENABLE_REMOTE_LAYER_STORE ++#include "remote_support.h" ++#endif + + struct io_read_wrapper; + struct layer_list; +@@ -79,6 +83,13 @@ int layer_store_check(const char *id); + + container_inspect_graph_driver *layer_store_get_metadata_by_layer_id(const char *id); + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++remote_support *layer_store_impl_remote_support(); ++bool layer_remote_layer_valid(const char *layer_id); ++int load_one_layer(const char *id); ++int remove_memory_stores_with_lock(const char *id); ++#endif ++ + #ifdef __cplusplus + } + #endif +diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/CMakeLists.txt b/src/daemon/modules/image/oci/storage/remote_layer_support/CMakeLists.txt +new file mode 100644 +index 00000000..06c78678 +--- /dev/null ++++ b/src/daemon/modules/image/oci/storage/remote_layer_support/CMakeLists.txt +@@ -0,0 +1,12 @@ ++# get current directory sources files ++aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_remote_layer_support_srcs) ++ ++set(REMOTE_LAYER_SUPPORT_SRCS ++ ${local_remote_layer_support_srcs} ++ PARENT_SCOPE ++ ) ++ ++set(REMOTE_LAYER_SUPPORT_INCS ++ ${CMAKE_CURRENT_SOURCE_DIR} ++ PARENT_SCOPE ++) +\ No newline at end of file +diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c +new file mode 100644 +index 00000000..9dc096f7 +--- /dev/null ++++ b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c +@@ -0,0 +1,122 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: wangrunze ++ * Create: 2023-03-03 ++ * Description: provide image store functions ++ ******************************************************************************/ ++ ++#include "remote_support.h" ++ ++#include "layer_store.h" ++#include "image_store.h" ++#include "isula_libutils/log.h" ++#include "driver_overlay2.h" ++#include "utils.h" ++ ++remote_supporter *create_layer_supporter(const char *remote_home, const char *remote_ro) ++{ ++ remote_support *handlers = layer_store_impl_remote_support(); ++ if (handlers == NULL || handlers->create == NULL) { ++ return NULL; ++ } ++ ++ remote_supporter *supporter = (remote_supporter *)util_common_calloc_s(sizeof(remote_supporter)); ++ if (supporter == NULL) { ++ goto err_out; ++ } ++ ++ supporter->handlers = handlers; ++ supporter->data = handlers->create(remote_home, remote_ro); ++ ++ return supporter; ++ ++err_out: ++ free(handlers); ++ free(supporter); ++ return NULL; ++} ++ ++remote_supporter *create_image_supporter(const char *remote_home, const char *remote_ro) ++{ ++ remote_support *handlers = image_store_impl_remote_support(); ++ if (handlers == NULL || handlers->create == NULL) { ++ return NULL; ++ } ++ ++ remote_supporter *supporter = (remote_supporter *)util_common_calloc_s(sizeof(remote_supporter)); ++ if (supporter == NULL) { ++ goto err_out; ++ } ++ ++ supporter->handlers = handlers; ++ supporter->data = handlers->create(remote_home, remote_ro); ++ ++ return supporter; ++ ++err_out: ++ free(handlers); ++ free(supporter); ++ return NULL; ++} ++ ++remote_supporter *create_overlay_supporter(const char *remote_home, const char *remote_ro) ++{ ++ remote_support *handlers = overlay_driver_impl_remote_support(); ++ if (handlers == NULL || handlers->create == NULL) { ++ return NULL; ++ } ++ ++ remote_supporter *supporter = (remote_supporter *)util_common_calloc_s(sizeof(remote_supporter)); ++ if (supporter == NULL) { ++ goto err_out; ++ } ++ ++ supporter->handlers = handlers; ++ supporter->data = handlers->create(remote_home, remote_ro); ++ ++ return supporter; ++ ++err_out: ++ free(handlers); ++ free(supporter); ++ return NULL; ++ ++} ++ ++void destroy_suppoter(remote_supporter *supporter) ++{ ++ if (supporter->handlers->destroy == NULL) { ++ ERROR("destroy_supporter operation not supported"); ++ return; ++ } ++ ++ supporter->handlers->destroy(supporter->data); ++ free(supporter->handlers); ++ free(supporter); ++} ++ ++int scan_remote_dir(remote_supporter *supporter) ++{ ++ if (supporter->handlers->scan_remote_dir == NULL) { ++ ERROR("scan_remote_dir operation not supported"); ++ return -1; ++ } ++ return supporter->handlers->scan_remote_dir(supporter->data); ++} ++ ++int load_item(remote_supporter *supporter) ++{ ++ if (supporter->handlers->scan_remote_dir == NULL) { ++ ERROR("load_item operation not supported"); ++ return -1; ++ } ++ return supporter->handlers->load_item(supporter->data); ++} +diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h +new file mode 100644 +index 00000000..d1f7af35 +--- /dev/null ++++ b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h +@@ -0,0 +1,58 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2020-2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: wangrunze ++ * Create: 2023-03-03 ++ * Description: provide remote support functions ++ ******************************************************************************/ ++ ++#ifndef DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_REMOTE_SUPPORT_H ++#define DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_REMOTE_SUPPORT_H ++ ++#include "linked_list.h" ++#define REMOTE_RO_LAYER_DIR "RO" ++#define OVERLAY_RO_DIR "RO" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++typedef struct { ++ void *(*create)(const char *remote_home, const char *remote_ro); ++ void (*destroy)(void *data); ++ // populate the list contains all dirs ++ int (*scan_remote_dir)(void *data); ++ // consume the list contains all dirs ++ int (*load_item)(void *data); ++} remote_support; ++ ++typedef struct { ++ void *data; ++ remote_support *handlers; ++} remote_supporter; ++ ++// RemoteSupport *impl_remote_support(); ++remote_supporter *create_image_supporter(const char *remote_home, const char *remote_ro); ++ ++remote_supporter *create_layer_supporter(const char *remote_home, const char *remote_ro); ++ ++remote_supporter *create_overlay_supporter(const char *remote_home, const char *remote_ro); ++ ++void destroy_suppoter(remote_supporter *supporter); ++ ++int scan_remote_dir(remote_supporter *supporter); ++ ++int load_item(remote_supporter *supporter); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif +diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c b/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c +new file mode 100644 +index 00000000..7df7a221 +--- /dev/null ++++ b/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c +@@ -0,0 +1,347 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: wangrunze ++ * Create: 2023-01-12 ++ * Description: provide remote symlink maintain functions ++ ******************************************************************************/ ++#define _GNU_SOURCE ++#include "ro_symlink_maintain.h" ++ ++#include <sys/prctl.h> ++#include <stdio.h> ++#include <pthread.h> ++#include <unistd.h> ++#include "map.h" ++#include "path.h" ++#include "linked_list.h" ++#include "layer_store.h" ++#include "layer.h" ++#include "isula_libutils/log.h" ++#include "image_store.h" ++#include "remote_support.h" ++#include "utils.h" ++#include "utils_file.h" ++#include "stdlib.h" ++ ++#define REMOTE_RO_LAYER_DIR "RO" ++ ++// overlay-layers and overlay-layers/RO ++static char *image_home; ++ ++static char *layer_ro_dir; ++static char *layer_home; ++ ++// overlay and overlay/RO ++static char *overlay_ro_dir; ++static char *overlay_home; ++ ++struct supporters { ++ remote_supporter *image_supporter; ++ remote_supporter *layer_supporter; ++ remote_supporter *overlay_supporter; ++}; ++ ++static struct supporters supporters; ++ ++int remote_image_init(const char *root_dir) ++{ ++ if (root_dir == NULL) { ++ goto out; ++ } ++ ++ image_home = util_strdup_s(root_dir); ++ if (image_home == NULL) { ++ ERROR("Failed create path for remote image home"); ++ goto out; ++ } ++ return 0; ++ ++out: ++ remote_maintain_cleanup(); ++ return -1; ++} ++ ++int remote_layer_init(const char *root_dir) ++{ ++ if (root_dir == NULL) { ++ goto out; ++ } ++ ++ layer_home = util_strdup_s(root_dir); ++ layer_ro_dir = util_path_join(root_dir, REMOTE_RO_LAYER_DIR); ++ if (layer_ro_dir == NULL) { ++ ERROR("Failed join path when init remote layer maintainer"); ++ goto out; ++ } ++ if (!util_file_exists(layer_ro_dir) && util_mkdir_p(layer_ro_dir, 0700) != 0) { ++ ERROR("Failed to create RO dir under overlay"); ++ goto out; ++ } ++ ++ return 0; ++ ++out: ++ remote_maintain_cleanup(); ++ return -1; ++} ++ ++int remote_overlay_init(const char *driver_home) ++{ ++ if (driver_home == NULL) { ++ goto out; ++ } ++ ++ overlay_home = util_strdup_s(driver_home); ++ overlay_ro_dir = util_path_join(driver_home, REMOTE_RO_LAYER_DIR); ++ if (overlay_ro_dir == NULL) { ++ ERROR("Failed to join path when init remote maintainer"); ++ goto out; ++ } ++ // build RO dir if not exist ++ if (!util_file_exists(overlay_ro_dir) && util_mkdir_p(overlay_ro_dir, 0700) != 0) { ++ ERROR("Failed to create RO dir under overlay"); ++ goto out; ++ } ++ ++ return 0; ++ ++out: ++ remote_maintain_cleanup(); ++ return -1; ++} ++ ++void remote_maintain_cleanup(void) ++{ ++ free(image_home); ++ image_home = NULL; ++ ++ free(layer_home); ++ layer_home = NULL; ++ free(layer_ro_dir); ++ layer_ro_dir = NULL; ++ free(overlay_home); ++ ++ overlay_home = NULL; ++ free(overlay_ro_dir); ++ overlay_ro_dir = NULL; ++} ++ ++// to maintain the symbol links, add new symbol link and delete invalid symbol link ++// arg is const char *driver_home ++// scanning driver->home/RO/ directory, build symlink in driver->home ++static void *remote_refresh_ro_symbol_link(void *arg) ++{ ++ struct supporters *supporters = (struct supporters *)arg; ++ prctl(PR_SET_NAME, "RoLayerRefresh"); ++ ++ while (true) { ++ util_usleep_nointerupt(5 * 1000 * 1000); ++ DEBUG("remote refresh start\n"); ++ scan_remote_dir(supporters->overlay_supporter); ++ load_item(supporters->overlay_supporter); ++ scan_remote_dir(supporters->layer_supporter); ++ load_item(supporters->layer_supporter); ++ scan_remote_dir(supporters->image_supporter); ++ load_item(supporters->image_supporter); ++ DEBUG("remote refresh end\n"); ++ } ++ return NULL; ++} ++ ++int start_refresh_thread(void) ++{ ++ int res = 0; ++ pthread_t a_thread; ++ ++ supporters.image_supporter = create_image_supporter(image_home, NULL); ++ if (supporters.image_supporter == NULL) { ++ goto free_out; ++ } ++ ++ supporters.layer_supporter = create_layer_supporter(layer_home, layer_ro_dir); ++ if (supporters.layer_supporter == NULL) { ++ goto free_out; ++ } ++ ++ supporters.overlay_supporter = create_overlay_supporter(overlay_home, overlay_ro_dir); ++ if (supporters.overlay_supporter == NULL) { ++ goto free_out; ++ } ++ ++ res = pthread_create(&a_thread, NULL, remote_refresh_ro_symbol_link, (void *)&supporters); ++ if (res != 0) { ++ CRIT("Thread creation failed"); ++ return -1; ++ } ++ ++ if (pthread_detach(a_thread) != 0) { ++ SYSERROR("Failed to detach 0x%lx", a_thread); ++ return -1; ++ } ++ ++ return 0; ++ ++free_out: ++ destroy_suppoter(supporters.image_supporter); ++ destroy_suppoter(supporters.layer_supporter); ++ destroy_suppoter(supporters.overlay_supporter); ++ ++ return -1; ++} ++ ++static int do_build_ro_dir(const char *home, const char *id) ++{ ++ char *ro_symlink = NULL; ++ char *ro_layer_dir = NULL; ++ int nret = 0; ++ // bool ret = true; ++ int ret = 0; ++ ++ nret = asprintf(&ro_symlink, "%s/%s", home, id); ++ if (nret < 0 || nret > PATH_MAX) { ++ SYSERROR("Failed create ro layer dir sym link path"); ++ return -1; ++ } ++ ++ nret = asprintf(&ro_layer_dir, "%s/%s/%s", home, REMOTE_RO_LAYER_DIR, id); ++ if (nret < 0 || nret > PATH_MAX) { ++ SYSERROR("Failed to create ro layer dir path"); ++ return -1; ++ } ++ ++ if (util_mkdir_p(ro_layer_dir, IMAGE_STORE_PATH_MODE) != 0) { ++ ret = -1; ++ ERROR("Failed to create layer direcotry %s", ro_layer_dir); ++ goto out; ++ } ++ ++ if (symlink(ro_layer_dir, ro_symlink) != 0) { ++ ret = -1; ++ SYSERROR("Failed to create symlink to layer dir %s", ro_layer_dir); ++ goto err_out; ++ } ++ ++ goto out; ++ ++err_out: ++ if (util_recursive_rmdir(ro_layer_dir, 0)) { ++ ERROR("Failed to delete layer path: %s", ro_layer_dir); ++ } ++ ++out: ++ free(ro_layer_dir); ++ free(ro_symlink); ++ return ret; ++} ++ ++int remote_overlay_build_ro_dir(const char *id) ++{ ++ return do_build_ro_dir(overlay_home, id); ++} ++ ++int remote_layer_build_ro_dir(const char *id) ++{ ++ return do_build_ro_dir(layer_home, id); ++} ++ ++int do_remove_ro_dir(const char *home, const char *id) ++{ ++ char *ro_layer_dir = NULL; ++ char *ro_symlink = NULL; ++ char clean_path[PATH_MAX] = { 0 }; ++ int ret = 0; ++ int nret = 0; ++ ++ if (id == NULL) { ++ return 0; ++ } ++ ++ nret = asprintf(&ro_symlink, "%s/%s", home, id); ++ if (nret < 0 || nret > PATH_MAX) { ++ SYSERROR("Create layer sym link path failed"); ++ return -1; ++ } ++ ++ if (util_clean_path(ro_symlink, clean_path, sizeof(clean_path)) == NULL) { ++ ERROR("Failed to clean path: %s", ro_symlink); ++ ret = -1; ++ goto out; ++ } ++ ++ if (util_path_remove(clean_path) != 0) { ++ SYSERROR("Failed to remove link path %s", clean_path); ++ } ++ ++ nret = asprintf(&ro_layer_dir, "%s/%s/%s", home, REMOTE_RO_LAYER_DIR, id); ++ if (nret < 0 || nret > PATH_MAX) { ++ SYSERROR("Create layer json path failed"); ++ ret = -1; ++ goto out; ++ } ++ ++ ret = util_recursive_rmdir(ro_layer_dir, 0); ++ ++out: ++ free(ro_layer_dir); ++ free(ro_symlink); ++ return ret; ++} ++ ++int remote_layer_remove_ro_dir(const char *id) ++{ ++ return do_remove_ro_dir(layer_home, id); ++} ++ ++int remote_overlay_remove_ro_dir(const char *id) ++{ ++ return do_remove_ro_dir(overlay_home, id); ++} ++ ++static char **map_diff(map_t *map_a, map_t *map_b) ++{ ++ char **array = NULL; ++ map_itor *itor = map_itor_new(map_a); ++ bool *found = NULL; ++ ++ // iter new_map, every item not in old, append them to new_layers ++ for (; map_itor_valid(itor); map_itor_next(itor)) { ++ char *id = map_itor_key(itor); ++ found = map_search(map_b, id); ++ if (found == NULL) { ++ util_array_append(&array, util_strdup_s(id)); ++ } ++ } ++ ++ map_itor_free(itor); ++ ++ return array; ++} ++ ++char **deleted_layers(map_t *old, map_t *new) ++{ ++ return map_diff(old, new); ++} ++ ++char **added_layers(map_t *old, map_t *new) ++{ ++ return map_diff(new, old); ++} ++ ++int empty_map(map_t *mp) ++{ ++ if (mp == NULL) { ++ return -1; ++ } ++ ++ map_clear(mp); ++ mp->store->root = mp->store->nil; ++ return 0; ++} +diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.h b/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.h +new file mode 100644 +index 00000000..25712d40 +--- /dev/null ++++ b/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.h +@@ -0,0 +1,52 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: wangrunze ++ * Create: 2023-01-12 ++ * Description: provide remote symlink maintain functions ++ ******************************************************************************/ ++#ifndef DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_RO_SYMLINK_MAINTAIN_H ++#define DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_RO_SYMLINK_MAINTAIN_H ++ ++#include "map.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++int remote_image_init(const char *root_dir); ++ ++int remote_layer_init(const char *root_dir); ++ ++int remote_overlay_init(const char *driver_home); ++ ++void remote_maintain_cleanup(void); ++ ++int start_refresh_thread(void); ++ ++int remote_layer_build_ro_dir(const char *id); ++ ++int remote_overlay_build_ro_dir(const char *id); ++ ++int remote_layer_remove_ro_dir(const char *id); ++ ++int remote_overlay_remove_ro_dir(const char *id); ++ ++char **deleted_layers(map_t *old, map_t *new); ++ ++char **added_layers(map_t *old, map_t *new); ++ ++int empty_map(map_t *mp); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif +diff --git a/src/daemon/modules/image/oci/storage/storage.c b/src/daemon/modules/image/oci/storage/storage.c +index 2f4bdf5f..31812a22 100644 +--- a/src/daemon/modules/image/oci/storage/storage.c ++++ b/src/daemon/modules/image/oci/storage/storage.c +@@ -42,6 +42,9 @@ + #include "utils_string.h" + #include "utils_verify.h" + #include "sha256.h" ++#ifdef ENABLE_REMOTE_LAYER_STORE ++#include "ro_symlink_maintain.h" ++#endif + + static pthread_rwlock_t g_storage_rwlock; + static char *g_storage_run_root; +@@ -1870,6 +1873,12 @@ int storage_module_init(struct storage_module_init_options *opts) + goto out; + } + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ if (opts->enable_remote_layer && start_refresh_thread() != 0) { ++ ERROR("Failed to start remote refresh thread"); ++ } ++#endif ++ + if (restore_images_size() != 0) { + ERROR("Failed to recal image size"); + ret = -1; +@@ -1906,4 +1915,4 @@ out: + char *storage_rootfs_get_dir() + { + return rootfs_store_get_data_dir(); +-} +\ No newline at end of file ++} +diff --git a/src/daemon/modules/image/oci/storage/storage.h b/src/daemon/modules/image/oci/storage/storage.h +index 3ec47959..7404ee54 100644 +--- a/src/daemon/modules/image/oci/storage/storage.h ++++ b/src/daemon/modules/image/oci/storage/storage.h +@@ -70,6 +70,9 @@ struct storage_module_init_options { + char **driver_opts; + size_t driver_opts_len; + bool integration_check; ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ bool enable_remote_layer; ++#endif + }; + + struct storage_img_create_options { +-- +2.25.1 + diff --git a/0033-add-ci-for-remote-ro.patch b/0033-add-ci-for-remote-ro.patch new file mode 100644 index 0000000..88c9279 --- /dev/null +++ b/0033-add-ci-for-remote-ro.patch @@ -0,0 +1,119 @@ +From 6311ccd6e367f965da4dc1b4c9efb4bf43275f64 Mon Sep 17 00:00:00 2001 +From: Neil <wrz750726@gmail.com> +Date: Sun, 5 Mar 2023 12:23:29 +0000 +Subject: [PATCH 33/53] add ci for remote ro + +Signed-off-by: Neil <wangrunze13@huawei.com> +--- + CI/make-and-install.sh | 4 +- + .../container_cases/test_data/daemon.json | 1 + + CI/test_cases/image_cases/ro_separate.sh | 69 +++++++++++++++++++ + 3 files changed, 72 insertions(+), 2 deletions(-) + create mode 100644 CI/test_cases/image_cases/ro_separate.sh + +diff --git a/CI/make-and-install.sh b/CI/make-and-install.sh +index 3dbff480..fa9c2250 100755 +--- a/CI/make-and-install.sh ++++ b/CI/make-and-install.sh +@@ -103,9 +103,9 @@ rm -rf build + mkdir build + cd build + if [[ ${enable_gcov} -ne 0 ]]; then +- cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_UT=ON -DENABLE_METRICS=ON .. ++ cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_UT=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON .. + else +- cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DENABLE_EMBEDDED=ON -DENABLE_METRICS=ON .. ++ cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DENABLE_EMBEDDED=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON .. + fi + make -j $(nproc) + make install +diff --git a/CI/test_cases/container_cases/test_data/daemon.json b/CI/test_cases/container_cases/test_data/daemon.json +index aa88c9da..2664c6b2 100644 +--- a/CI/test_cases/container_cases/test_data/daemon.json ++++ b/CI/test_cases/container_cases/test_data/daemon.json +@@ -19,6 +19,7 @@ + "hook-spec": "/etc/default/isulad/hooks/default.json", + "start-timeout": "2m", + "storage-driver": "overlay2", ++ "storage-enable-remote-layer": false, + "storage-opts": [ + "overlay2.override_kernel_check=true" + ], +diff --git a/CI/test_cases/image_cases/ro_separate.sh b/CI/test_cases/image_cases/ro_separate.sh +new file mode 100644 +index 00000000..47e04abb +--- /dev/null ++++ b/CI/test_cases/image_cases/ro_separate.sh +@@ -0,0 +1,69 @@ ++#!/bin/bash ++# ++# attributes: isulad basic image ++# concurrent: NA ++# spend time: 22 ++ ++####################################################################### ++##- Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. ++# - iSulad licensed under the Mulan PSL v2. ++# - You can use this software according to the terms and conditions of the Mulan PSL v2. ++# - You may obtain a copy of Mulan PSL v2 at: ++# - http://license.coscl.org.cn/MulanPSL2 ++# - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++# - IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++# - PURPOSE. ++# - See the Mulan PSL v2 for more details. ++##- @Description:CI ++##- @Author: wangrunze ++##- @Create: 2023-03-03 ++####################################################################### ++ ++declare -r curr_path=$(dirname $(readlink -f "$0")) ++source ../helpers.sh ++single_image="${curr_path}/busybox.tar" ++ ++function test_separate_ro() ++{ ++ local ret=0 ++ local test="isula separate ro test => (${FUNCNAME[@]})" ++ ++ msg_info "${test} starting..." ++ ++ sed -i 's/"storage-enable-remote-layer": false/"storage-enable-remote-layer": true/' /etc/isulad/daemon.json ++ start_isulad_with_valgrind ++ wait_isulad_running ++ ++ isula rmi busybox ++ ++ isula pull busybox ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - pull image failed" && ((ret++)) ++ ++ isula run -tid --name test_separate busybox /bin/sh ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - run container failed" && ((ret++)) ++ ++ isula stop test_separate ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stop container failed" && ((ret++)) ++ ++ isula rmi busybox ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - remove image failed" && ((ret++)) ++ ++ isula load -i $single_image ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - load image failed: ${rootfs_tar}" && ((ret++)) ++ ++ check_valgrind_log ++ [[ $? -ne 0 ]] && msg_err "separate ro test - memory leak, please check...." && ((ret++)) ++ ++ sed -i 's/"storage-enable-remote-layer": true/"storage-enable-remote-layer": false/' /etc/isulad/daemon.json ++ start_isulad_with_valgrind ++ wait_isulad_running ++ ++ msg_info "${test} finished with return ${ret}..." ++ return ${ret} ++} ++ ++declare -i ans=0 ++ ++test_separate_ro || ((ans++)) ++ ++show_result ${ans} "${curr_path}/${0}" +-- +2.25.1 + diff --git a/0034-change-sleep-to-usleep-to-avoid-lossing-of-accuracy.patch b/0034-change-sleep-to-usleep-to-avoid-lossing-of-accuracy.patch new file mode 100644 index 0000000..8092e85 --- /dev/null +++ b/0034-change-sleep-to-usleep-to-avoid-lossing-of-accuracy.patch @@ -0,0 +1,26 @@ +From 9663e62598570d16c0e8a70be4341ff72663b8df Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Mon, 6 Mar 2023 09:51:17 +0800 +Subject: [PATCH 34/53] change sleep() to usleep() to avoid lossing of accuracy + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/utils/cutils/utils.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/utils/cutils/utils.c b/src/utils/cutils/utils.c +index 2c4c01e4..983d81d8 100644 +--- a/src/utils/cutils/utils.c ++++ b/src/utils/cutils/utils.c +@@ -345,7 +345,7 @@ int util_waitpid_with_timeout(pid_t pid, const int64_t timeout, handle_timeout_c + return -1; + } + // sleep some time instead to avoid cpu full running and then retry. +- sleep(0.1); ++ usleep(100); + } + return 0; + } +-- +2.25.1 + diff --git a/0035-fix-compile-error-when-not-enable-remote-ro.patch b/0035-fix-compile-error-when-not-enable-remote-ro.patch new file mode 100644 index 0000000..091c939 --- /dev/null +++ b/0035-fix-compile-error-when-not-enable-remote-ro.patch @@ -0,0 +1,62 @@ +From 6a0b11ae6584ee2eefff9bd20c96bc60582ccb6b Mon Sep 17 00:00:00 2001 +From: "Neil.wrz" <wangrunze13@huawei.com> +Date: Sun, 5 Mar 2023 18:55:40 -0800 +Subject: [PATCH 35/53] fix compile error when not enable remote ro + +Signed-off-by: Neil.wrz <wangrunze13@huawei.com> +--- + .../modules/image/oci/storage/image_store/CMakeLists.txt | 3 +++ + .../modules/image/oci/storage/layer_store/CMakeLists.txt | 4 ++++ + .../storage/layer_store/graphdriver/overlay2/CMakeLists.txt | 3 +++ + 3 files changed, 10 insertions(+) + +diff --git a/src/daemon/modules/image/oci/storage/image_store/CMakeLists.txt b/src/daemon/modules/image/oci/storage/image_store/CMakeLists.txt +index ecf21caa..7d4fb77c 100644 +--- a/src/daemon/modules/image/oci/storage/image_store/CMakeLists.txt ++++ b/src/daemon/modules/image/oci/storage/image_store/CMakeLists.txt +@@ -1,5 +1,8 @@ + # get current directory sources files + aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_image_store_srcs) ++IF (NOT ENABLE_REMOTE_LAYER_STORE) ++list(REMOVE_ITEM local_image_store_srcs "${CMAKE_CURRENT_SOURCE_DIR}/image_remote_impl.c") ++ENDIF() + + set(IMAGE_STORE_SRCS + ${local_image_store_srcs} +diff --git a/src/daemon/modules/image/oci/storage/layer_store/CMakeLists.txt b/src/daemon/modules/image/oci/storage/layer_store/CMakeLists.txt +index f964f709..e04b4ad7 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/CMakeLists.txt ++++ b/src/daemon/modules/image/oci/storage/layer_store/CMakeLists.txt +@@ -1,5 +1,8 @@ + # get current directory sources files + aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_layer_store_srcs) ++IF (NOT ENABLE_REMOTE_LAYER_STORE) ++list(REMOVE_ITEM local_layer_store_srcs "${CMAKE_CURRENT_SOURCE_DIR}/layer_remote_impl.c") ++ENDIF() + add_subdirectory(graphdriver) + + set(LAYER_STORE_SRCS +@@ -7,6 +10,7 @@ set(LAYER_STORE_SRCS + ${GRAPHDRIVER_SRCS} + PARENT_SCOPE + ) ++ + set(LAYER_STORE_INCS + ${CMAKE_CURRENT_SOURCE_DIR} + ${GRAPHDRIVER_INCS} +diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/CMakeLists.txt b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/CMakeLists.txt +index ceed16b7..dd4e82aa 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/CMakeLists.txt ++++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/CMakeLists.txt +@@ -1,5 +1,8 @@ + # get current directory sources files + aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_overlay2_srcs) ++IF (NOT ENABLE_REMOTE_LAYER_STORE) ++list(REMOVE_ITEM local_overlay2_srcs "${CMAKE_CURRENT_SOURCE_DIR}/overlay_remote_impl.c") ++ENDIF() + + set(OVERLAY2_SRCS + ${local_overlay2_srcs} +-- +2.25.1 + diff --git a/0036-adapt-to-repo-of-openeuler-url-changed.patch b/0036-adapt-to-repo-of-openeuler-url-changed.patch new file mode 100644 index 0000000..dd476a9 --- /dev/null +++ b/0036-adapt-to-repo-of-openeuler-url-changed.patch @@ -0,0 +1,33 @@ +From 42a46f64e533ea0a89f95bef80a421fe06a5bfa2 Mon Sep 17 00:00:00 2001 +From: haozi007 <liuhao27@huawei.com> +Date: Mon, 6 Mar 2023 14:38:58 +0800 +Subject: [PATCH 36/53] adapt to repo of openeuler url changed + +Signed-off-by: haozi007 <liuhao27@huawei.com> +--- + CI/pr-gateway.sh | 8 ++++++++ + 1 file changed, 8 insertions(+) + +diff --git a/CI/pr-gateway.sh b/CI/pr-gateway.sh +index c715fc39..7c696de3 100755 +--- a/CI/pr-gateway.sh ++++ b/CI/pr-gateway.sh +@@ -18,7 +18,15 @@ if [ $# -eq 1 ]; then + tbranch=$1 + fi + ++sed -i "s#http://repo.openeuler.org#https://repo.huaweicloud.com/openeuler#g" /etc/yum.repos.d/openEuler.repo ++ ++dnf update -y ++ + dnf install -y gtest-devel gmock-devel diffutils cmake gcc-c++ yajl-devel patch make libtool libevent-devel libevhtp-devel grpc grpc-plugins grpc-devel protobuf-devel libcurl libcurl-devel sqlite-devel libarchive-devel device-mapper-devel http-parser-devel libseccomp-devel libcap-devel libselinux-devel libwebsockets libwebsockets-devel systemd-devel git chrpath ++if [ $? -ne 0 ]; then ++ echo "install dependences failed" ++ exit 1 ++fi + + # dnf install -y cargo rust rust-packaging + +-- +2.25.1 + diff --git a/0037-change-goto-branch.patch b/0037-change-goto-branch.patch new file mode 100644 index 0000000..a2352f4 --- /dev/null +++ b/0037-change-goto-branch.patch @@ -0,0 +1,27 @@ +From 66532035178723175ad9522201440372316dc914 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Mon, 6 Mar 2023 17:54:55 +0800 +Subject: [PATCH 37/53] change goto branch + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/daemon/executor/container_cb/execution_stream.c | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/src/daemon/executor/container_cb/execution_stream.c b/src/daemon/executor/container_cb/execution_stream.c +index 1a7353b5..7e928cf7 100644 +--- a/src/daemon/executor/container_cb/execution_stream.c ++++ b/src/daemon/executor/container_cb/execution_stream.c +@@ -161,7 +161,8 @@ static int container_exec_cb(const container_exec_request *request, container_ex + + if (exec_container(cont, request, *response, stdinfd, stdout_handler, stderr_handler) != 0) { + ret = -1; +- goto pack_err_response; ++ // pack err response in exec_container, there is no need to pack here. ++ goto out; + } + + goto out; +-- +2.25.1 + diff --git a/0038-CI-not-enable-remote-ro-for-ut.patch b/0038-CI-not-enable-remote-ro-for-ut.patch new file mode 100644 index 0000000..cc41858 --- /dev/null +++ b/0038-CI-not-enable-remote-ro-for-ut.patch @@ -0,0 +1,41 @@ +From c414e2226542660579e14b9401fc28bc91b709c7 Mon Sep 17 00:00:00 2001 +From: "Neil.wrz" <wangrunze13@huawei.com> +Date: Mon, 6 Mar 2023 18:59:43 -0800 +Subject: [PATCH 38/53] CI not enable remote ro for ut + +Signed-off-by: Neil.wrz <wangrunze13@huawei.com> +--- + CI/make-and-install.sh | 2 +- + CI/test_cases/image_cases/ro_separate.sh | 3 +++ + 2 files changed, 4 insertions(+), 1 deletion(-) + +diff --git a/CI/make-and-install.sh b/CI/make-and-install.sh +index fa9c2250..faeaf005 100755 +--- a/CI/make-and-install.sh ++++ b/CI/make-and-install.sh +@@ -103,7 +103,7 @@ rm -rf build + mkdir build + cd build + if [[ ${enable_gcov} -ne 0 ]]; then +- cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_UT=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON .. ++ cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_UT=ON -DENABLE_METRICS=ON .. + else + cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DENABLE_EMBEDDED=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON .. + fi +diff --git a/CI/test_cases/image_cases/ro_separate.sh b/CI/test_cases/image_cases/ro_separate.sh +index 47e04abb..df45e120 100644 +--- a/CI/test_cases/image_cases/ro_separate.sh ++++ b/CI/test_cases/image_cases/ro_separate.sh +@@ -45,6 +45,9 @@ function test_separate_ro() + isula stop test_separate + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stop container failed" && ((ret++)) + ++ isula rm test_separate ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - remove container failed" && ((ret++)) ++ + isula rmi busybox + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - remove image failed" && ((ret++)) + +-- +2.25.1 + diff --git a/0039-use-auto-free-to-proc_t.patch b/0039-use-auto-free-to-proc_t.patch new file mode 100644 index 0000000..25de87e --- /dev/null +++ b/0039-use-auto-free-to-proc_t.patch @@ -0,0 +1,43 @@ +From 0a68a53ccb6582384dad478d4197ec9386306027 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Mon, 6 Mar 2023 10:56:16 +0800 +Subject: [PATCH 39/53] use auto free to proc_t + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/daemon/modules/runtime/isula/isula_rt_ops.c | 12 ++---------- + 1 file changed, 2 insertions(+), 10 deletions(-) + +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index 536e36b2..5ec0e639 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -951,8 +951,8 @@ int rt_isula_start(const char *id, const char *runtime, const rt_start_params_t + pid_t shim_pid = -1; + int ret = -1; + int splice_ret = 0; +- proc_t *proc = NULL; +- proc_t *p_proc = NULL; ++ __isula_auto_free proc_t *proc = NULL; ++ __isula_auto_free proc_t *p_proc = NULL; + + if (id == NULL || runtime == NULL || params == NULL || pid_info == NULL) { + ERROR("nullptr arguments not allowed"); +@@ -1009,14 +1009,6 @@ out: + show_shim_runtime_errlog(workdir); + shim_kill_force(workdir); + } +- +- if (proc != NULL) { +- free(proc); +- } +- if (p_proc != NULL) { +- free(p_proc); +- } +- + return ret; + } + +-- +2.25.1 + diff --git a/0040-modifying-cpurt-file-permissions.patch b/0040-modifying-cpurt-file-permissions.patch new file mode 100644 index 0000000..8eb5455 --- /dev/null +++ b/0040-modifying-cpurt-file-permissions.patch @@ -0,0 +1,72 @@ +From 2d9b9d88f3150027609fe984930af1dcf06dfd00 Mon Sep 17 00:00:00 2001 +From: songbuhuang <544824346@qq.com> +Date: Thu, 9 Mar 2023 18:32:19 +0800 +Subject: [PATCH 40/53] modifying cpurt file permissions + +Signed-off-by: songbuhuang <544824346@qq.com> +--- + src/common/constants.h | 4 ++++ + src/daemon/common/sysinfo.c | 3 +-- + src/daemon/executor/container_cb/execution.c | 3 ++- + 3 files changed, 7 insertions(+), 3 deletions(-) + +diff --git a/src/common/constants.h b/src/common/constants.h +index b43d8c80..06015f1e 100644 +--- a/src/common/constants.h ++++ b/src/common/constants.h +@@ -22,6 +22,10 @@ extern "C" { + + /* mode of file and directory */ + ++#define DEFAULT_CGROUP_FILE_MODE 0644 ++ ++#define DEFAULT_CGROUP_DIR_MODE 0755 ++ + #define DEFAULT_SECURE_FILE_MODE 0640 + + #define DEFAULT_SECURE_DIRECTORY_MODE 0750 +diff --git a/src/daemon/common/sysinfo.c b/src/daemon/common/sysinfo.c +index 7559d653..baf53510 100644 +--- a/src/daemon/common/sysinfo.c ++++ b/src/daemon/common/sysinfo.c +@@ -27,6 +27,7 @@ + #include <isula_libutils/auto_cleanup.h> + #include <isula_libutils/log.h> + ++#include "constants.h" + #include "err_msg.h" + #include "utils.h" + #include "utils_array.h" +@@ -73,8 +74,6 @@ + + #define CGROUP_MOUNTPOINT "/sys/fs/cgroup" + #define CGROUP_ISULAD_PATH CGROUP_MOUNTPOINT"/isulad" +-#define DEFAULT_CGROUP_DIR_MODE 0755 +-#define DEFAULT_CGROUP_FILE_MODE 0644 + #define CGROUP2_CONTROLLERS_PATH CGROUP_MOUNTPOINT"/cgroup.controllers" + #define CGROUP2_SUBTREE_CONTROLLER_PATH CGROUP_MOUNTPOINT"/cgroup.subtree_control" + #define CGROUP2_CPUSET_CPUS_EFFECTIVE_PATH CGROUP_MOUNTPOINT"/cpuset.cpus.effective" +diff --git a/src/daemon/executor/container_cb/execution.c b/src/daemon/executor/container_cb/execution.c +index 198052d3..c2a0bdb9 100644 +--- a/src/daemon/executor/container_cb/execution.c ++++ b/src/daemon/executor/container_cb/execution.c +@@ -46,6 +46,7 @@ + #include "isulad_config.h" + #include "sysinfo.h" + #include "container_api.h" ++#include "constants.h" + #include "specs_api.h" + #include "execution_extend.h" + #include "execution_information.h" +@@ -324,7 +325,7 @@ static int maybe_create_cpu_realtime_file(int64_t value, const char *file, const + return 0; + } + +- ret = util_mkdir_p(path, CONFIG_DIRECTORY_MODE); ++ ret = util_mkdir_p(path, DEFAULT_CGROUP_DIR_MODE); + if (ret != 0) { + ERROR("Failed to mkdir: %s", path); + return -1; +-- +2.25.1 + diff --git a/0041-use-CURLOPT_XFERINFOFUNCTION-instead-of-deprecated-C.patch b/0041-use-CURLOPT_XFERINFOFUNCTION-instead-of-deprecated-C.patch new file mode 100644 index 0000000..af12eec --- /dev/null +++ b/0041-use-CURLOPT_XFERINFOFUNCTION-instead-of-deprecated-C.patch @@ -0,0 +1,131 @@ +From d0b0baa3f2624b6de0ca92c051c154f0cff43f1a Mon Sep 17 00:00:00 2001 +From: zhangxiaoyu <zhangxiaoyu58@huawei.com> +Date: Tue, 14 Mar 2023 10:33:38 +0800 +Subject: [PATCH 41/53] use CURLOPT_XFERINFOFUNCTION instead of deprecated + CURLOPT_PROGRESSFUNCTION since curl 7.32.0 + +Signed-off-by: zhangxiaoyu <zhangxiaoyu58@huawei.com> +--- + .../modules/image/oci/registry/http_request.c | 12 +++++++++++ + src/utils/http/http.c | 21 ++++++++++++++----- + src/utils/http/http.h | 7 +++++++ + test/image/oci/registry/registry_ut.cc | 8 +++++++ + 4 files changed, 43 insertions(+), 5 deletions(-) + +diff --git a/src/daemon/modules/image/oci/registry/http_request.c b/src/daemon/modules/image/oci/registry/http_request.c +index f29c2017..ce8b7667 100644 +--- a/src/daemon/modules/image/oci/registry/http_request.c ++++ b/src/daemon/modules/image/oci/registry/http_request.c +@@ -691,6 +691,16 @@ static int progress(void *p, double dltotal, double dlnow, double ultotal, doubl + return 0; + } + ++static int xfer(void *p, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) ++{ ++ bool *cancel = p; ++ if (*cancel) { ++ // return nonzero code means abort transition ++ return -1; ++ } ++ return 0; ++} ++ + int http_request_file(pull_descriptor *desc, const char *url, const char **custom_headers, char *file, + resp_data_type type, CURLcode *errcode) + { +@@ -721,6 +731,8 @@ int http_request_file(pull_descriptor *desc, const char *url, const char **custo + options->show_progress = 1; + options->progressinfo = &desc->cancel; + options->progress_info_op = progress; ++ options->xferinfo = &desc->cancel; ++ options->xferinfo_op = xfer; + options->timeout = true; + + ret = setup_common_options(desc, options, url, custom_headers); +diff --git a/src/utils/http/http.c b/src/utils/http/http.c +index bf163d86..986f1f0d 100644 +--- a/src/utils/http/http.c ++++ b/src/utils/http/http.c +@@ -219,12 +219,23 @@ static void http_custom_general_options(CURL *curl_handle, const struct http_get + /* disable progress meter, set to 0L to enable and disable debug output */ + if (options->show_progress == 0) { + curl_easy_setopt(curl_handle, CURLOPT_NOPROGRESS, 1L); +- } else if (options->show_progress && options->progressinfo && options->progress_info_op) { +- curl_easy_setopt(curl_handle, CURLOPT_PROGRESSFUNCTION, options->progress_info_op); +- /* pass the struct pointer into the progress function */ +- curl_easy_setopt(curl_handle, CURLOPT_PROGRESSDATA, options->progressinfo); +- curl_easy_setopt(curl_handle, CURLOPT_NOPROGRESS, 0L); + } else { ++ /* libcurl support option CURLOPT_XFERINFOFUNCTION when version >= 7.32.0 ++ * #define CURL_VERSION_BITS(x,y,z) ((x)<<16|(y)<<8|(z)) ++ * CURL_VERSION_BITS(7,32,0) = 0x072000 */ ++#if (LIBCURL_VERSION_NUM >= 0x072000) ++ if (options->xferinfo && options->xferinfo_op) { ++ curl_easy_setopt(curl_handle, CURLOPT_XFERINFOFUNCTION, options->xferinfo_op); ++ /* pass the struct pointer into the progress function */ ++ curl_easy_setopt(curl_handle, CURLOPT_XFERINFODATA, options->xferinfo); ++ } ++#else ++ if (options->progressinfo && options->progress_info_op) { ++ curl_easy_setopt(curl_handle, CURLOPT_PROGRESSFUNCTION, options->progress_info_op); ++ /* pass the struct pointer into the progress function */ ++ curl_easy_setopt(curl_handle, CURLOPT_PROGRESSDATA, options->progressinfo); ++ } ++#endif + curl_easy_setopt(curl_handle, CURLOPT_NOPROGRESS, 0L); + } + +diff --git a/src/utils/http/http.h b/src/utils/http/http.h +index 343d92c3..cdd6d64f 100644 +--- a/src/utils/http/http.h ++++ b/src/utils/http/http.h +@@ -17,6 +17,7 @@ + + #include <stdbool.h> + #include <stddef.h> ++#include <curl/curl.h> + + #ifdef __cplusplus + extern "C" { +@@ -25,6 +26,9 @@ extern "C" { + typedef int(*progress_info_func)(void *p, + double dltotal, double dlnow, + double ultotal, double ulnow); ++typedef int(*xferinfo_func)(void *p, ++ curl_off_t dltotal, curl_off_t dlnow, ++ curl_off_t ultotal, curl_off_t ulnow); + + struct http_get_options { + unsigned with_head : 1, /* if set, means write output with response HEADER */ +@@ -77,6 +81,9 @@ struct http_get_options { + + void *progressinfo; + progress_info_func progress_info_op; ++ ++ void *xferinfo; ++ xferinfo_func xferinfo_op; + }; + + #define HTTP_RES_OK 0 +diff --git a/test/image/oci/registry/registry_ut.cc b/test/image/oci/registry/registry_ut.cc +index 8d9ea92b..f4f8a763 100644 +--- a/test/image/oci/registry/registry_ut.cc ++++ b/test/image/oci/registry/registry_ut.cc +@@ -221,6 +221,14 @@ int invokeHttpRequestV2(const char *url, struct http_get_options *options, long + if (options->progress_info_op(options->progressinfo, 0, 0, 0, 0) != 0) { + return -1; + } ++ ++ cancel = (bool *)options->xferinfo; ++ while (!(*cancel)) { ++ sleep(1); // schedule out to let cancel variable set to be true ++ } ++ if (options->xferinfo_op(options->xferinfo, 0, 0, 0, 0) != 0) { ++ return -1; ++ } + } + } else if (util_has_prefix(url, "http://hub-mirror.c.163.com/v2/library/busybox/blobs/sha256:91f30d77")) { + if (retry) { +-- +2.25.1 + diff --git a/0042-bugfix-remote-ro-try-add-or-remove-image-layer-twice.patch b/0042-bugfix-remote-ro-try-add-or-remove-image-layer-twice.patch new file mode 100644 index 0000000..2741ecf --- /dev/null +++ b/0042-bugfix-remote-ro-try-add-or-remove-image-layer-twice.patch @@ -0,0 +1,99 @@ +From a08e511eb6ee1955bb62e774190dfe7a7599fbdd Mon Sep 17 00:00:00 2001 +From: "Neil.wrz" <wangrunze13@huawei.com> +Date: Tue, 7 Mar 2023 23:59:56 -0800 +Subject: [PATCH 42/53] bugfix remote ro try add or remove image/layer twice + +Signed-off-by: Neil.wrz <wangrunze13@huawei.com> +--- + .../image/oci/storage/image_store/image_store.c | 14 ++++++++++++++ + .../oci/storage/layer_store/layer_remote_impl.c | 2 +- + .../image/oci/storage/layer_store/layer_store.c | 11 +++++++++++ + 3 files changed, 26 insertions(+), 1 deletion(-) + +diff --git a/src/daemon/modules/image/oci/storage/image_store/image_store.c b/src/daemon/modules/image/oci/storage/image_store/image_store.c +index caff3705..84187ded 100644 +--- a/src/daemon/modules/image/oci/storage/image_store/image_store.c ++++ b/src/daemon/modules/image/oci/storage/image_store/image_store.c +@@ -3668,6 +3668,11 @@ int append_image_by_directory_with_lock(const char *id) + return -1; + } + ++ if (map_search(g_image_store->byid, (void *)id) != NULL ) { ++ DEBUG("remote image already exist, not added: %s", id); ++ goto out; ++ } ++ + nret = snprintf(image_path, sizeof(image_path), "%s/%s", g_image_store->dir, id); + if (nret < 0 || (size_t)nret >= sizeof(image_path)) { + ERROR("Failed to get image path"); +@@ -3675,6 +3680,8 @@ int append_image_by_directory_with_lock(const char *id) + } + + ret = append_image_by_directory(image_path); ++ ++out: + image_store_unlock(); + + return ret; +@@ -3689,7 +3696,14 @@ int remove_image_from_memory_with_lock(const char *id) + return -1; + } + ++ if (map_search(g_image_store->byid, (void *)id) == NULL) { ++ DEBUG("remote image already remvoed, don't delete twice: %s", id); ++ goto out; ++ } ++ + ret = remove_image_from_memory(id); ++ ++out: + image_store_unlock(); + + return ret; +diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c b/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c +index d03fc20b..d676458c 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c ++++ b/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c +@@ -175,7 +175,7 @@ static int remote_support_add(void *data) + } + + if (add_one_remote_layer(data, array_added[i]) != 0) { +- ERROR("Failed to add remote overlay layer: %s", array_added[i]); ++ ERROR("Failed to add remote layer: %s", array_added[i]); + ret = -1; + } + } +diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c +index c00c3356..e88067bc 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c ++++ b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c +@@ -1852,6 +1852,11 @@ int load_one_layer(const char *id) + return -1; + } + ++ if (map_search(g_metadata.by_id, (void *)id) != NULL) { ++ DEBUG("remote layer already exist, not added: %s", id); ++ goto unlock_out; ++ } ++ + tl = load_one_layer_from_json(id); + if (tl == NULL) { + ret = -1; +@@ -2482,8 +2487,14 @@ int remove_memory_stores_with_lock(const char *id) + ERROR("Failed to lock layer store when handle: %s", id); + return -1; + } ++ if (map_search(g_metadata.by_id, (void *)id) == NULL) { ++ DEBUG("remote layer already removed, don't delete: %s", id); ++ goto unlock_out; ++ } + + ret = remove_memory_stores(id); ++ ++unlock_out: + layer_store_unlock(); + + return ret; +-- +2.25.1 + diff --git a/0043-bugfix-can-t-delete-layers-under-dir-overlay-layers.patch b/0043-bugfix-can-t-delete-layers-under-dir-overlay-layers.patch new file mode 100644 index 0000000..a4d9296 --- /dev/null +++ b/0043-bugfix-can-t-delete-layers-under-dir-overlay-layers.patch @@ -0,0 +1,34 @@ +From c8702b62fd6016a96794d74abcae2e74551a9c07 Mon Sep 17 00:00:00 2001 +From: "Neil.wrz" <wangrunze13@huawei.com> +Date: Tue, 14 Mar 2023 20:32:23 -0700 +Subject: [PATCH 43/53] bugfix can't delete layers under dir overlay-layers + +Signed-off-by: Neil.wrz <wangrunze13@huawei.com> +--- + .../modules/image/oci/storage/layer_store/layer_store.c | 8 ++++++-- + 1 file changed, 6 insertions(+), 2 deletions(-) + +diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c +index e88067bc..4edd0cad 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c ++++ b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c +@@ -1409,10 +1409,14 @@ static int do_delete_layer(const char *id) + } + + #ifdef ENABLE_REMOTE_LAYER_STORE +- if (l->slayer->writable) { ++ if (!g_enable_remote_layer) { + ret = layer_store_remove_layer(l->slayer->id); + } else { +- ret = remote_layer_remove_ro_dir(l->slayer->id); ++ if (l->slayer->writable) { ++ ret = layer_store_remove_layer(l->slayer->id); ++ } else { ++ ret = remote_layer_remove_ro_dir(l->slayer->id); ++ } + } + #else + ret = layer_store_remove_layer(l->slayer->id); +-- +2.25.1 + diff --git a/0044-doc-add-document-about-support-remote-ro-directory.patch b/0044-doc-add-document-about-support-remote-ro-directory.patch new file mode 100644 index 0000000..71af0e9 --- /dev/null +++ b/0044-doc-add-document-about-support-remote-ro-directory.patch @@ -0,0 +1,396 @@ +From 6911c68b96f3a0fc35b40e5c387da51cd84e5461 Mon Sep 17 00:00:00 2001 +From: "Neil.wrz" <wangrunze13@huawei.com> +Date: Mon, 13 Feb 2023 03:34:31 -0800 +Subject: [PATCH 44/53] doc: add document about support remote ro directory + +--- + docs/design/README.md | 4 +- + docs/design/README_zh.md | 6 +- + .../design/detailed/Image/remote_ro_design.md | 333 ++++++++++++++++++ + 3 files changed, 340 insertions(+), 3 deletions(-) + create mode 100644 docs/design/detailed/Image/remote_ro_design.md + +diff --git a/docs/design/README.md b/docs/design/README.md +index a9d357b4..cf29c0a1 100644 +--- a/docs/design/README.md ++++ b/docs/design/README.md +@@ -34,6 +34,8 @@ This section contains some design documents for users who want to learn more abo + + - You can see how the isula search are designed in [image_search_design](./detailed/Image/image_search_design_zh.md) + ++- You can see how the remote ro are designed in [remote_ro_design](./detailed/Image/remote_ro_design.md) ++ + ## Network + + - You can see how the cni operator modules are designed in [cni_operator_design](./detailed/Network/cni_operator_design.md). +@@ -54,4 +56,4 @@ This section contains some design documents for users who want to learn more abo + + ## Volume + +-- You can see how the local volume modules are designed in [local_volume_design](./detailed/Volume/local_volume_design.md). +\ No newline at end of file ++- You can see how the local volume modules are designed in [local_volume_design](./detailed/Volume/local_volume_design.md). +diff --git a/docs/design/README_zh.md b/docs/design/README_zh.md +index f51930e4..1f1c94b5 100644 +--- a/docs/design/README_zh.md ++++ b/docs/design/README_zh.md +@@ -32,7 +32,9 @@ + + - 查看 registry 模块的设计文档: [registry_degisn](./detailed/Image/registry_degisn_zh.md) 。 + +-- 查看 isula search 的设计文档:[image_search_design](./detailed/Image/image_search_design_zh.md) ++- 查看 isula search 的设计文档:[image_search_design](./detailed/Image/image_search_design_zh.md) 。 ++ ++- 查看 ro目录分离的设计文档: [remote_ro_design](./detailed/Image/remote_ro_design.md) 。 + + ## Network + +@@ -54,4 +56,4 @@ + + ## Volume + +-- 查看 local volume 模块的设计文档: [local_volume_design](./detailed/Volume/local_volume_design_zh.md). +\ No newline at end of file ++- 查看 local volume 模块的设计文档: [local_volume_design](./detailed/Volume/local_volume_design_zh.md). +diff --git a/docs/design/detailed/Image/remote_ro_design.md b/docs/design/detailed/Image/remote_ro_design.md +new file mode 100644 +index 00000000..fee33835 +--- /dev/null ++++ b/docs/design/detailed/Image/remote_ro_design.md +@@ -0,0 +1,333 @@ ++| Author | 王润泽 | ++| ------ | ---------------------- | ++| Date | 2023-2-13 | ++| Email | wangrunze13@huawei.com | ++ ++# 1. 方案目标 ++目标有两个: ++1. 把isulad当前的layer store里的RO层分离出来,把RW layer和RO layer分开到不同到目录存储。 ++2. isulad在运行时如果在相关目录里恢复了正确的镜像数据(image和layer数据), 可实现不重启isulad, 直接使用新恢复的镜像。如果移除当前没有容器正在使用的镜像数据,可实现不重启isulad,isulad更新当前管理的镜像列表去除该镜像。只考虑完全正确的新增和删除。 ++ ++ ++## 1.1 用法说明 ++通过源码编译打开编译选项来开启功能,cmake添加`cmake -DENABLE_REMOTE_LAYER_STORE=ON ..`, 然后`make -j`即可。启动iSulad之前,还需要在配置文件`/etc/isulad/daemon.json`里面添加`"storage-enable-remote-layer": true`来打开开关。 ++ ++# 2. 总体设计 ++ ++*Modules Dependencies* ++``` ++=> New Added Module: ++ ++ +===================================+ ++ | Remote Supporter Module | ++ +===================================+ ++ | | ++ | +-------------------------+ | ++ | | maintainer submod | | ++ | +-------------------------+ | ++ | | global data initer | | ++ | | symbol link maintainer | | ++ | | global data getter | | ++ | +-------------------------+ | ++ | | ++ | +-------------------------+ | ++ | | Supporter submod | | ++ | +-------------------------+ | ++ | | supporter interface | | ++ | | overlay supporter impl | | ++ | | layer supporter impl | | ++ | | image supporter impl | | ++ | | remote refresh thread | | ++ | +-------------------------+ | ++ +-----------------------------------+ ++ ++ ++=> Modified Modules: ++ ++ +===================================+ ++ | Storage Module | ++ +===================================+ ++ | | ++ | +-------------------------+ | ++ | | Image Store submod | | ++ | | Added Functions | | ++ | +-------------------------+ | ++ | | add image in memory | | ++ | | delete image in memory | | ++ | | get image top layer | | ++ | | valid image manifest | | ++ | +-------------------------+ | ++ | | ++ | +-------------------------+ | ++ | | Layer Store submod | | ++ | | Added Functions | | ++ | +-------------------------+ | ++ | | add layer in memory | | ++ | | delete layer in memory | | ++ | +-------------------------+ | ++ | | ++ | +-------------------------+ | ++ | | Driver Overlay submod | | ++ | | Added Functions | | ++ | +-------------------------+ | ++ | | - | | ++ | +-------------------------+ | ++ +-----------------------------------+ ++ ++ ++=> Modules Dependencies: ++ ++ +-------------------------+ +---------------+ ++ | Supporter submod | | Image Store | ++ +-------------------------+ +----▶| submod ---------+ ++ | supporter interface | | +---------------+ | ++ | overlay supporter impl -----+ | ++ | layer supporter impl -----+ | +++----------------+ | image supporter impl | | +----------------+ | ++| storage module |-------▶| remote refresh thread | +----▶| Layer Store | | init +-----------------------+ +++----------------+ +-------------------------+ | submod --------+--------▶| maintainer submod | ++ | +----------------+ | +-----------------------+ ++ | | ▲ ++ | | | ++ | +----------------+ | | ++ | | Layer Store | | | ++ | | submod --------+ | ++ | +----------------+ | ++ | | ++ +------------------------------------------------------------------------+ ++ get global data ++``` ++ ++ ++总体来说有两部分的功能: ++- iSulad原有的image storage适配分离的RO目录结构,*分离的RO目录*可用于远程挂载 ++- iSulad实例同步内存数据,镜像数据和layer数据*定期更新*,不通过`isula pull` 和 `isula rmi` 等命令,直接通过分离目录里面的数据来更新镜像数据。 ++ ++*分离RO目录* ++修改前后storage目录结构对比: ++ ++``` ++old: ++overlay-layer ++├── b703587 ++│ └── layer.json ++└── b64792c ++ └── layer.json ++ └── b64792.tar.gz ++ ++new: ++overlay-layer ++├── b64792c -> ../RO/b64792c ++├── b703587 ++│ └── layer.json ++└── RO ++ └── b64792c ++ └── layer.json ++ └── b64792.tar.gz ++``` ++ ++以overlay-layers目录为例,创建新layer时,如果是只读层,就把层数据放到RO目录下,在RO上层目录创建软连接指向真实数据。删除layer时需要额外删除软连接。 ++ ++ ++*定期更新* ++定期更新通过启动一个线程周期扫描`overlay`, `overlay-layers`, `overlay-image`这三个目录,通过比较当前时刻与上一时刻的目录差异,来获取镜像和层的删减情况,进而同步isulad的storage内存数据和维护软链接。 ++ ++``` +++---------------------+ +---------------------+ +---------------------+ +-----------------------+ ++| refresh thread loop | | overlay remote impl | | layer remote impl | | image remote impl | +++---------------------+ +---------------------+ +---------------------+ +-----------------------+ ++ | | | | ++ | refresh start | | | ++ |-----------------------------▶| | | ++ | | overlay dir scan | | ++ | | | | ++ | | to added layers | | ++ | | memory and symlink add | | ++ | | to deleted layers | | ++ | | memory and symlink del | | ++ | | valid overlay layers | | ++ | |---------------------------------▶| | ++ | | next scan | | ++ | | | | ++ | | | | ++ | | | | ++ | | | overlay-layers dir scan | ++ | | check overlay layer ready | | ++ | |◀---------------------------------| to added layers | ++ | |---------------------------------▶| filter invalid layers | ++ | | result | memory and symlink add | ++ | | | to deleted layers | ++ | | | memory and symlink del | ++ | | | valid overlay layers | ++ | | |---------------------------------▶| ++ | | | next scan | ++ | | | | ++ | | | | ++ | | | | ++ | | | | overlay-image dir scan ++ | | | check layers ready | ++ | | |◀---------------------------------| to added images ++ | | |---------------------------------▶| filter invalid images ++ | | | result | memory add images ++ | | | | to deleted images ++ | | | | memory del images ++ | | | | ++ |◀---------------------------------------------------------------------------------------------------| ++ | refresh end | | | ++ | | | | +++---------------------+ +---------------------+ +---------------------+ +-----------------------+ ++| refresh thread loop | | image remote module | | layer remote module | | overlay remote module | +++---------------------+ +---------------------+ +---------------------+ +-----------------------+ ++ ++``` ++ ++# 3. 接口描述 ++ ++```c ++// 初始化remote模块里的layer data ++int remote_layer_init(const char *root_dir); ++ ++// 初始化remote模块里的overlay data ++int remote_overlay_init(const char *driver_home); ++ ++// 清理remote模块的资源 ++void remote_maintain_cleanup(); ++ ++// 启动 定期更新的thread ++int start_refresh_thread(void); ++ ++// 创建新layer目录 ++int remote_layer_build_ro_dir(const char *id); ++ ++// 创建新overlay目录 ++int remote_overlay_build_ro_dir(const char *id); ++ ++// 删除layer目录 ++int remote_layer_remove_ro_dir(const char *id); ++ ++// 删除overlay目录 ++int remote_overlay_remove_ro_dir(const char *id); ++``` ++ ++# 4. 详细设计 ++分离RO目录的关键在于适配原来的代码逻辑,原先的代码在操作镜像和层的时候,不管是RO层还是RW层,从创建到删除都是在当前目录下进行的,这就是我们额外创建一个软连接的作用: ++- RO目录的作用是为了支持远程挂载 ++- 软连接的作用是模拟原来的目录结构 ++ ++这样以来,image module的逻辑几乎不需要改动,除了以下几点需要注意: ++- 创建和删除的时候需要处理一个额外的资源:软连接,之前只需要关注目录即可,现在如果创建的是只读层,就需要额外创建软连接,如果删除的是只读层,就需要额外删除软连接 ++- 以`overlay-layers`目录为例,isulad启动时会以正则规则扫描当前目录下的子目录是否合法,所以需要屏蔽`RO`目录 ++ ++定时刷新的逻辑如下: ++以`overlay-image`目录的刷新为例,通过维护两个集合`new` 和 `old`, 这两个集合初始都为空,通过扫描目录里面所有的子目录,把合法的image id 加入`new`集合, 通过计算两个集合的差, 在集合`new`里面存在而在集合`old`里面不存在的id则为新增加的镜像, 在集合`old`里面存在而在集合`new`里面不存在的id则为删除的镜像。处理新增加的镜像还需要额外的一个判断,就是判断镜像的层数据是否已经加载,如果没加载则该镜像本轮不加载。`overlay-layers` 和 `overlay` 目录的处理逻辑类似。 ++ ++ ++*可能的使用场景* ++一个可能的使用场景就是通过远程文件共享(nfs)让多台启动的isulad实例共享某些只读的数据;具体来说,在两个host A和B上都启动了iSulad, 如果A pull或者load了镜像busybox, 那么B上的isulad同样可以使用这个镜像。 ++ ++``` ++operations: ++ +++--------------------+ +--------------------+ +--------------------+ ++| isula pull busybox | | without pull | | without pull | ++| isula pull nginx | | isula run busybox | | isula run ngxinx | ++| isula pull ... | | isula run ... | | isula run centos | +++--------------------+ +--------------------+ +--------------------+ ++ | | | ++ ▼ ▼ ▼ +++======================+ +======================+ +======================+ ++| isulad on Host A | | isulad on Host B | | isulad on Host C | +++======================+ +======================+ +======================+ ++| image store module | | image store module | | image store module | +++----------------------+ +----------------------+ +----------------------+ ++| refresh thread off | | refresh thread on | | refresh thread on | +++----------------------+ +----------------------+ +----------------------+ ++| local rw | remote ro | | local rw | remote ro | | local rw | remote ro | +++----------------------+ +----------------------+ +----------------------+ ++ | | | ++ | enable nfs | mounted on | mounted on ++ ▼ ▼ ▼ ++ +=====================================================================+ ++ | nfs directory over network | ++ +=====================================================================+ ++ | image store | ++ +---------------------------------------------------------------------+ ++ | image | image | image | image | image | ++ | busybox | nginx | my-app | ubuntu | centos | ++ | 4MB | 100MB | 1.2GB | 5MB | 6MB | ++ +---------------------------------------------------------------------+ ++ | layers store | ++ +---------------------------------------------------------------------+ ++ | layer | layer | layer | ++ | 05c361054 | 8a9d75caad | 789d605ac | ++ +---------------------------------------------------------------------+ ++``` ++ ++*同步问题* ++共享资源并发使用发生竞争的条件是:`two process access the same resource concurrently and at least one of the access is a writer`。这里的共享资源有: ++ ++``` +++============================+ +=====================================+ ++| Sharing Resource | | Storage | +++============================+ +=====================================+ ++| read-only overlay layers | mounted | /var/lib/isulad/overlay/RO | +++----------------------------+ ======▶ +-------------------------------------+ ++| reald-only layers metadata | shared | /var/lib/isulad/overlay-layers/RO | +++----------------------------+ +-------------------------------------+ ++| reald-only images | | /var/lib/isulad/overlay-images | +++----------------------------+ +-------------------------------------+ ++ ++``` ++而分布在不同host上的isulad进程通过网络共享这些资源,如果不考虑新增删除的情况,不会出现资源竞争,所有的节点都是reader。 ++ ++对于主节点pull镜像,其他节点使用镜像的情况,主节点是writer,其他节点是reader。这时候可能出现的问题是主节点pull镜像的流程没有完全结束,但是其他节点开始使用这个不完整的镜像。对于这个问题的解决方案是通过扫描image目录来新增镜像,通过这种方式能确保一定该新增镜像的信息完整。 ++ ++``` +++---------------------+ +--------------------+ +-----------------------+ +-----------------------+ ++| registry | | image module | | layer store module | | driver overlay module | +++---------------------+ +--------------------+ +-----------------------+ +-----------------------+ ++ | | | | ++ | | | | ++ | registry_pull | | | ++ | fetch_manifest | | | ++ | check reuse and fetch | | | ++ | +----------------+ | | | ++ | | register_layer | | | | ++ | +----------------+ | | | ++ |-----------------------------------------------------------------▶| | ++ | | | layer_store_create | ++ | | |--------------------------------▶| ++ | | | | driver_create_layer ++ | | | | +--------------------+ ++ | | | | | setup overlay dir | ++ | | | | +--------------------+ ++ | | | | driver_create_layer done ++ | | |◀--------------------------------| ++ | | | +------------+ | ++ | | | | save layer | | ++ | | | +------------+ | ++ | | | layer create done | ++ |◀-----------------------------------------------------------------| | ++ | all layer setup | | | ++ | +----------------+ | | | ++ | | register image | | | | ++ | +----------------+ | | | ++ |-------------------------------▶| | | ++ | | storage_img_create | | ++ | | set img top layer | | ++ | | img create | | ++ | | +------------+ | | ++ | | | save image | | | ++ | | +------------+ | | ++ | | create image done | | ++ |◀-------------------------------| | | ++ | pull img done | | | ++ | | | | ++ | | | | +++---------------------+ +--------------------+ +-----------------------+ +-----------------------+ ++| registry | | image module | | layer store module | | driver overlay module | +++---------------------+ +--------------------+ +-----------------------+ +-----------------------+ ++ ++``` ++ ++至于主节点删除镜像的情况,主节点是writer,其他节点是reader,可能出现的情况是其他节点还有容器在使用镜像的时候,镜像被删除,但是根据需求场景暂不处理这种情况。其他的处理与新增镜像相同,依然以image dir作为入口,扫描发现删除的镜像。删除时需要关注layer和overlay目录下的软链接。 +-- +2.25.1 + diff --git a/0045-Refine-a-minor-log-message.patch b/0045-Refine-a-minor-log-message.patch new file mode 100644 index 0000000..675984b --- /dev/null +++ b/0045-Refine-a-minor-log-message.patch @@ -0,0 +1,34 @@ +From 9b7e37018612f7443a0f8f966106995da31cc292 Mon Sep 17 00:00:00 2001 +From: sailorvii <chenw66@chinaunicom.cn> +Date: Fri, 17 Mar 2023 06:34:37 +0000 +Subject: [PATCH 45/53] Refine a minor log message. + +--- + src/daemon/modules/network/native/adaptor_native.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/src/daemon/modules/network/native/adaptor_native.c b/src/daemon/modules/network/native/adaptor_native.c +index 02ff3642..8bc386d1 100644 +--- a/src/daemon/modules/network/native/adaptor_native.c ++++ b/src/daemon/modules/network/native/adaptor_native.c +@@ -1764,8 +1764,7 @@ bool has_connected_container(native_network *network) + message[pos - 1] = '\0'; + } + +- ERROR("network %s has connected containers [ %s ]", network->conflist->list->name, message); +- isulad_set_error_message("network %s has connected containers [ %s ]", network->conflist->list->name, message); ++ INFO("network %s has connected containers [ %s ]", network->conflist->list->name, message); + + out: + native_network_unlock(network); +@@ -1923,6 +1922,7 @@ int native_config_remove(const char *name, char **res_name) + } + + if (has_connected_container(network)) { ++ isulad_set_error_message("network %s has connected containers", network->conflist->list->name); + ret = -1; + goto out; + } +-- +2.25.1 + diff --git a/0046-modify-the-return-value-of-the-util_waitpid_with_tim.patch b/0046-modify-the-return-value-of-the-util_waitpid_with_tim.patch new file mode 100644 index 0000000..f5a1aed --- /dev/null +++ b/0046-modify-the-return-value-of-the-util_waitpid_with_tim.patch @@ -0,0 +1,44 @@ +From ea8962c6ee3ed05cfabe84c58c47cf56e3388242 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Mon, 20 Mar 2023 14:17:00 +0800 +Subject: [PATCH 46/53] modify the return value of the + util_waitpid_with_timeout to status + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + src/utils/cutils/utils.c | 5 +++-- + 1 file changed, 3 insertions(+), 2 deletions(-) + +diff --git a/src/utils/cutils/utils.c b/src/utils/cutils/utils.c +index 983d81d8..3500d8f8 100644 +--- a/src/utils/cutils/utils.c ++++ b/src/utils/cutils/utils.c +@@ -319,6 +319,7 @@ rep: + int util_waitpid_with_timeout(pid_t pid, const int64_t timeout, handle_timeout_callback_t cb) + { + int nret = 0; ++ int st; + time_t start_time = time(NULL); + time_t end_time; + double interval; +@@ -328,7 +329,7 @@ int util_waitpid_with_timeout(pid_t pid, const int64_t timeout, handle_timeout_c + } + + for (;;) { +- nret = waitpid(pid, NULL, WNOHANG); ++ nret = waitpid(pid, &st, WNOHANG); + if (nret == pid) { + break; + } +@@ -347,7 +348,7 @@ int util_waitpid_with_timeout(pid_t pid, const int64_t timeout, handle_timeout_c + // sleep some time instead to avoid cpu full running and then retry. + usleep(100); + } +- return 0; ++ return st; + } + + int util_wait_for_pid_status(pid_t pid) +-- +2.25.1 + diff --git a/0047-fix-util_getgrent_r-overflow.patch b/0047-fix-util_getgrent_r-overflow.patch new file mode 100644 index 0000000..038aabc --- /dev/null +++ b/0047-fix-util_getgrent_r-overflow.patch @@ -0,0 +1,74 @@ +From fc853c60c56fc3e8dc3c78a8081fea2f9ecc64dc Mon Sep 17 00:00:00 2001 +From: zhangxiaoyu <zhangxiaoyu58@huawei.com> +Date: Tue, 21 Mar 2023 18:52:43 +0800 +Subject: [PATCH 47/53] fix util_getgrent_r overflow + +Signed-off-by: zhangxiaoyu <zhangxiaoyu58@huawei.com> +--- + src/utils/cutils/utils_pwgr.c | 3 +++ + test/cutils/utils_pwgr/long_sample | 1 + + test/cutils/utils_pwgr/utils_pwgr_ut.cc | 28 +++++++++++++++++++++++++ + 3 files changed, 32 insertions(+) + create mode 100644 test/cutils/utils_pwgr/long_sample + +diff --git a/src/utils/cutils/utils_pwgr.c b/src/utils/cutils/utils_pwgr.c +index 75efadb2..064bae1f 100644 +--- a/src/utils/cutils/utils_pwgr.c ++++ b/src/utils/cutils/utils_pwgr.c +@@ -157,6 +157,9 @@ static char **hold_string_list(char **line, char *buf_start, char *buf_end, cons + } + + out: ++ if ((char *)(walker + 2) > buf_end) { ++ return NULL; ++ } + *walker = NULL; + return result; + } +diff --git a/test/cutils/utils_pwgr/long_sample b/test/cutils/utils_pwgr/long_sample +new file mode 100644 +index 00000000..7e991ffd +--- /dev/null ++++ b/test/cutils/utils_pwgr/long_sample +@@ -0,0 +1 @@ ++adm:x:4:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +\ No newline at end of file +diff --git a/test/cutils/utils_pwgr/utils_pwgr_ut.cc b/test/cutils/utils_pwgr/utils_pwgr_ut.cc +index 431372d3..20de8a89 100644 +--- a/test/cutils/utils_pwgr/utils_pwgr_ut.cc ++++ b/test/cutils/utils_pwgr/utils_pwgr_ut.cc +@@ -140,3 +140,31 @@ TEST(utils_pwgr, test_getgrent_r) + + fclose(f_gr); + } ++ ++TEST(utils_pwgr, test_long_getgrent_r) ++{ ++ std::string path = "../../../../test/cutils/utils_pwgr/long_sample"; ++ FILE *f_gr = fopen(path.c_str(), "r"); ++ ASSERT_NE(f_gr, nullptr); ++ ++ struct group gr { ++ 0 ++ }; ++ struct group *pgr = nullptr; ++ char mark_before[BUFSIZ] = { 0 }; ++ char buf[BUFSIZ] = { 0 }; ++ char mark_after[BUFSIZ] = { 0 }; ++ ++ for (int num = 0; num < BUFSIZ; num++) { ++ mark_before[num] = 0xEE; ++ mark_after[num] = 0xDD; ++ } ++ ++ (void)util_getgrent_r(f_gr, &gr, buf, sizeof(buf), &pgr); ++ for (int num = 0; num < BUFSIZ; num++) { ++ ASSERT_EQ(mark_before[num], (char)0xEE); ++ ASSERT_EQ(mark_after[num], (char)0xDD); ++ } ++ ++ fclose(f_gr); ++} +-- +2.25.1 + diff --git a/0048-add-ut-for-runc.patch b/0048-add-ut-for-runc.patch new file mode 100644 index 0000000..e7e4a01 --- /dev/null +++ b/0048-add-ut-for-runc.patch @@ -0,0 +1,188 @@ +From c20a580a7d8c6a8a16273513c6d11edd827b90b0 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Thu, 23 Feb 2023 14:39:23 +0800 +Subject: [PATCH 48/53] add ut for runc + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + .../modules/runtime/isula/isula_rt_ops.c | 5 ++ + test/cutils/utils_utils/CMakeLists.txt | 2 + + test/cutils/utils_utils/utils_utils_ut.cc | 61 +++++++++++++++++++ + test/runtime/isula/isula_rt_ops_ut.cc | 56 +++++++++++++++++ + 4 files changed, 124 insertions(+) + +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index 5ec0e639..fe0f227c 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -1553,6 +1553,11 @@ out: + + int rt_isula_kill(const char *id, const char *runtime, const rt_kill_params_t *params) + { ++ if (id == NULL || runtime == NULL || params == NULL || params->pid < 0) { ++ ERROR("Invalid arguments not allowed"); ++ return -1; ++ } ++ + if (util_process_alive(params->pid, params->start_time) == false) { + if (params->signal == params->stop_signal || params->signal == SIGKILL) { + WARN("Process %d is not alive", params->pid); +diff --git a/test/cutils/utils_utils/CMakeLists.txt b/test/cutils/utils_utils/CMakeLists.txt +index 99a83e7a..6d276390 100644 +--- a/test/cutils/utils_utils/CMakeLists.txt ++++ b/test/cutils/utils_utils/CMakeLists.txt +@@ -12,6 +12,8 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/map + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils + ) ++ ++set_target_properties(${EXE} PROPERTIES LINK_FLAGS "-Wl,--wrap,waitpid") + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) + set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/utils_utils/utils_utils_ut.cc b/test/cutils/utils_utils/utils_utils_ut.cc +index 5308351a..18f0a506 100644 +--- a/test/cutils/utils_utils/utils_utils_ut.cc ++++ b/test/cutils/utils_utils/utils_utils_ut.cc +@@ -15,6 +15,44 @@ + + #include <gtest/gtest.h> + #include "utils.h" ++#include "mock.h" ++ ++static pid_t test_pid = -1; ++ ++extern "C" { ++ DECLARE_WRAPPER_V(waitpid, pid_t, (__pid_t pid, int *stat_loc, int options)); ++ DEFINE_WRAPPER_V(waitpid, pid_t, (__pid_t pid, int *stat_loc, int options),(pid, stat_loc, options)); ++} ++ ++static pid_t waitpid_none_zero(__pid_t pid, int *stat_loc, int options) ++{ ++ *stat_loc = 256; ++ return test_pid; ++} ++ ++static pid_t waitpid_zero(__pid_t pid, int *stat_loc, int options) ++{ ++ *stat_loc = 0; ++ return test_pid; ++} ++ ++#define ExitSignalOffset 128 ++static int status_to_exit_code(int status) ++{ ++ int exit_code = 0; ++ ++ if (WIFEXITED(status)) { ++ exit_code = WEXITSTATUS(status); ++ } else { ++ exit_code = -1; ++ } ++ if (WIFSIGNALED(status)) { ++ int signal; ++ signal = WTERMSIG(status); ++ exit_code = ExitSignalOffset + signal; ++ } ++ return exit_code; ++} + + TEST(utils_utils, test_util_mem_realloc) + { +@@ -284,4 +322,27 @@ TEST(utils_utils, test_do_retry_call) + DO_RETRY_CALL(10, 100, nret, retry_call_test, 11); + ASSERT_EQ(global_total, 10); + ASSERT_EQ(nret, -1); ++} ++ ++TEST(utils_utils, test_util_waitpid_with_timeout) ++{ ++ int64_t timeout = 2; ++ pid_t pid = getpid(); ++ int status = 0; ++ ++ test_pid = pid; ++ MOCK_SET_V(waitpid, waitpid_none_zero); ++ status = util_waitpid_with_timeout(test_pid, timeout, nullptr); ++ ASSERT_EQ(status, 256); ++ ASSERT_EQ(status_to_exit_code(status), 1); ++ MOCK_CLEAR(waitpid); ++ ++ MOCK_SET_V(waitpid, waitpid_zero); ++ status = util_waitpid_with_timeout(test_pid, timeout, nullptr); ++ ASSERT_EQ(status, 0); ++ ASSERT_EQ(status_to_exit_code(status), 0); ++ MOCK_CLEAR(waitpid); ++ ++ ASSERT_EQ(util_waitpid_with_timeout(pid, timeout, nullptr), -1); ++ + } +\ No newline at end of file +diff --git a/test/runtime/isula/isula_rt_ops_ut.cc b/test/runtime/isula/isula_rt_ops_ut.cc +index 03c213a5..f37e62a0 100644 +--- a/test/runtime/isula/isula_rt_ops_ut.cc ++++ b/test/runtime/isula/isula_rt_ops_ut.cc +@@ -163,3 +163,59 @@ TEST_F(IsulaRtOpsUnitTest, test_rt_isula_exec_resize) + close(fd); + ASSERT_EQ(system(rm_path.c_str()), 0); + } ++ ++TEST_F(IsulaRtOpsUnitTest, test_rt_isula_update) ++{ ++ rt_update_params_t params = {}; ++ ASSERT_EQ(rt_isula_update(nullptr, nullptr, nullptr), -1); ++ ++ ASSERT_EQ(rt_isula_update("123", nullptr, nullptr), -1); ++ ASSERT_EQ(rt_isula_update("123", "runtime", nullptr), -1); ++ ASSERT_EQ(rt_isula_update("123", "runtime", ¶ms), -1); ++} ++ ++TEST_F(IsulaRtOpsUnitTest, test_rt_isula_pause) ++{ ++ rt_pause_params_t params = {}; ++ ASSERT_EQ(rt_isula_pause(nullptr, nullptr, nullptr), -1); ++ ++ ASSERT_EQ(rt_isula_pause("123", nullptr, nullptr), -1); ++ ASSERT_EQ(rt_isula_pause("123", "runtime", nullptr), -1); ++ ASSERT_EQ(rt_isula_pause("123", "runtime", ¶ms), -1); ++} ++ ++TEST_F(IsulaRtOpsUnitTest, test_rt_isula_resume) ++{ ++ rt_resume_params_t params = {}; ++ ASSERT_EQ(rt_isula_resume(nullptr, nullptr, nullptr), -1); ++ ++ ASSERT_EQ(rt_isula_resume("123", nullptr, nullptr), -1); ++ ASSERT_EQ(rt_isula_resume("123", "runtime", nullptr), -1); ++ ASSERT_EQ(rt_isula_resume("123", "runtime", ¶ms), -1); ++} ++ ++TEST_F(IsulaRtOpsUnitTest, test_rt_isula_resources_stats) ++{ ++ rt_stats_params_t params = {}; ++ struct runtime_container_resources_stats_info stats = {}; ++ ++ ASSERT_EQ(rt_isula_resources_stats(nullptr, nullptr, nullptr, nullptr), -1); ++ ++ ASSERT_EQ(rt_isula_resources_stats("123", nullptr, nullptr, nullptr), -1); ++ ASSERT_EQ(rt_isula_resources_stats("123", "runtime", nullptr, nullptr), -1); ++ ASSERT_EQ(rt_isula_resources_stats("123", "runtime", ¶ms, nullptr), -1); ++ params.state = "/var/run/isulad/runtime"; ++ ASSERT_EQ(rt_isula_resources_stats("123", "runtime", ¶ms, &stats), -1); ++} ++ ++TEST_F(IsulaRtOpsUnitTest, test_rt_isula_kill) ++{ ++ rt_kill_params_t kill_params = { ++ .pid = -1, ++ }; ++ ASSERT_EQ(rt_isula_kill(nullptr, nullptr, nullptr), -1); ++ ++ ASSERT_EQ(rt_isula_kill("123", nullptr, nullptr), -1); ++ ASSERT_EQ(rt_isula_kill("123", "runtime", nullptr), -1); ++ ASSERT_EQ(rt_isula_kill("123", "runtime", &kill_params), -1); ++} +\ No newline at end of file +-- +2.25.1 + diff --git a/0049-add-runc-doc.patch b/0049-add-runc-doc.patch new file mode 100644 index 0000000..11f8de2 --- /dev/null +++ b/0049-add-runc-doc.patch @@ -0,0 +1,506 @@ +From eb3e8dd9e87e926a121715d103cc6c3cb20388a7 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Fri, 17 Feb 2023 14:33:15 +0800 +Subject: [PATCH 49/53] add runc doc + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + docs/design/README_zh.md | 1 + + .../design/detailed/Runtime/runc_design_zh.md | 264 ++++++++++++++++++ + docs/images/isulad_shim_flow_chart.svg | 16 ++ + docs/images/k8s_isulad_flow_chart.svg | 16 ++ + docs/manual/README_zh.md | 6 +- + docs/manual/runc_config_zh.md | 134 +++++++++ + 6 files changed, 436 insertions(+), 1 deletion(-) + create mode 100644 docs/design/detailed/Runtime/runc_design_zh.md + create mode 100755 docs/images/isulad_shim_flow_chart.svg + create mode 100644 docs/images/k8s_isulad_flow_chart.svg + create mode 100644 docs/manual/runc_config_zh.md + +diff --git a/docs/design/README_zh.md b/docs/design/README_zh.md +index 1f1c94b5..e7763a42 100644 +--- a/docs/design/README_zh.md ++++ b/docs/design/README_zh.md +@@ -49,6 +49,7 @@ + ## Runtime + + - 查看 runtime 模块的设计文档: [runtime_design](./detailed/Runtime/runtime_design_zh.md) 。 ++- 查看 isulad 支持runc 的设计文档:[runc_design](./detailed/Runtime/runc_design_zh.md)。 + + ## Security + +diff --git a/docs/design/detailed/Runtime/runc_design_zh.md b/docs/design/detailed/Runtime/runc_design_zh.md +new file mode 100644 +index 00000000..3af3ea02 +--- /dev/null ++++ b/docs/design/detailed/Runtime/runc_design_zh.md +@@ -0,0 +1,264 @@ ++| Author | zhongtao | ++| ------ | --------------------- | ++| Date | 2023-03-21 | ++| Email | zhongtao17@huawei.com | ++ ++# 方案目标 ++ ++isulad 支持使用runc作为容器runtime。用户可以在启动容器时配置`--runtime=runc`创建runtime为runc的容器: ++```bash ++$ isula run -tid -n test --runtime=runc busybox sh ++eb77c672a148cc4cefa0d6e7c5847f5a87d0c5353e245461b68820bd9af90c67 ++$ isula inspect eb77 | grep -i runc ++ "ResolvConfPath": "/var/lib/isulad/engines/runc/eb77c672a148cc4cefa0d6e7c5847f5a87d0c5353e245461b68820bd9af90c67/resolv.conf", ++ "HostsPath": "/var/lib/isulad/engines/runc/eb77c672a148cc4cefa0d6e7c5847f5a87d0c5353e245461b68820bd9af90c67/hosts", ++ "LogPath": "/var/lib/isulad/engines/runc/eb77c672a148cc4cefa0d6e7c5847f5a87d0c5353e245461b68820bd9af90c67/console.log", ++ "Runtime": "runc", ++ "log.console.file": "/var/lib/isulad/engines/runc/eb77c672a148cc4cefa0d6e7c5847f5a87d0c5353e245461b68820bd9af90c67/console.log", ++``` ++ ++同时,也可以在`/etc/isulad/daemon.json`中配置`default-runtime`为runc并重启isulad,修改isulad创建容器时默认使用的runtime。 ++```sh ++$ vim /etc/isulad/daemon.json ++ ... ++ "default-runtime": "runc" ++ ... ++$ sudo isulad ++``` ++ ++# 总体设计 ++ ++由于isulad与runc之间的交互存在gap,且将容器创建成功之后,容器进程的生命周期与isulad进程的生命周期没有必然联系,因此我们设计了一个isulad-shim进程,用于isulad与runc的交互并将isulad与容器实例解耦。同时,由于只有create以及exec涉及到在容器中新建进程,因此只有这两个子命令需要创建isulad-shim。其他的子命令直接通过调用runc二进制实现。 ++ ++## 时序图 ++ ++```mermaid ++sequenceDiagram ++ participant isula ++ participant kubelet ++ participant isulad ++ participant supervisor ++ participant shim ++ participant runc ++ participant container ++ ++ isula->>isulad: request ++ kubelet->>isulad:request ++ alt create ++ isulad->>shim:shim_creat() ++ shim->>runc: execvp(runc, params) ++ runc ->> container:create request ++ container ->> runc:success ++ runc ->> shim:get process pid ++ isulad ->> isulad:get process pid ++ isulad ->> supervisor:add monitor ++ loop epoll exit_fd ++ supervisor ->> shim:if exit? ++ end ++ else exec ++ isulad->>shim:shim_creat() ++ par ++ shim->>runc: execvp(runc, params) ++ runc ->> container:exec request ++ container ->> runc:success ++ runc ->> shim:get process pid ++ shim ->> container:wait process pid ++ shim ->> shim:exit ++ and ++ isulad ->> isulad: wait isulad-shim pid ++ end ++ else others container cmd ++ isulad->>runc: runtime_call_simple() ++ runc ->> container:cmd ++ container ->>runc:success ++ runc ->>isulad:success ++ end ++ isulad ->> isula:response ++ isulad ->> kubelet:response ++``` ++ ++# 接口描述 ++ ++## cri接口 ++ ++### PodSandboxManagerService ++ ++```h ++auto RunPodSandbox(const runtime::v1alpha2::PodSandboxConfig &config, const std::string &runtimeHandler,Errors &error) -> std::string; ++ ++void StopPodSandbox(const std::string &podSandboxID, Errors &error); ++ ++void RemovePodSandbox(const std::string &podSandboxID, Errors &error); ++ ++auto PodSandboxStatus(const std::string &podSandboxID, Errors &error) ++ -> std::unique_ptr<runtime::v1alpha2::PodSandboxStatus>; ++ ++void ListPodSandbox(const runtime::v1alpha2::PodSandboxFilter *filter,std::vector<std::unique_ptr<runtime::v1alpha2::PodSandbox>> *pods, Errors &error); ++ ++// This feature is temporarily not supported ++void PortForward(const runtime::v1alpha2::PortForwardRequest &req,runtime::v1alpha2::PortForwardResponse *resp,Errors &error); ++ ... ... ++}; ++} // namespace CRI ++``` ++ ++### ContainerManagerService ++ ++```c ++auto CreateContainer(const std::string &podSandboxID, const runtime::v1alpha2::ContainerConfig &containerConfig,const runtime::v1alpha2::PodSandboxConfig &podSandboxConfig, Errors &error)-> std::string override; ++ ++void StartContainer(const std::string &containerID, Errors &error) override; ++ ++void StopContainer(const std::string &containerID, int64_t timeout, Errors &error) override; ++ ++void RemoveContainer(const std::string &containerID, Errors &error) override; ++ ++void ListContainers(const runtime::v1alpha2::ContainerFilter *filter, ++ std::vector<std::unique_ptr<runtime::v1alpha2::Container>> *containers, Errors &error) override; ++ ++void ListContainerStats(const runtime::v1alpha2::ContainerStatsFilter *filter,std::vector<std::unique_ptr<runtime::v1alpha2::ContainerStats>> *containerstats,Errors &error) override; ++ ++auto ContainerStats(const std::string &containerID, Errors &error) ++ -> std::unique_ptr<runtime::v1alpha2::ContainerStats> override; ++ ++auto ContainerStatus(const std::string &containerID, Errors &error) -> std::unique_ptr<runtime::v1alpha2::ContainerStatus> override; ++ ++void ExecSync(const std::string &containerID, const google::protobuf::RepeatedPtrField<std::string> &cmd, int64_t timeout, runtime::v1alpha2::ExecSyncResponse *reply, Errors &error) override; ++ ++void Exec(const runtime::v1alpha2::ExecRequest &req, runtime::v1alpha2::ExecResponse *resp, Errors &error) override; ++ ++void Attach(const runtime::v1alpha2::AttachRequest &req, runtime::v1alpha2::AttachResponse *resp, Errors &error) override; ++ ++void UpdateContainerResources(const std::string &containerID, ++ const runtime::v1alpha2::LinuxContainerResources &resources, Errors &error) override; ++``` ++ ++### RuntimeManagerService ++ ++```c ++void UpdateRuntimeConfig(const runtime::v1alpha2::RuntimeConfig &config, Errors &error) override; ++ ++// 对应crictl info ++auto Status(Errors &error) -> std::unique_ptr<runtime::v1alpha2::RuntimeStatus> override; ++``` ++ ++### ImageManagerService ++ ++```c ++void ListImages(const runtime::v1alpha2::ImageFilter &filter, std::vector<std::unique_ptr<runtime::v1alpha2::Image>> *images, Errors &error) override; std::unique_ptr<runtime::v1alpha2::Image> ImageStatus(const runtime::v1alpha2::ImageSpec &image, Errors &error) override; ++ ++std::string PullImage(const runtime::v1alpha2::ImageSpec &image, const runtime::v1alpha2::AuthConfig &auth, Errors &error) override; ++ ++void RemoveImage(const runtime::v1alpha2::ImageSpec &image, Errors &error) override; ++ ++void ImageFsInfo(std::vector<std::unique_ptr<runtime::v1alpha2::FilesystemUsage>> *usages, Errors &error) override ++``` ++ ++ ++ ++## isula_rt_ops模块 ++ ++```c ++// 检测runtime是否为isula_rt_ops模块处理的目标runtime ++bool rt_isula_detect(const char *runtime); ++ ++int rt_isula_create(const char *name, const char *runtime, const rt_create_params_t *params); ++ ++int rt_isula_start(const char *name, const char *runtime, const rt_start_params_t *params, pid_ppid_info_t *pid_info); ++ ++// restart not implemented ++int rt_isula_restart(const char *name, const char *runtime, const rt_restart_params_t *params); ++ ++int rt_isula_clean_resource(const char *name, const char *runtime, const rt_clean_params_t *params); ++ ++int rt_isula_rm(const char *name, const char *runtime, const rt_rm_params_t *params); ++ ++int rt_isula_exec(const char *id, const char *runtime, const rt_exec_params_t *params, int *exit_code); ++ ++int rt_isula_status(const char *name, const char *runtime, const rt_status_params_t *params, ++ struct runtime_container_status_info *status); ++ ++// isula attach not support on isulad-shim ++int rt_isula_attach(const char *id, const char *runtime, const rt_attach_params_t *params); ++ ++int rt_isula_update(const char *id, const char *runtime, const rt_update_params_t *params); ++ ++int rt_isula_pause(const char *id, const char *runtime, const rt_pause_params_t *params); ++ ++int rt_isula_resume(const char *id, const char *runtime, const rt_resume_params_t *params); ++ ++// isula top/listpids not support on isulad-shim ++int rt_isula_listpids(const char *name, const char *runtime, const rt_listpids_params_t *params, ++ rt_listpids_out_t *out); ++ ++int rt_isula_resources_stats(const char *name, const char *runtime, const rt_stats_params_t *params, ++ struct runtime_container_resources_stats_info *rs_stats); ++// rt_isula_resize not impl ++int rt_isula_resize(const char *id, const char *runtime, const rt_resize_params_t *params); ++ ++int rt_isula_exec_resize(const char *id, const char *runtime, const rt_exec_resize_params_t *params); ++ ++int rt_isula_kill(const char *id, const char *runtime, const rt_kill_params_t *params); ++ ++``` ++ ++ ++ ++# 详细设计 ++ ++## create 实现流程 ++ ++isulad端: ++ ++1. 创建process文件:create_process_json_file(); ++2. 获得runtime二进制:get_runtime_cmd(runtime, &cmd); ++3. 利用两次fork()创建isulad-shim进程, 此时的isulad-shim进程是1号进程的子进程,与isulad无父子关系:shim_create(); ++4. 若shim创建成功则直接返回成功,若失败则调用runtime_call_delete_force()将容器进程都force delete。 ++ ++isulad-shim端: ++ ++1. 根据解析process文件新建一个process,new_process(); ++2. 开启exit_fifo:open_no_inherit("exit_fifo", O_WRONLY, -1); ++3. 为io_copy创建io_epoll_loop线程:process_io_init(); ++4. 创建进行io copy的线程,并将isulad的console与runtime的console连接起来:open_io(); ++5. 创建运行runtime的子进程,获得容器中进程的pid,create_process(); ++6. 循环wait子进程,直到wait到的子进程为容器进程pid则退出shim进程,process_signal_handle_routine(); ++ ++## exec 实现流程 ++ ++isulad端: ++ ++1. 创建process文件:create_process_json_file(); ++2. 获得runtime二进制:get_runtime_cmd(runtime, &cmd); ++3. 若`--detach=false`,直接fork()创建isulad-shim进程,此时的isulad-shim进程是isulad进程的子进程,isulad进程wait等待isulad-shim进程退出;若-`-detach=true`,则与create一样,创建独立的isulad-shim进程:shim_create(); ++4. 循环读取pid文件获得容器进程id说明exec命令成功,get_container_process_pid。 ++ ++isulad-shim端: ++ ++1. 根据解析process文件新建一个process,new_process(); ++2. 为io_copy创建io_epoll_loop线程:process_io_init(); ++3. 创建进行io copy的线程,并将isulad的console与runtime的console连接起来:open_io(); ++4. 创建运行runtime的子进程,获得容器中进程的pid,create_process(); ++5. 循环wait子进程,直到wait到的子进程为容器进程pid则退出shim进程,process_signal_handle_routine(); ++ ++ ++ ++## start 实现流程 ++ ++isulad端: ++ ++1. 分别读取pid以及shim-pid文件获得容器进程pid以及isulad-shim pid; ++2. 依次获得容器进程以及isulad-shim进程的proc信息:util_get_process_proc_info(); ++3. 根据proc信息为pid_ppid_info_t结构体赋值; ++4. 直接调用runc二进制start容器:runtime_call_simple(); ++5. 之后isulad根据监听exit_fifo_fd,感知容器是否退出,从而更新容器状态。 ++ ++其他子命令与satrt类似,均是调用runtime_call_simple()函数直接调用runc二进制,此处不再赘述。 ++ ++ ++ ++## 流程图 ++ ++### isulad与isulad-shim交互流程图 ++ ++ +diff --git a/docs/images/isulad_shim_flow_chart.svg b/docs/images/isulad_shim_flow_chart.svg +new file mode 100755 +index 00000000..9a509b80 +--- /dev/null ++++ b/docs/images/isulad_shim_flow_chart.svg +@@ -0,0 +1,16 @@ ++<svg version="1.1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 2639.476104494126 3305.9641492328947" width="2639.476104494126" height="3305.9641492328947"> ++ <!-- svg-source:excalidraw --> ++ ++ <defs> ++ <style class="style-fonts"> ++ @font-face { ++ font-family: "Virgil"; ++ src: url("https://excalidraw.com/Virgil.woff2"); ++ } ++ @font-face { ++ font-family: "Cascadia"; ++ src: url("https://excalidraw.com/Cascadia.woff2"); ++ } ++ </style> ++ </defs> ++ <rect x="0" y="0" width="2639.476104494126" height="3305.9641492328947" fill="#ffffff"></rect><g stroke-linecap="round" transform="translate(1275.9641960991746 11.21154906257766) rotate(0 120.37038167317712 60.37036471896701)"><path d="M6.04 41.03 C6.04 41.03, 6.04 41.03, 6.04 41.03 M6.04 41.03 C6.04 41.03, 6.04 41.03, 6.04 41.03 M1.89 50.28 C10.86 41.95, 17.89 34.75, 23.14 27.57 M3.36 49.86 C9.94 41.26, 16.81 33.63, 23.19 26.46 M2.54 56.73 C12.6 45.58, 26.28 30.71, 31.39 24.81 M1.92 56.91 C8.58 49.78, 15.58 42.06, 31.04 24.48 M1.23 65.98 C11.64 51.97, 21.31 39.65, 40.11 19.77 M2.19 65.46 C8.16 55.41, 16.46 46.9, 40.01 19.8 M4 68.85 C20.81 48.94, 38.12 29.76, 47.91 16.27 M2.31 69.06 C12.43 57.87, 21.63 46.91, 48.45 16.81 M6.97 71.12 C24.21 49.61, 42.44 27.46, 57.14 13.91 M5.77 71.54 C20.4 56.65, 34.51 40.03, 55.67 14.45 M7.16 77.05 C21.69 59.96, 33.11 46.63, 66.65 9.53 M7.44 76.75 C26.03 53, 46.55 30.86, 66.95 7.03 M7.98 78.68 C24.92 62.79, 38.95 46.19, 73.6 4.93 M9.28 81.19 C34.63 50.29, 60.3 21.48, 72.53 6.18 M10.8 83.67 C33.46 59.32, 53.79 33.88, 77.26 6.49 M11.09 84.09 C24.41 68.62, 37.8 52.56, 79.62 5.78 M11.75 89.35 C33.51 64.11, 55.5 35.61, 86.75 6.6 M11.64 88.79 C34.48 63.32, 55.42 39.62, 85.43 5.67 M13.96 89.29 C33.55 69.98, 48.72 50.45, 89.23 6.47 M15.99 91.52 C43.46 58.24, 74.65 23.12, 91.5 3.71 M17.75 90.86 C34 73.63, 50.54 52.08, 97.35 2.88 M19.98 92.32 C49.65 59.37, 77.28 24.76, 96.42 2.88 M24.14 94.04 C52.6 58.44, 83.42 22.22, 104.83 0.34 M22.96 96.1 C39.82 74, 58.18 55.87, 103.62 2.62 M27.64 97.48 C55.75 63.72, 82.18 33.9, 110.64 1.56 M26.76 97.94 C57.02 61.68, 87.45 28.54, 108.72 1.67 M30.54 99.84 C49.38 78.06, 67.01 54.13, 115.15 2.31 M29.24 98.75 C51.87 72.51, 74.69 47.07, 113.97 2.17 M33.05 100.29 C56.49 73.75, 84.94 43.48, 117.61 4.46 M32.6 101.09 C50.74 79.46, 69.05 58.35, 118.92 3 M37.44 105.13 C64.33 72.28, 88.06 42.13, 124.51 3.82 M36.8 104 C68.28 66.77, 100.85 28.2, 122.98 2.88 M39.67 107.84 C63.82 78.55, 88.08 51.23, 126.91 2.67 M39.34 106.36 C62.68 81.54, 83.94 55.45, 127.99 3.2 M44.79 107.88 C72.42 75.48, 102.65 42.25, 133.41 4.35 M43.94 107.98 C77.9 68.06, 111.98 28.09, 133.39 3.87 M47.22 110 C71.18 81.78, 90.35 54.78, 137.59 4.55 M47.02 108.19 C82.35 67.85, 118.38 28.09, 138.94 2.93 M53.59 108.33 C78.84 80.12, 105.77 48.44, 143.65 5.16 M51.71 110.19 C88.06 68.63, 121.49 28.5, 144.46 3.62 M56.19 111.66 C92.14 71.79, 124.59 32.47, 148.45 2.53 M55.62 111.63 C84.46 80.8, 111.43 49.27, 149.22 5 M62.6 113.02 C80.86 88.25, 101.34 63.36, 155.8 4.57 M61.57 112.48 C89.81 79.16, 120.8 44.76, 154.35 4.24 M63.77 112.5 C85.9 90.21, 102.66 66.37, 160.64 5.8 M64 113.09 C93.7 80.53, 120.73 46.21, 160.29 4.05 M67.63 114.79 C104.16 75.61, 137.91 35.07, 164.51 6.41 M67.71 116.07 C99.48 80.37, 130.97 45.3, 164.47 4.94 M73.86 116.99 C100.06 85.73, 128.6 52.73, 169.08 5.82 M72.9 117.34 C96.05 90.02, 118.78 64.02, 168.07 7.11 M76.16 117.95 C99.06 91.99, 120.48 65.46, 175.07 7.03 M76.66 117.78 C98.72 93.57, 122.31 67.65, 173.1 8.03 M81.06 118.09 C116.48 76.4, 154.63 33.96, 176.37 9.36 M81.95 118.26 C118.09 77.25, 154.15 35.09, 175.81 9.54 M84.79 121.66 C119.09 78.46, 154.74 38.54, 182.88 12.33 M85.73 120.29 C108.94 95.16, 131.76 67.99, 181.73 10.93 M88.73 119.83 C125.67 81.58, 160.3 41, 186.79 12.94 M90.66 120.33 C124.97 81.45, 158.6 41.8, 185.13 10.56 M95.74 122.3 C128.89 87.13, 156.7 48.83, 188.69 14.23 M94.57 121.64 C119.41 93.74, 144.62 64.83, 189.4 13.68 M99.56 122.01 C135.61 81.43, 172.6 40.5, 193.75 15.87 M100.1 120.39 C121.23 97.7, 141.09 74.55, 194.43 15.27 M107.05 120.56 C140.95 80.2, 174.41 42.64, 195.97 17.44 M106.24 121.23 C139.23 82.3, 173.91 42.41, 197.07 14.66 M110.36 122.46 C136.63 89.96, 163.86 57.31, 202.33 17.01 M110.6 120.91 C143.24 85.42, 172.75 50.83, 200.48 18.35 M114.45 122.94 C143.04 90.12, 172.68 60.99, 206.29 20.44 M116.24 122.33 C146.51 86.65, 175.64 52.27, 205.92 18.63 M121.95 122.04 C142.43 101.06, 163.15 75.65, 207.37 21.23 M120.69 121.95 C150.5 87.02, 182.66 51.18, 207.46 21.64 M127.36 121.42 C149.23 96.83, 170.69 72.3, 210.57 23.51 M125.39 122.81 C152.53 93.76, 180.49 62.15, 211.69 23.72 M129.87 122.4 C154.37 96.71, 174.87 73, 216.69 27.44 M130.76 121.93 C149.46 100.75, 167.96 79.86, 215.21 25.03 M138.25 120.28 C164.36 87.62, 196.51 54.61, 220.43 29.69 M138.25 120.88 C165.2 90.23, 193.64 59.37, 218.98 27.38 M142.38 119.95 C160.97 102.28, 177.01 82.01, 222.57 30.07 M144.12 120.64 C170.06 90.81, 195.54 61.57, 221.7 31.05 M151.36 120.1 C179.88 86.21, 204.98 53.94, 226.22 30.81 M148.8 120.15 C167.76 97.91, 185.18 77.27, 226.3 31.44 M157.13 117.39 C184.28 84.88, 213.92 56.03, 229.62 34.35 M156.65 117.71 C171.79 99.92, 188.97 80.96, 229.91 33.09 M163.27 116.21 C181.88 94.13, 202.13 71.3, 231.73 37.24 M162.04 117.05 C185.03 90.67, 206.36 66.21, 231.75 38.85 M167.86 116.99 C181.83 101.61, 196.68 83.84, 232.93 43.31 M168.44 116.92 C185.72 96.34, 202.44 77.36, 232.96 41.41 M172.97 113.26 C190.06 99.12, 204.65 83.59, 234.74 44.37 M175.08 114.41 C193.62 93.89, 212.48 72.58, 234.61 45.17 M184.25 111.73 C199.51 94.25, 216.18 75.07, 236.52 49.45 M182.77 112.34 C202.35 90.26, 221.9 67.29, 237.58 49.85 M189.76 108.36 C200.45 97.07, 210.72 83.31, 239.08 52.87 M191.98 107.6 C209.15 89.17, 225.25 68.55, 240.38 51.79 M198.14 104.92 C212.13 90.19, 227.23 71.85, 240.84 56.83 M199.2 104.97 C207.84 95.19, 217.38 84.61, 242.59 56.86 M206.13 100.25 C214.22 92.34, 221.39 86.78, 238.75 65.98 M207.66 102.69 C215.64 93.08, 223.51 83.89, 237.9 66.27 M212.78 99.91 C221.17 93.7, 224.22 88.7, 234.89 73.88 M213.85 101.39 C221.19 90.75, 229.14 83.11, 236.05 75.35" stroke="#fa5252" stroke-width="0.5" fill="none"></path><path d="M66.35 7.25 C78.4 3.15, 96.02 1.82, 111.43 1.02 C126.83 0.22, 144.24 0.09, 158.78 2.44 C173.33 4.8, 187.16 10.1, 198.69 15.16 C210.23 20.21, 220.97 25.99, 227.99 32.77 C235 39.54, 239.79 47.95, 240.8 55.81 C241.8 63.68, 238.92 72.72, 234.02 79.95 C229.11 87.17, 221.35 93.48, 211.34 99.16 C201.33 104.85, 187.56 110.41, 173.96 114.06 C160.36 117.7, 144.86 120.32, 129.74 121.03 C114.61 121.74, 97.79 120.68, 83.21 118.32 C68.63 115.97, 54.15 111.95, 42.25 106.91 C30.35 101.88, 18.73 95.26, 11.82 88.12 C4.9 80.99, 1.74 71.95, 0.76 64.12 C-0.22 56.29, 1.11 48.2, 5.94 41.14 C10.78 34.08, 18.21 27.93, 29.74 21.76 C41.28 15.59, 66.24 7.15, 75.15 4.13 C84.06 1.1, 82.58 2.52, 83.22 3.58 M119.96 -1.35 C134.68 -2.48, 151.76 0.9, 165.75 3.91 C179.75 6.93, 192.85 11.28, 203.93 16.74 C215 22.19, 226.06 29.3, 232.18 36.65 C238.31 44, 240.76 53.14, 240.69 60.82 C240.61 68.51, 237.76 75.48, 231.75 82.73 C225.74 89.98, 215.18 98.65, 204.62 104.32 C194.06 109.99, 182.51 113.89, 168.38 116.75 C154.26 119.6, 135.62 121.47, 119.88 121.47 C104.15 121.48, 88.12 119.77, 73.95 116.78 C59.79 113.78, 45.81 109.11, 34.9 103.51 C24 97.91, 14.21 90.38, 8.52 83.18 C2.83 75.98, 0.64 67.97, 0.75 60.3 C0.87 52.63, 3.58 44.05, 9.19 37.14 C14.8 30.23, 23.39 24.19, 34.39 18.84 C45.4 13.49, 60.95 7.99, 75.21 5.04 C89.46 2.09, 112.43 1.83, 119.93 1.12 C127.43 0.41, 120.37 0.04, 120.2 0.77" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1344.3345777723516 59.3319137815447) rotate(0 52 12.5)"><text x="52" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">isulad-shim</text></g><g stroke-linecap="round" transform="translate(1269.852966308593 230.28572203621047) rotate(0 135 38)"><path d="M19 0 M19 0 C85.7 -0.07, 148.36 0.28, 251 0 M19 0 C98.77 -1.99, 179.38 -1.34, 251 0 M251 0 C261.92 1.83, 269.1 6.67, 270 19 M251 0 C262.04 -1.43, 268.95 4.88, 270 19 M270 19 C269.69 26.21, 270.36 33.64, 270 57 M270 19 C269.49 28.59, 270.73 37.86, 270 57 M270 57 C269.44 69.32, 263.03 74.63, 251 76 M270 57 C268.64 69.67, 262.67 76.35, 251 76 M251 76 C197.82 74.5, 145.4 76.66, 19 76 M251 76 C188.96 74.2, 127.02 73.32, 19 76 M19 76 C6.2 77.27, 1.74 71.08, 0 57 M19 76 C4.17 75.59, -0.9 70.18, 0 57 M0 57 C-0.4 43.87, 1.13 34.35, 0 19 M0 57 C-0.32 43.68, -0.06 29.92, 0 19 M0 19 C0.57 6.87, 4.9 0.35, 19 0 M0 19 C-0.68 7.32, 4.19 1.8, 19 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1376.352966308593 255.78572203621047) rotate(0 28.5 12.5)"><text x="28.5" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">main()</text></g><g stroke-linecap="round"><g transform="translate(1395.2652702292294 132.95562525855428) rotate(0 -0.606445564897319 47.95117267061053)"><path d="M0.03 -0.72 C0.21 15.33, 0.31 80.26, -0.01 96.63 M-1.41 1.51 C-1.21 17.16, -0.78 78.96, -0.47 94.66" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1395.2652702292294 132.95562525855428) rotate(0 -0.606445564897319 47.95117267061053)"><path d="M-12.29 68.17 C-8.3 70.78, -4.67 78.55, 0.24 95.61 M-10.52 66.51 C-7.66 76.54, -2.95 87.48, 0.22 94.8" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1395.2652702292294 132.95562525855428) rotate(0 -0.606445564897319 47.95117267061053)"><path d="M8.23 67.91 C7.96 70.58, 7.34 78.41, 0.24 95.61 M10 66.25 C5.36 76.24, 2.57 87.27, 0.22 94.8" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g transform="translate(830.2604505750855 21.915314931718285) rotate(0 192 12.5)"><text x="0" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="start" style="white-space: pre;" direction="ltr">isulad-shim id bundle runtime info 2m0s</text></g><g stroke-linecap="round"><g transform="translate(1402.8160536024297 304.08191802009713) rotate(0 0.4947381429789992 31.467739513671688)"><path d="M1.18 -0.05 C1.34 10.35, 0.66 52.15, 0.13 62.89 M0.33 -1.12 C0.51 9.38, -0.22 53.45, -0.21 64.05" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1402.8160536024297 304.08191802009713) rotate(0 0.4947381429789992 31.467739513671688)"><path d="M-10.3 37.71 C-9 40.56, -4.14 47.03, -0.98 65.69 M-9.88 35.67 C-6.57 43.64, -5.57 49.82, -0.24 64.42" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1402.8160536024297 304.08191802009713) rotate(0 0.4947381429789992 31.467739513671688)"><path d="M10.22 37.89 C6.8 40.9, 6.94 47.33, -0.98 65.69 M10.64 35.85 C8.56 43.87, 4.17 50, -0.24 64.42" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(1268.4827609592005 806.6931088159956) rotate(0 129.5 70.5)"><path d="M32 0 M32 0 C78.24 -0.97, 121.03 0.72, 227 0 M32 0 C89.88 0.61, 146.77 1.07, 227 0 M227 0 C249.66 -1.84, 259.9 11.34, 259 32 M227 0 C250.44 -2.26, 257.47 9.39, 259 32 M259 32 C257.46 48.89, 259.57 62.67, 259 109 M259 32 C259.38 60.19, 260.34 89.08, 259 109 M259 109 C257.74 129.53, 247.53 140.86, 227 141 M259 109 C256.83 132.57, 248.59 140.55, 227 141 M227 141 C153.19 141.45, 79.07 140.71, 32 141 M227 141 C172.02 139.04, 116.1 139.51, 32 141 M32 141 C8.95 141.05, 0.18 129.4, 0 109 M32 141 C9.1 139.07, 1.83 131.98, 0 109 M0 109 C-1.17 80.68, -1.81 50.81, 0 32 M0 109 C-0.81 85.51, 0.07 61.98, 0 32 M0 32 C-1.24 9.12, 12.42 1.35, 32 0 M0 32 C-2.24 11.68, 12.82 0.03, 32 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1279.9827609592005 852.6931088159956) rotate(0 118 24.5)"><text x="118" y="17.5" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">从process.json中加载proc</text><text x="118" y="42" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">ess:new_process</text></g><g stroke-linecap="round"><g transform="translate(1395.0851367644639 950.0830134924811) rotate(0 1.2255874045993416 30.817684593971194)"><path d="M-1 -0.52 C-0.98 9.4, 0.56 50.54, 1.33 60.65 M0.68 1.83 C0.95 12.45, 2.99 52.27, 3.45 62.15" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1395.0851367644639 950.0830134924811) rotate(0 1.2255874045993416 30.817684593971194)"><path d="M-9.03 32.63 C-3.37 47.51, 0.37 56.31, 5.17 60.73 M-7.68 34.9 C-3.9 44.13, 0.64 54.93, 3.05 61.38" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1395.0851367644639 950.0830134924811) rotate(0 1.2255874045993416 30.817684593971194)"><path d="M11.47 31.63 C8.97 46.74, 4.55 55.93, 5.17 60.73 M12.81 33.9 C9.27 43.67, 6.48 54.83, 3.05 61.38" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(1280.705071343315 1030.2485490829226) rotate(0 123.33333333333331 67.22223917643225)"><path d="M155 17 M155 17 C172.37 28.54, 193.35 38.83, 215.67 51 M155 17 C175.68 29.59, 199.03 40.43, 215.67 51 M215.67 51 C247.69 66.09, 247.42 69.56, 215.67 85 M215.67 51 C244.4 69.72, 245.86 69.16, 215.67 85 M215.67 85 C199.52 95.9, 177.94 104.11, 155 117.44 M215.67 85 C201.72 92.94, 190.64 98.38, 155 117.44 M155 117.44 C123.73 134.74, 123.58 135.52, 93 117.44 M155 117.44 C126 134.59, 124.25 135.41, 93 117.44 M93 117.44 C76.11 105.89, 57.59 99.42, 31 85 M93 117.44 C79.07 110.06, 63.83 102.53, 31 85 M31 85 C0.49 69.67, 0.14 68.74, 31 51 M31 85 C-0.23 68.25, -1.16 68.98, 31 51 M31 51 C50.16 39.37, 68.01 31.28, 93 17 M31 51 C48.61 40.35, 66.5 31.71, 93 17 M93 17 C123.11 -0.98, 125.15 1.53, 155 17 M93 17 C125.3 0.74, 124.04 1.71, 155 17" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1313.0384046766485 1085.2207882593548) rotate(0 91 12.5)"><text x="91" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">if p->state->exec?</text></g><g stroke-linecap="round" transform="translate(347.2018897646942 325.19050816127236) rotate(0 139.5 42)"><path d="M21 0 M21 0 C72.86 1.02, 125.88 2.27, 258 0 M21 0 C91.98 1.4, 161.72 1.01, 258 0 M258 0 C273.14 1.71, 277.21 7.08, 279 21 M258 0 C273.15 1.9, 278.48 5.28, 279 21 M279 21 C280.16 34.13, 279.32 49.61, 279 63 M279 21 C279.99 29.11, 278.83 39.08, 279 63 M279 63 C278.87 78.51, 270.58 84.05, 258 84 M279 63 C276.93 76.15, 272.56 84.19, 258 84 M258 84 C172.55 83.63, 86.43 83.24, 21 84 M258 84 C179.99 84.78, 102.9 85.4, 21 84 M21 84 C8.08 85.65, 0.77 76.45, 0 63 M21 84 C7.84 83.36, 1.71 78.77, 0 63 M0 63 C-1.45 48.87, -1.39 34.52, 0 21 M0 63 C0.45 50.05, 0.63 38.9, 0 21 M0 21 C-0.57 6.26, 6.59 0.66, 21 0 M0 21 C-0.91 8.34, 5.47 -0.57, 21 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(357.7018897646942 330.19050816127236) rotate(0 128.5 37)"><text x="128.5" y="17.666666666666668" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">将create的process存储在pr</text><text x="128.5" y="42.333333333333336" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">ocess.jsonwe文件中:</text><text x="128.5" y="67" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">create_process_json_file</text></g><g stroke-linecap="round" transform="translate(394.2018592471162 10) rotate(0 131 52)"><path d="M7.8 35.93 C7.8 35.93, 7.8 35.93, 7.8 35.93 M7.8 35.93 C7.8 35.93, 7.8 35.93, 7.8 35.93 M4.84 45.91 C9.8 38.82, 17.39 32, 23.35 25.16 M3.53 46.54 C9.23 40.49, 13.21 34.22, 22.4 24.94 M3.31 56.12 C11.69 44.11, 20.68 35.11, 35.05 20.2 M0.66 55.99 C12.75 44.05, 22.79 30.18, 34.32 18.42 M2.21 59.38 C14.28 46.72, 24.73 31.8, 40.74 18.4 M3.56 59.29 C13.76 48.49, 23.07 36.39, 41.16 16 M7.8 61.49 C18.22 46.32, 32.7 32.87, 46.92 12.08 M6.37 63.15 C22.72 43.24, 38.29 25.34, 48.64 14.42 M8.63 64.73 C19.26 51.46, 32.78 37.16, 54.92 10.12 M7.77 66.54 C17.78 53.72, 28.55 41.89, 54.52 11.41 M11.2 67.9 C30.55 46.61, 49.73 23.08, 63.67 9.03 M10.82 68.06 C29.6 47.73, 47.25 27.49, 62.6 10.91 M13.44 71.93 C28.13 54.16, 44.75 35.39, 67.23 6.74 M11.69 73.83 C23.74 59.77, 36.55 45.52, 68.42 8.19 M16.21 74.91 C26.34 60.37, 41.26 47.08, 76.33 8.16 M15.35 75.78 C28 62.38, 39.32 47.56, 74.05 7.71 M20.22 78.94 C35.51 58.03, 56.83 36.53, 79.15 6.75 M19.75 76.79 C42.09 52.44, 62.88 26.91, 81.08 5.95 M24.2 78.64 C44.98 55.67, 66.43 31.85, 84.95 4.83 M22.16 80.5 C45.95 54.7, 66.6 27.92, 87.49 6 M26.36 82.26 C41.74 64.89, 52.89 50.43, 94.15 5.66 M25.88 82.67 C52.93 53.34, 79.19 22.83, 91.92 4.92 M27.99 83.73 C46.68 66.2, 61.83 46.42, 99.3 4.21 M30.06 84.52 C49.4 61.53, 67.58 39.47, 97.86 4.83 M34.26 84.44 C48.63 68.7, 62.78 52.86, 106.46 4.02 M33.7 86.83 C54.3 62.11, 76.06 37.07, 105.27 2.96 M37.77 88.85 C55.87 63.79, 76.66 41.93, 112.2 3.98 M37.43 88.76 C64.85 55.71, 92.68 23.96, 110.79 2.83 M40.73 89.29 C62.85 65.77, 84.67 39.35, 116.36 3.91 M40.88 88.98 C61.91 65.31, 83.88 41.91, 116.69 3.01 M43.12 92.73 C66.04 66.63, 92.22 41.26, 122.88 2.86 M45.15 91.74 C70.92 60.74, 96.72 30.28, 122.98 1.77 M48.28 94.41 C71.82 66.2, 98.69 36.36, 126.82 -0.05 M47.5 94.7 C62.24 74.96, 79.98 56.22, 128.59 1.15 M50.32 96.84 C85.17 59.64, 117.38 22.55, 134.32 -0.13 M51.04 95.32 C72.06 71.47, 93.05 45.6, 132.59 0.78 M54.44 97.41 C85.93 61.61, 115.98 24.97, 139.61 0.03 M56.29 96.66 C81.57 65.44, 109.41 35.02, 137.37 2.3 M62.34 97.18 C81.5 72.29, 102.14 49.07, 141.79 0.28 M60.76 97.49 C81.8 74.16, 99.99 50.85, 142.71 1.34 M66.51 98.21 C96.67 62.79, 128.14 24.11, 150.44 0.81 M64.46 97.91 C81.94 77.74, 98.78 58.5, 149.53 2.73 M70.29 97.83 C95.42 68.75, 122.31 38.78, 151.69 2.41 M70.24 98.87 C99.51 63.15, 130.32 27.97, 153.24 3.02 M75.5 100.51 C100.94 69.08, 127.02 37.58, 157.82 4.27 M74.58 100.35 C106.23 64.83, 136.98 29.34, 159.54 3.6 M79.25 99.87 C98.15 80.74, 118.3 54.16, 162.1 4.09 M78.63 100.64 C103.1 73.45, 127.33 45.51, 164.45 3.04 M84.13 103.52 C115.88 64.93, 149.93 26.56, 166.95 4.87 M83.62 101.49 C101.49 80.22, 121.64 58.21, 168.74 3.16 M86.4 103.89 C109.37 78.87, 133.32 50.18, 174.17 3.72 M87.28 104.14 C108.19 77.38, 130.73 52.65, 173.33 4.18 M90.83 102.25 C112.23 81.41, 132.83 56.48, 176.42 3.4 M91.89 103.7 C123.51 66.59, 158.56 28.51, 177.04 5.52 M96.93 105.79 C113.47 84.88, 132.62 61.61, 181.44 6.95 M96.75 103.51 C124 73.81, 148.72 43.58, 181.95 6.64 M103.09 106.22 C128.66 71.43, 160.92 34.99, 186.87 5.9 M101.49 103.96 C129.9 70.03, 160.22 36.35, 186.53 6.65 M106.12 103.26 C132.76 76.12, 159.27 45.07, 192.48 7.46 M107.88 105.32 C129.58 78.57, 152.26 52.49, 191.05 7.67 M111.45 105.51 C132.22 80.57, 154.47 59.52, 196.32 8.7 M111.21 104.86 C143.82 70.18, 175.42 32.57, 195.92 8.38 M117.94 104.48 C145.82 72.49, 172.82 37.63, 198.27 8.48 M117.35 104.18 C144.13 74.66, 171.04 41.99, 200.48 10.17 M121.5 106.45 C145.06 77.36, 169.16 52.01, 204.85 9.42 M121.55 105.34 C143.24 83.06, 161.71 60.94, 204.09 10.41 M127.13 106.71 C152.74 74, 181.16 42.41, 207.48 10.76 M126.77 106.37 C152.14 77.07, 176.09 49.17, 209.47 11.62 M132.57 105.44 C153.44 80.5, 178.79 56.98, 214.59 11.19 M132.46 105.47 C156.31 79.24, 177.47 53.82, 213.17 13.52 M139.06 103.45 C156.33 82.86, 177.74 58.75, 218.52 12.84 M139.59 103.99 C167.98 71.94, 194.45 42.05, 217.6 13.38 M146.32 103.35 C163.96 81.22, 181.39 61.41, 220.39 15.93 M144.85 103.89 C170.46 73.49, 197.53 43.18, 222.17 15.88 M150.58 105.02 C171.39 80.57, 192.55 57.85, 226.22 16.78 M150.88 102.51 C167.54 82.81, 185.77 61.24, 224.59 17.43 M157.53 100.41 C181.3 76.69, 201.9 52.2, 230.5 19.87 M156.46 101.66 C173.75 82.94, 191.11 63.31, 228.7 19.52 M161.19 101.98 C184.52 74.74, 210.91 48.68, 231.72 21.18 M163.55 101.27 C185.99 75.43, 207.83 48.54, 233.15 20.32 M170.02 102.84 C191.78 74.26, 215.56 48.22, 236.53 22.98 M168.7 101.91 C186.31 78.84, 205.08 57.6, 235.85 22.8 M172.5 102.7 C191.55 85.65, 202.93 66.77, 241.93 22.66 M174.77 99.95 C195.88 74.86, 220.66 46.54, 239.38 23.75 M179.97 99.87 C202.89 75.52, 222.92 53.35, 244.13 26.78 M179.04 99.87 C202.76 75.7, 224.4 51.88, 243.99 25.2 M187.45 97.33 C200.91 79.86, 220.87 62.84, 250.1 26.88 M186.24 99.61 C199.54 82.9, 213.41 67.52, 248.45 28.49 M191.94 100.06 C206.39 80.91, 220.89 66.39, 253.05 30.57 M190.02 100.17 C201.75 86.16, 215.85 71.18, 251.71 30.6 M199.07 97.79 C219.08 72.88, 241.44 46.21, 253.3 31.8 M197.89 97.66 C212.79 81.58, 226.89 63.75, 252.99 32.73 M202.85 98.43 C220.74 76.7, 241.56 53.17, 254.44 37.7 M204.16 96.97 C217.61 81.88, 231.39 66.68, 256.18 38.44 M212.38 94.77 C227.77 77.55, 240.52 61.51, 258.88 39.88 M210.84 93.83 C227.87 75.84, 244.43 57.06, 258.59 41.21 M216.75 91.36 C230.68 74.86, 244.27 61.03, 260.73 45.01 M217.82 91.89 C232.73 76.55, 245.5 61.73, 260.03 44.28 M225.21 90.77 C239.93 76.34, 252.53 61.99, 262.59 48.61 M225.19 90.29 C239.52 74.18, 254.76 56.79, 262.9 48.79 M230.05 90.22 C242.27 79.53, 250.29 68.74, 263.14 50.64 M231.39 88.6 C240.13 78.31, 248.65 70.24, 263.87 50.55 M243.32 82.53 C252.3 73.59, 257.27 65.84, 261.36 63.07 M244.13 80.99 C248.64 76.35, 251.96 72.74, 261.44 61.99" stroke="#fa5252" stroke-width="0.5" fill="none"></path><path d="M73.15 6.09 C86.49 2.59, 105.36 0.75, 121.89 0.13 C138.43 -0.48, 156.39 0.26, 172.37 2.41 C188.35 4.56, 204.94 8.7, 217.75 13.03 C230.55 17.36, 241.87 22.37, 249.21 28.4 C256.54 34.42, 260.84 42.35, 261.76 49.18 C262.68 56.02, 260.22 63.17, 254.74 69.41 C249.26 75.64, 240.01 81.67, 228.89 86.62 C217.76 91.56, 202.96 96.2, 187.99 99.07 C173.02 101.95, 155.52 103.39, 139.05 103.86 C122.58 104.34, 104.81 103.93, 89.14 101.93 C73.47 99.93, 57.59 96.42, 45.04 91.88 C32.49 87.33, 21.22 80.8, 13.82 74.66 C6.42 68.53, 1.65 61.5, 0.63 55.06 C-0.38 48.61, 2.35 42.19, 7.75 35.99 C13.15 29.79, 20.29 23.36, 33.04 17.84 C45.8 12.31, 73.92 5.41, 84.26 2.84 C94.6 0.27, 94.39 1.52, 95.09 2.43 M128.78 0.94 C145.04 -0.11, 165.31 1.06, 180.86 3.33 C196.42 5.59, 210.25 9.93, 222.11 14.54 C233.96 19.16, 245.2 25.02, 251.99 31.01 C258.79 37, 262.84 43.63, 262.87 50.46 C262.9 57.29, 258.68 65.62, 252.17 71.99 C245.65 78.36, 235.57 84.05, 223.77 88.68 C211.96 93.31, 196.66 97.35, 181.31 99.77 C165.97 102.18, 147.97 102.89, 131.68 103.17 C115.38 103.45, 99.1 103.97, 83.53 101.44 C67.97 98.92, 50.63 93.13, 38.3 88.05 C25.96 82.96, 16.07 76.86, 9.52 70.93 C2.97 64.99, -1.19 58.99, -1 52.43 C-0.81 45.88, 4.43 37.53, 10.66 31.6 C16.9 25.67, 24.87 21.42, 36.42 16.86 C47.96 12.31, 64.61 7.2, 79.95 4.26 C95.28 1.32, 120.11 -0.28, 128.44 -0.78 C136.77 -1.28, 129.72 0.13, 129.92 1.25" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(496.2018592471162 49.75) rotate(0 29 12.5)"><text x="29" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">isulad</text></g><g stroke-linecap="round"><g transform="translate(512.8214598974533 117.8008081170097) rotate(0 -10.485013029000811 55.436327466953855)"><path d="M-0.22 -0.58 C-3.7 17.8, -16.62 91.46, -20.08 109.97 M-1.79 1.73 C-5.46 20.34, -17.39 93.69, -20.75 111.45" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(512.8214598974533 117.8008081170097) rotate(0 -10.485013029000811 55.436327466953855)"><path d="M-26.99 82.8 C-24.31 92.18, -21.53 103.06, -22.45 110.16 M-26.93 82.37 C-23.48 92.34, -21.91 101.4, -19.77 111.11" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(512.8214598974533 117.8008081170097) rotate(0 -10.485013029000811 55.436327466953855)"><path d="M-6.77 86.32 C-11.54 94.37, -16.22 103.95, -22.45 110.16 M-6.72 85.89 C-10.16 94.59, -15.5 102.44, -19.77 111.11" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(354.5352637881317 440.52385166713174) rotate(0 131 30)"><path d="M15 0 M15 0 C85.77 0.04, 156.42 -1.78, 247 0 M15 0 C88.37 0.91, 160.24 1.18, 247 0 M247 0 C258.83 0.95, 263.58 3.48, 262 15 M247 0 C258.58 -1.47, 262.16 2.82, 262 15 M262 15 C261.03 20.65, 262.62 29.08, 262 45 M262 15 C261.13 23.16, 261.99 30.37, 262 45 M262 45 C263.55 53.51, 258.55 58.51, 247 60 M262 45 C263.26 56.02, 255.85 58.89, 247 60 M247 60 C179.13 58.36, 113.78 57.59, 15 60 M247 60 C179.32 58.37, 110.05 59.13, 15 60 M15 60 C4.01 59.44, 0.28 54.09, 0 45 M15 60 C3.81 58.48, 1.95 54.58, 0 45 M0 45 C-0.14 37.94, -0.66 30.12, 0 15 M0 45 C-0.48 34.42, 0.17 21, 0 15 M0 15 C1.74 3.67, 6.98 -1.47, 15 0 M0 15 C1.39 6.7, 4.78 -2.26, 15 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(400.5352637881317 458.02385166713174) rotate(0 85 12.5)"><text x="85" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">get_runtime_cmd</text></g><g stroke-linecap="round"><g transform="translate(485.59013253353805 410.31550833293375) rotate(0 0.3081041104769042 14.85253082677633)"><path d="M-0.59 1 C-0.72 5.93, -0.81 24.41, -0.87 29.22 M1.3 0.49 C1.57 5.09, 1.45 22.77, 1.48 27.36" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(485.59013253353805 410.31550833293375) rotate(0 0.3081041104769042 14.85253082677633)"><path d="M-2.23 13.06 C-1.42 16.94, 0.76 21.83, 0.08 28.61 M-3 14.41 C-2.22 16.36, -0.46 20.54, 1.04 26.85" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(485.59013253353805 410.31550833293375) rotate(0 0.3081041104769042 14.85253082677633)"><path d="M7.59 13.05 C5.51 16.89, 4.8 21.78, 0.08 28.61 M6.82 14.4 C5.31 16.28, 4.77 20.46, 1.04 26.85" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(361.20195079985047 546.5238516671318) rotate(0 126 30)"><path d="M15 0 M15 0 C97.43 -1.35, 176.71 -1.27, 237 0 M15 0 C61.98 1.21, 107.72 0.89, 237 0 M237 0 C247.91 -0.62, 253.74 5.29, 252 15 M237 0 C244.83 1.85, 250.23 4.53, 252 15 M252 15 C252.35 22.43, 251.16 32.62, 252 45 M252 15 C252.45 24.43, 251.01 33.42, 252 45 M252 45 C253.35 55.96, 245.26 60.03, 237 60 M252 45 C253.39 54.84, 248.11 61.19, 237 60 M237 60 C188.34 62.41, 142.95 59.89, 15 60 M237 60 C163.67 58.58, 89.31 59.23, 15 60 M15 60 C6.47 60.75, -1.59 53.51, 0 45 M15 60 C6.06 58.38, 1.33 54.8, 0 45 M0 45 C-1.58 33.7, 1.63 25.07, 0 15 M0 45 C0.03 38.59, 0.76 31.94, 0 15 M0 15 C-1.37 6.83, 6.31 0.48, 15 0 M0 15 C2.07 4.43, 3.53 -0.08, 15 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(395.20195079985047 564.2738516671318) rotate(0 92 12.5)"><text x="92" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">ret = shim_create</text></g><g mask="url(#mask-Tzaa8PGhGc0Kypz7a8y5z)" stroke-linecap="round"><g transform="translate(482.17083832775165 608.5238516671318) rotate(0 -1.9802706059182924 54.236628025015676)"><path d="M-0.82 1.1 C-1.74 19.1, -3.83 90.15, -4.38 107.84 M0.95 0.63 C-0.16 18.26, -4.04 87.91, -4.91 105.87" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(482.17083832775165 608.5238516671318) rotate(0 -1.9802706059182924 54.236628025015676)"><path d="M-11.84 78.5 C-9.77 90.37, -7.29 98.61, -4.98 106.9 M-13.36 76.27 C-10.45 87.22, -6.9 98.71, -5.71 105.72" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(482.17083832775165 608.5238516671318) rotate(0 -1.9802706059182924 54.236628025015676)"><path d="M8.65 79.58 C2.63 90.88, -2.97 98.7, -4.98 106.9 M7.13 77.35 C2.43 88.08, -1.63 99.17, -5.71 105.72" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask id="mask-Tzaa8PGhGc0Kypz7a8y5z"><rect x="0" y="0" fill="#fff" width="587.3349549115763" height="816.0793936593191"></rect><rect x="440.08878003583936" y="649.8016226632253" fill="#000" width="79" height="25" opacity="1"></rect></mask><g transform="translate(440.08878003583936 649.8016226632254) rotate(0 40.101787685994 12.95885702892204)"><text x="39.5" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">ret = -1</text></g><g stroke-linecap="round"><g transform="translate(482.5352637881317 501.1905081612724) rotate(0 1.5629761065542311 20.751710404008605)"><path d="M0.89 -0.47 C1.48 6.77, 2.79 36.1, 3.23 43.26 M-0.1 -1.76 C0.41 5.14, 2.03 33.86, 2.57 41.45" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(482.5352637881317 501.1905081612724) rotate(0 1.5629761065542311 20.751710404008605)"><path d="M-6.77 22.81 C-2.7 28.53, -1.6 30.84, 3.52 40.21 M-5.9 22.37 C-4.3 28.3, -1.14 34.03, 2.88 42.29" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(482.5352637881317 501.1905081612724) rotate(0 1.5629761065542311 20.751710404008605)"><path d="M7.82 21.88 C7.93 27.96, 5.07 30.52, 3.52 40.21 M8.69 21.44 C5.84 27.69, 4.57 33.71, 2.88 42.29" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(335.8685157412567 717.4127371651784) rotate(0 144 29.5)"><path d="M14.75 0 M14.75 0 C87.71 0.97, 162.25 2.27, 273.25 0 M14.75 0 C107.99 -0.42, 201.37 0.69, 273.25 0 M273.25 0 C281.64 0.79, 289.89 6.62, 288 14.75 M273.25 0 C282.92 2.21, 288.75 5.19, 288 14.75 M288 14.75 C289.38 24.87, 288.61 34.69, 288 44.25 M288 14.75 C288.1 24.21, 288.73 32.35, 288 44.25 M288 44.25 C286.95 55.99, 283.48 60.31, 273.25 59 M288 44.25 C286.63 52.53, 280.81 59.81, 273.25 59 M273.25 59 C171.55 60.52, 73.85 60.43, 14.75 59 M273.25 59 C180.3 61.12, 87.47 61.66, 14.75 59 M14.75 59 C5.73 60.69, 0.69 55.88, 0 44.25 M14.75 59 C6.44 56.94, -0.85 56, 0 44.25 M0 44.25 C-0.25 37.04, -1.1 25.52, 0 14.75 M0 44.25 C0.76 36.32, -0.51 29.49, 0 14.75 M0 14.75 C-0.45 5.45, 3.65 -0.6, 14.75 0 M0 14.75 C1.87 2.62, 5.54 1.12, 14.75 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(347.3685157412567 734.4127371651787) rotate(0 132.5 12.5)"><text x="132.5" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">runtime_call_delete_force</text></g><g stroke-linecap="round" transform="translate(338.19410850888175 231.89676121302983) rotate(0 146.19049072265625 24.285714285714306)"><path d="M3.23 2.81 C3.23 2.81, 3.23 2.81, 3.23 2.81 M3.23 2.81 C3.23 2.81, 3.23 2.81, 3.23 2.81 M0.35 12.23 C2.08 8.61, 4.88 4.77, 10.19 0.91 M0.35 12.23 C2.8 8.85, 5.76 4.86, 10.19 0.91 M0.09 18.62 C5.34 13.85, 11.71 4.76, 14.52 2.02 M0.09 18.62 C3.73 13.81, 7.9 8.1, 14.52 2.02 M0.48 24.27 C6.66 15.6, 14.95 11.86, 19.51 2.38 M0.48 24.27 C5.5 18.98, 10.85 13.24, 19.51 2.38 M0.87 29.91 C11.77 20.87, 20.12 10.6, 25.15 1.99 M0.87 29.91 C8.95 20.58, 18.98 10.73, 25.15 1.99 M0.61 36.31 C9.01 26.4, 17.28 20.32, 30.14 2.35 M0.61 36.31 C10.86 25.11, 21.61 13.45, 30.14 2.35 M1.01 41.95 C8.41 31.91, 18.19 26.47, 35.78 1.95 M1.01 41.95 C13.95 26.45, 26.07 11.85, 35.78 1.95 M3.37 45.33 C16.7 27.74, 33.77 11.48, 40.77 2.31 M3.37 45.33 C13.84 33.26, 24.72 21.81, 40.77 2.31 M6.39 47.95 C14.2 37.12, 23.76 28.72, 46.41 1.92 M6.39 47.95 C19.52 32.34, 34.7 17.05, 46.41 1.92 M10.72 49.07 C19.5 35.55, 32.67 23.61, 51.4 2.28 M10.72 49.07 C22.92 33.39, 35.8 18.88, 51.4 2.28 M15.71 49.43 C28.01 34.62, 41.56 19.63, 57.04 1.88 M15.71 49.43 C31.2 32.48, 46.5 14.67, 57.04 1.88 M21.35 49.03 C31.98 34.13, 45.97 23.32, 62.03 2.24 M21.35 49.03 C36.93 32.5, 53.08 13.51, 62.03 2.24 M26.34 49.39 C40.33 35.01, 48.77 22.97, 67.67 1.85 M26.34 49.39 C42.11 31.01, 57.35 13.35, 67.67 1.85 M31.32 49.75 C42.21 34.51, 54.92 19.45, 72.66 2.21 M31.32 49.75 C45.7 35.01, 57.69 18.23, 72.66 2.21 M36.97 49.36 C50.2 36.04, 58.96 22.93, 78.3 1.81 M36.97 49.36 C51.55 31.84, 66.77 13.45, 78.3 1.81 M41.95 49.72 C52.73 33.74, 66.9 22.72, 83.29 2.17 M41.95 49.72 C56.06 34.07, 68.33 18.38, 83.29 2.17 M47.6 49.33 C59.77 35.09, 71.81 21.89, 88.93 1.78 M47.6 49.33 C63.54 29.86, 80.47 10.79, 88.93 1.78 M52.58 49.69 C64.5 40.26, 73.9 24.9, 93.92 2.14 M52.58 49.69 C64.55 35.29, 75.79 22.76, 93.92 2.14 M58.23 49.29 C72.32 35.4, 83.77 20.03, 99.56 1.74 M58.23 49.29 C67.96 40.8, 75.62 30.3, 99.56 1.74 M63.21 49.65 C72.28 40.52, 83.21 29.6, 104.55 2.1 M63.21 49.65 C76.88 33.86, 90.06 18.13, 104.55 2.1 M68.86 49.26 C84.33 32.48, 95.78 14.78, 109.53 2.46 M68.86 49.26 C84.27 31.67, 97.71 16.38, 109.53 2.46 M73.84 49.62 C86.67 32.42, 103.57 18.16, 115.18 2.07 M73.84 49.62 C82.19 39.11, 91.92 28.76, 115.18 2.07 M79.49 49.22 C91.79 36.34, 100.41 26.65, 120.16 2.43 M79.49 49.22 C91.47 35.61, 103 21.02, 120.16 2.43 M84.47 49.58 C95.72 38.49, 106.06 26.7, 125.81 2.04 M84.47 49.58 C99.98 32.55, 113.35 16.18, 125.81 2.04 M90.12 49.19 C96.92 39.13, 107.67 29, 130.79 2.4 M90.12 49.19 C102.57 34.11, 115.97 17.42, 130.79 2.4 M95.1 49.55 C108.4 35.45, 121.34 17.81, 136.44 2 M95.1 49.55 C103.82 37.36, 115.17 25.98, 136.44 2 M100.75 49.15 C112.8 32.69, 126.06 15.99, 141.42 2.36 M100.75 49.15 C110.65 37.71, 118.92 27.08, 141.42 2.36 M105.73 49.51 C122.87 31.02, 137.92 16.68, 147.07 1.97 M105.73 49.51 C118.62 34.57, 130.68 19.15, 147.07 1.97 M111.38 49.12 C121.86 38.91, 132.75 27.25, 152.05 2.33 M111.38 49.12 C119.78 40.16, 129.32 29.96, 152.05 2.33 M116.36 49.48 C128.45 33.98, 141.3 22.41, 157.7 1.93 M116.36 49.48 C129.75 34.5, 142.68 18.89, 157.7 1.93 M122.01 49.08 C130.91 36.64, 141.25 23.28, 162.68 2.29 M122.01 49.08 C137.4 32.44, 151.94 13.95, 162.68 2.29 M126.99 49.44 C135.79 38.88, 144.01 29.42, 168.33 1.9 M126.99 49.44 C140.27 33.87, 153.97 18.87, 168.33 1.9 M131.98 49.8 C142.44 37.72, 152.22 28.78, 173.31 2.26 M131.98 49.8 C142.53 38.56, 152.78 25.79, 173.31 2.26 M137.62 49.41 C147.14 40.53, 155.81 30.73, 178.96 1.86 M137.62 49.41 C149.22 35.84, 161.21 21.55, 178.96 1.86 M142.61 49.77 C155.65 35.86, 170.31 21.46, 183.94 2.22 M142.61 49.77 C152.9 36.4, 164.67 24.58, 183.94 2.22 M148.26 49.38 C160.14 34.51, 171.24 23.21, 189.59 1.83 M148.26 49.38 C160.27 35.81, 169.7 23.24, 189.59 1.83 M153.24 49.74 C161.15 40.82, 171.12 30.19, 194.57 2.19 M153.24 49.74 C163.78 36.88, 175.94 23.94, 194.57 2.19 M158.89 49.34 C170.48 38.35, 180.07 25.74, 199.56 2.55 M158.89 49.34 C171.77 35.8, 183.14 21.48, 199.56 2.55 M163.87 49.7 C175.98 36.61, 184.93 25.76, 205.2 2.15 M163.87 49.7 C174.39 37.75, 182.74 27.88, 205.2 2.15 M169.52 49.31 C180.09 37.27, 188.81 24.34, 210.19 2.51 M169.52 49.31 C178.14 38.94, 187.65 27.26, 210.19 2.51 M174.5 49.67 C190.99 29.63, 206.82 13.02, 215.83 2.12 M174.5 49.67 C187.06 36.11, 198.52 22.17, 215.83 2.12 M180.15 49.27 C187 40.31, 196.87 29.29, 220.82 2.48 M180.15 49.27 C193.38 35.02, 204.58 22.49, 220.82 2.48 M185.13 49.63 C194.61 38.27, 204.58 27.82, 226.46 2.09 M185.13 49.63 C198.85 33.72, 213.04 15.54, 226.46 2.09 M190.78 49.24 C199.57 39.8, 207.81 27.56, 231.45 2.45 M190.78 49.24 C199.99 38.81, 207.57 29.56, 231.45 2.45 M195.76 49.6 C205.98 35.1, 217.53 21.33, 237.09 2.05 M195.76 49.6 C206.68 36.85, 217.87 23.18, 237.09 2.05 M201.41 49.2 C214.01 33.02, 226.64 21.47, 242.08 2.41 M201.41 49.2 C209.67 39.43, 217.72 29.45, 242.08 2.41 M206.39 49.56 C217.03 38.04, 224.12 29.58, 247.72 2.02 M206.39 49.56 C219.12 37.1, 230.54 23.35, 247.72 2.02 M212.04 49.17 C224.44 34.65, 237.62 19.08, 252.71 2.38 M212.04 49.17 C220.56 39.78, 230.32 29.31, 252.71 2.38 M217.02 49.53 C233.19 31.84, 248.86 14.12, 258.35 1.98 M217.02 49.53 C232.22 31.24, 247.8 12.92, 258.35 1.98 M222.01 49.89 C231.49 41.2, 237.4 31.1, 263.34 2.34 M222.01 49.89 C236.34 33.91, 248.47 18.17, 263.34 2.34 M227.65 49.49 C234.55 38.65, 245.24 29.36, 268.98 1.95 M227.65 49.49 C238.98 36.39, 250.87 23.42, 268.98 1.95 M232.64 49.85 C244.69 36.63, 260.92 20.43, 273.97 2.31 M232.64 49.85 C244.32 37.09, 253.87 23.39, 273.97 2.31 M238.28 49.46 C251.27 34.88, 265.94 15.66, 279.61 1.91 M238.28 49.46 C251.9 34.31, 265.25 18.41, 279.61 1.91 M243.27 49.82 C257.58 30.8, 276.83 13.68, 285.26 1.52 M243.27 49.82 C257.23 33.14, 271.04 17.99, 285.26 1.52 M248.91 49.43 C256.7 36.1, 268.57 25.9, 288.93 3.39 M248.91 49.43 C264.54 34, 278.26 17.03, 288.93 3.39 M253.9 49.79 C261.7 40.96, 271.49 28.69, 291.95 6.01 M253.9 49.79 C265.4 36.56, 275.31 24.06, 291.95 6.01 M259.54 49.39 C272.68 34.18, 286.92 19.73, 294.31 9.39 M259.54 49.39 C267.88 39.2, 275.01 30.71, 294.31 9.39 M264.53 49.75 C274.95 37.2, 284.94 28.94, 294.71 15.03 M264.53 49.75 C273.39 40.71, 280.85 31.39, 294.71 15.03 M270.17 49.36 C275.36 44.69, 279.69 39.08, 294.45 21.43 M270.17 49.36 C278.57 39.91, 284.91 31.61, 294.45 21.43 M275.16 49.72 C281.04 45.37, 283.15 38.43, 294.84 27.07 M275.16 49.72 C280.32 45.12, 284.25 38.91, 294.84 27.07 M280.8 49.32 C283.72 46.75, 287.89 40.77, 294.58 33.47 M280.8 49.32 C285.91 44.61, 290.88 38.62, 294.58 33.47" stroke="#228be6" stroke-width="0.5" fill="none"></path><path d="M12.14 0 M12.14 0 C107.95 0.88, 202.3 1.4, 280.24 0 M12.14 0 C106.56 2.01, 200.18 1.95, 280.24 0 M280.24 0 C289.25 1.51, 291.46 4.64, 292.38 12.14 M280.24 0 C288.74 1.81, 293.4 3.73, 292.38 12.14 M292.38 12.14 C293.38 18.66, 292.1 27.47, 292.38 36.43 M292.38 12.14 C291.84 19.01, 292.44 25.65, 292.38 36.43 M292.38 36.43 C290.57 43.69, 290.26 50.14, 280.24 48.57 M292.38 36.43 C294.5 42.98, 287.88 47.44, 280.24 48.57 M280.24 48.57 C207.51 51.72, 135.44 50.82, 12.14 48.57 M280.24 48.57 C206.23 50.33, 131.86 50.76, 12.14 48.57 M12.14 48.57 C2.47 49.2, -0.57 44.93, 0 36.43 M12.14 48.57 C6.28 50.47, 1.23 46.12, 0 36.43 M0 36.43 C-1.24 30.41, 1.19 22.75, 0 12.14 M0 36.43 C0.64 27.3, 0.8 19.73, 0 12.14 M0 12.14 C1.84 5.33, 5.2 1.23, 12.14 0 M0 12.14 C0.99 3.09, 3.42 1.82, 12.14 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(401.884599231538 243.93247549874414) rotate(0 82.5 12.5)"><text x="82.5" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">rt_isula_create</text></g><g stroke-linecap="round"><g transform="translate(483.9083942231674 280.46823338099864) rotate(0 1.2594418939734737 22.305113040187532)"><path d="M1.17 -0.49 C1.39 7.14, 1.94 38.54, 2.19 46.4 M0.33 -1.79 C0.34 5.44, 1.06 36.87, 1.39 44.64" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(483.9083942231674 280.46823338099864) rotate(0 1.2594418939734737 22.305113040187532)"><path d="M-7.91 23.87 C-2.69 32.51, 0.92 37.37, -0.28 42.84 M-6.4 24.1 C-3.94 29.03, -2.09 34.47, 1.06 44.12" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(483.9083942231674 280.46823338099864) rotate(0 1.2594418939734737 22.305113040187532)"><path d="M7.73 23.39 C7.15 32.36, 4.96 37.4, -0.28 42.84 M9.24 23.62 C7.32 28.83, 4.78 34.41, 1.06 44.12" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(694.8608042399089 225.70629228864362) rotate(0 146.19049072265625 24.285714285714306)"><path d="M3.23 2.81 C3.23 2.81, 3.23 2.81, 3.23 2.81 M3.23 2.81 C3.23 2.81, 3.23 2.81, 3.23 2.81 M0.35 12.23 C3.22 8.65, 7.34 3.58, 10.19 0.91 M0.35 12.23 C3.15 7.84, 6.91 4.5, 10.19 0.91 M0.09 18.62 C3.68 14.19, 6.97 13.09, 14.52 2.02 M0.09 18.62 C4.63 14.28, 8.18 9.85, 14.52 2.02 M0.48 24.27 C4.11 19.95, 11.75 13.55, 19.51 2.38 M0.48 24.27 C5.36 20.7, 7.76 15.04, 19.51 2.38 M0.87 29.91 C5.26 22.28, 11.97 18.71, 25.15 1.99 M0.87 29.91 C9.4 19.72, 18.7 8.4, 25.15 1.99 M0.61 36.31 C9.96 23.48, 19.5 14.01, 30.14 2.35 M0.61 36.31 C10.55 24.89, 18.45 15.06, 30.14 2.35 M1.01 41.95 C13.97 25.73, 27.15 13.77, 35.78 1.95 M1.01 41.95 C12.68 28.2, 23.47 15.32, 35.78 1.95 M3.37 45.33 C13.16 32.54, 27.91 19.91, 40.77 2.31 M3.37 45.33 C13.09 33.05, 22.75 22.61, 40.77 2.31 M6.39 47.95 C21.5 30.56, 37.67 13.97, 46.41 1.92 M6.39 47.95 C14.64 37.76, 23.33 27.42, 46.41 1.92 M10.72 49.07 C25.78 35.11, 34.86 18.7, 51.4 2.28 M10.72 49.07 C21.29 35.55, 33.77 21.51, 51.4 2.28 M15.71 49.43 C33.45 29.93, 49.66 9.33, 57.04 1.88 M15.71 49.43 C25.64 39.18, 33.5 28.56, 57.04 1.88 M21.35 49.03 C31.85 39.86, 40.56 29.43, 62.03 2.24 M21.35 49.03 C35.37 32.44, 49.44 17.46, 62.03 2.24 M26.34 49.39 C41.13 30.34, 54.94 15.66, 67.67 1.85 M26.34 49.39 C34.5 38.25, 45.03 27.65, 67.67 1.85 M31.32 49.75 C42.31 37.51, 56.19 22.54, 72.66 2.21 M31.32 49.75 C46.64 32.35, 62.11 15.31, 72.66 2.21 M36.97 49.36 C44.24 39.42, 54.75 29.16, 78.3 1.81 M36.97 49.36 C48.95 35.3, 59.93 21.84, 78.3 1.81 M41.95 49.72 C53.06 38.58, 63.47 25.93, 83.29 2.17 M41.95 49.72 C50.37 38.87, 59.44 28.29, 83.29 2.17 M47.6 49.33 C61 34.4, 74.38 19.33, 88.93 1.78 M47.6 49.33 C61.37 33.37, 73.58 18.53, 88.93 1.78 M52.58 49.69 C68.63 32.32, 81.1 15.29, 93.92 2.14 M52.58 49.69 C67.57 32.45, 83.41 13.99, 93.92 2.14 M58.23 49.29 C68.24 35.1, 78.07 23.11, 99.56 1.74 M58.23 49.29 C73.82 30.04, 90.25 12.09, 99.56 1.74 M63.21 49.65 C71.96 39.75, 84.83 26.49, 104.55 2.1 M63.21 49.65 C73.79 36.42, 83.9 24.26, 104.55 2.1 M68.86 49.26 C79.66 39.32, 89.08 25.2, 109.53 2.46 M68.86 49.26 C80.43 37.77, 90.77 24.7, 109.53 2.46 M73.84 49.62 C82.64 38.09, 95.31 27.53, 115.18 2.07 M73.84 49.62 C85.13 36.46, 95.82 24.61, 115.18 2.07 M79.49 49.22 C87.71 37.66, 95.02 30.93, 120.16 2.43 M79.49 49.22 C96.44 30.41, 111.52 11.36, 120.16 2.43 M84.47 49.58 C99.33 31.39, 115.31 16.02, 125.81 2.04 M84.47 49.58 C96.6 35.62, 109.34 20.49, 125.81 2.04 M90.12 49.19 C98.65 38.75, 110.31 26.33, 130.79 2.4 M90.12 49.19 C98.97 39.52, 106.29 29.68, 130.79 2.4 M95.1 49.55 C105.91 38.65, 115.74 28, 136.44 2 M95.1 49.55 C108.35 34.37, 120.9 19.64, 136.44 2 M100.75 49.15 C113.47 37.28, 125.69 22.12, 141.42 2.36 M100.75 49.15 C115.09 32.31, 130.99 15, 141.42 2.36 M105.73 49.51 C120.09 31.1, 138.01 13.41, 147.07 1.97 M105.73 49.51 C120.32 30.81, 135.9 13.47, 147.07 1.97 M111.38 49.12 C121.19 39.57, 129.27 28.19, 152.05 2.33 M111.38 49.12 C125.36 33.65, 139.27 17.63, 152.05 2.33 M116.36 49.48 C132.02 32.97, 146.04 15.26, 157.7 1.93 M116.36 49.48 C129.33 34.83, 141.47 19.55, 157.7 1.93 M122.01 49.08 C131.79 36.24, 143.8 22.81, 162.68 2.29 M122.01 49.08 C129.8 38.11, 139.7 28.01, 162.68 2.29 M126.99 49.44 C135.96 37.4, 144.05 26.14, 168.33 1.9 M126.99 49.44 C140.27 34.44, 152.27 20.11, 168.33 1.9 M131.98 49.8 C142.69 35.52, 156.85 22.53, 173.31 2.26 M131.98 49.8 C144.23 35.73, 156.07 21.66, 173.31 2.26 M137.62 49.41 C149.37 36.64, 159.12 24.73, 178.96 1.86 M137.62 49.41 C145.45 39.62, 154.95 29.23, 178.96 1.86 M142.61 49.77 C155.41 35.2, 169.13 19.5, 183.94 2.22 M142.61 49.77 C154.64 35.99, 164.43 22.76, 183.94 2.22 M148.26 49.38 C158.42 37.75, 165.83 24.79, 189.59 1.83 M148.26 49.38 C157.75 37.1, 168.46 24.78, 189.59 1.83 M153.24 49.74 C164.13 34.53, 175.8 22.37, 194.57 2.19 M153.24 49.74 C165.95 35.07, 177.69 22.01, 194.57 2.19 M158.89 49.34 C171.52 32.97, 188.81 14.95, 199.56 2.55 M158.89 49.34 C166.99 39.26, 174.83 29.78, 199.56 2.55 M163.87 49.7 C171.26 39.23, 181.71 27.81, 205.2 2.15 M163.87 49.7 C177.58 35.42, 189.97 20.47, 205.2 2.15 M169.52 49.31 C181.58 35.67, 192.63 20.43, 210.19 2.51 M169.52 49.31 C181.87 36.68, 193.1 24.14, 210.19 2.51 M174.5 49.67 C184.57 39.06, 194.62 26.57, 215.83 2.12 M174.5 49.67 C184.04 37.86, 193.51 26.81, 215.83 2.12 M180.15 49.27 C197.3 29.15, 213.08 13.3, 220.82 2.48 M180.15 49.27 C193.32 32.76, 206.77 16.74, 220.82 2.48 M185.13 49.63 C195.08 37.81, 206.62 23.84, 226.46 2.09 M185.13 49.63 C197.17 36.12, 209.3 23.52, 226.46 2.09 M190.78 49.24 C203.49 38.76, 213.19 24.73, 231.45 2.45 M190.78 49.24 C204.04 32.84, 216.42 18.38, 231.45 2.45 M195.76 49.6 C206 39.13, 211.54 30.79, 237.09 2.05 M195.76 49.6 C212.83 30.6, 229.09 12.51, 237.09 2.05 M201.41 49.2 C214.5 35.45, 229.52 18.56, 242.08 2.41 M201.41 49.2 C210.9 37.28, 221.91 25.73, 242.08 2.41 M206.39 49.56 C219.82 32.36, 236.4 16.07, 247.72 2.02 M206.39 49.56 C216.33 39.27, 224.68 28.71, 247.72 2.02 M212.04 49.17 C228.42 32.41, 241.3 14.11, 252.71 2.38 M212.04 49.17 C226.8 30.82, 242.35 14.62, 252.71 2.38 M217.02 49.53 C229.99 35.89, 243.63 17.56, 258.35 1.98 M217.02 49.53 C228.61 34.41, 241.87 21.04, 258.35 1.98 M222.01 49.89 C234.14 35.43, 250.42 18.2, 263.34 2.34 M222.01 49.89 C231.36 39.26, 240.29 28.08, 263.34 2.34 M227.65 49.49 C244.82 30.77, 260.97 12.59, 268.98 1.95 M227.65 49.49 C240.2 33.58, 253.87 17.69, 268.98 1.95 M232.64 49.85 C244.22 36.04, 253.21 23.95, 273.97 2.31 M232.64 49.85 C249.58 30.66, 265.14 11.11, 273.97 2.31 M238.28 49.46 C253.91 33.6, 266.18 16.71, 279.61 1.91 M238.28 49.46 C251.52 35.67, 263.29 22.37, 279.61 1.91 M243.27 49.82 C254.02 36.39, 265.21 26.54, 285.26 1.52 M243.27 49.82 C258.76 32.6, 273.14 16.92, 285.26 1.52 M248.91 49.43 C264.51 33.59, 280.08 14.44, 288.93 3.39 M248.91 49.43 C260.6 36.06, 271.17 24.16, 288.93 3.39 M253.9 49.79 C264.09 35.87, 276.59 21.78, 291.95 6.01 M253.9 49.79 C262.84 41.17, 269.35 30.88, 291.95 6.01 M259.54 49.39 C269.99 40, 278.93 31.14, 294.31 9.39 M259.54 49.39 C271.97 36.08, 283.33 22.14, 294.31 9.39 M264.53 49.75 C276.15 36.34, 290.59 23.88, 294.71 15.03 M264.53 49.75 C274.34 36.91, 284.93 25.98, 294.71 15.03 M270.17 49.36 C276.86 41.22, 287.93 28.33, 294.45 21.43 M270.17 49.36 C279.48 39.64, 288.21 29.42, 294.45 21.43 M275.16 49.72 C279.93 41.96, 287.07 37.74, 294.84 27.07 M275.16 49.72 C280.21 44.72, 284.63 39.07, 294.84 27.07 M280.8 49.32 C286.62 46.15, 291.31 39.47, 294.58 33.47 M280.8 49.32 C284.52 46.36, 287.41 42.45, 294.58 33.47" stroke="#228be6" stroke-width="0.5" fill="none"></path><path d="M12.14 0 M12.14 0 C68.24 -1.5, 124.73 -0.49, 280.24 0 M12.14 0 C117.96 0.15, 224.7 0.36, 280.24 0 M280.24 0 C288.63 1.53, 292.16 5.42, 292.38 12.14 M280.24 0 C289.23 -0.82, 291.91 3.38, 292.38 12.14 M292.38 12.14 C294.38 21.62, 294.33 30.65, 292.38 36.43 M292.38 12.14 C291.47 20.24, 292.7 27.79, 292.38 36.43 M292.38 36.43 C292.63 44.16, 288.62 47.95, 280.24 48.57 M292.38 36.43 C290.41 45.49, 286.48 47.43, 280.24 48.57 M280.24 48.57 C210.7 48.17, 143.61 46.72, 12.14 48.57 M280.24 48.57 C221.71 49.38, 163.74 48.93, 12.14 48.57 M12.14 48.57 C4.03 49.5, 0.5 43.39, 0 36.43 M12.14 48.57 C5.16 47.29, 0.47 45.33, 0 36.43 M0 36.43 C-0.32 28.5, 1.75 21.61, 0 12.14 M0 36.43 C0.23 29.26, -0.08 21.49, 0 12.14 M0 12.14 C0.18 4.45, 4.43 -0.99, 12.14 0 M0 12.14 C-0.33 4.09, 5.4 -0.11, 12.14 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(769.5512949625652 237.74200657435793) rotate(0 71.5 12.5)"><text x="71.5" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">rt_isula_exec</text></g><g stroke-linecap="round"><g transform="translate(589.2804409540429 110.16247824948725) rotate(0 130.91060415913446 56.48489381949316)"><path d="M0.1 0.01 C43.47 18.77, 217.09 93.55, 260.75 112.38 M-1.31 -1.03 C42.42 17.96, 219.31 95.33, 263.13 114" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(589.2804409540429 110.16247824948725) rotate(0 130.91060415913446 56.48489381949316)"><path d="M233.2 111.17 C243.72 113.88, 254.95 111.66, 264.51 113.41 M232.48 113.19 C240.73 111.58, 246.63 112.99, 263.65 113.75" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(589.2804409540429 110.16247824948725) rotate(0 130.91060415913446 56.48489381949316)"><path d="M241.35 92.34 C249.04 101.51, 257.48 105.74, 264.51 113.41 M240.63 94.35 C246.9 96.96, 251 102.55, 263.65 113.75" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(684.7657201857792 449.9473521381349) rotate(0 131 30)"><path d="M15 0 M15 0 C86.85 0.75, 156.54 0.31, 247 0 M15 0 C72.24 -0.96, 128.6 0.01, 247 0 M247 0 C256.22 -0.57, 260.8 6.65, 262 15 M247 0 C258.81 -1.88, 260.97 6.41, 262 15 M262 15 C263.4 21.08, 261.49 28.54, 262 45 M262 15 C261.92 22.41, 261.18 27.74, 262 45 M262 45 C261.09 56.24, 255.83 58.83, 247 60 M262 45 C264.17 53.26, 259.26 61.58, 247 60 M247 60 C175 60.45, 101.9 59.27, 15 60 M247 60 C190.45 62.55, 134.19 62.46, 15 60 M15 60 C3.19 59.28, -0.35 54.43, 0 45 M15 60 C6.75 61.7, 0.85 55.04, 0 45 M0 45 C-1.27 37.53, 1.5 31.18, 0 15 M0 45 C0.66 36.26, 0.62 28.96, 0 15 M0 15 C-1.69 5.27, 3.15 -0.48, 15 0 M0 15 C-2.13 2.77, 6.73 2.16, 15 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(730.7657201857792 467.4473521381349) rotate(0 85 12.5)"><text x="85" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">get_runtime_cmd</text></g><g stroke-linecap="round"><g transform="translate(815.5913572347524 418.70777231699293) rotate(0 0.3378159892353665 14.637980658334314)"><path d="M0.9 -0.81 C0.93 3.94, 0.86 23.78, 0.51 28.9 M-0.09 1.37 C-0.19 6.25, -0.33 25.69, -0.13 30.09" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(815.5913572347524 418.70777231699293) rotate(0 0.3378159892353665 14.637980658334314)"><path d="M-6.38 17 C-4.43 19.37, -1.47 25.98, -0.79 30.01 M-5.82 16.88 C-2.91 20.79, -1.88 26.96, -0.44 30.34" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(815.5913572347524 418.70777231699293) rotate(0 0.3378159892353665 14.637980658334314)"><path d="M3.73 16.86 C2.38 19.35, 2.04 26, -0.79 30.01 M4.29 16.74 C3.46 20.68, 0.75 26.9, -0.44 30.34" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(691.4324071974979 555.9473521381349) rotate(0 126 30)"><path d="M15 0 M15 0 C89.09 0.35, 159.88 1.81, 237 0 M15 0 C93.14 -0.24, 169.83 -0.42, 237 0 M237 0 C248.65 1.57, 250.36 4.11, 252 15 M237 0 C248.41 -0.96, 252.77 4.18, 252 15 M252 15 C249.78 24.24, 250.36 38.76, 252 45 M252 15 C252.15 23.94, 252.91 33.2, 252 45 M252 45 C250.83 56.88, 245.49 61.97, 237 60 M252 45 C253.58 55.22, 246.12 60.71, 237 60 M237 60 C171.11 57.44, 106.85 60.05, 15 60 M237 60 C168.17 60.38, 97.85 61.01, 15 60 M15 60 C4.43 61.53, 1.48 55.74, 0 45 M15 60 C5.04 58.61, -2.21 54.23, 0 45 M0 45 C0.59 37.53, 1.31 30.99, 0 15 M0 45 C-0.17 32.61, 0.42 20.99, 0 15 M0 15 C-0.48 3.15, 3.06 1.5, 15 0 M0 15 C2.16 4.06, 6.32 1.71, 15 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(725.4324071974979 573.4473521381349) rotate(0 92 12.5)"><text x="92" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">ret = shim_create</text></g><g stroke-linecap="round"><g transform="translate(812.4012947253991 617.9473521381349) rotate(0 -2.1876358274671475 53.5160467881708)"><path d="M-0.66 -1.11 C-1.33 16.81, -3.97 90.05, -4.81 108.15 M1.19 0.91 C0.47 18.44, -4.25 88.36, -5.57 106.33" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(812.4012947253991 617.9473521381349) rotate(0 -2.1876358274671475 53.5160467881708)"><path d="M-15.7 78.09 C-10.8 83.35, -9.96 92.33, -6.35 105.77 M-14.66 77.05 C-11.01 83.94, -9.3 90.92, -4.92 105.5" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(812.4012947253991 617.9473521381349) rotate(0 -2.1876358274671475 53.5160467881708)"><path d="M4.77 79.52 C4.91 84.33, 0.99 92.99, -6.35 105.77 M5.81 78.48 C4.54 85.18, 1.33 91.82, -4.92 105.5" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round"><g transform="translate(812.7657201857792 510.6140086322755) rotate(0 0.608110223859569 21.98329870782794)"><path d="M0.36 0.59 C0.82 7.88, 1.61 35.67, 2.13 42.61 M-0.92 -0.14 C-0.6 7.44, 0.22 36.88, 0.9 44.11" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(812.7657201857792 510.6140086322755) rotate(0 0.608110223859569 21.98329870782794)"><path d="M-6.53 23.57 C-5.74 30.65, -4.36 33.5, -0.3 45.76 M-6.9 24.05 C-4.16 31.59, -0.65 38.55, 0.6 43.13" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(812.7657201857792 510.6140086322755) rotate(0 0.608110223859569 21.98329870782794)"><path d="M8.06 22.76 C5.27 30.07, 3.08 33.12, -0.3 45.76 M7.69 23.23 C4.91 31.01, 2.88 38.28, 0.6 43.13" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(666.0989721389042 726.8362376361815) rotate(0 144 29.5)"><path d="M14.75 0 M14.75 0 C67.57 -1.1, 122.95 -1.65, 273.25 0 M14.75 0 C85.61 -2.37, 154.86 -1.22, 273.25 0 M273.25 0 C282.25 0.67, 287.29 6.21, 288 14.75 M273.25 0 C281.16 -0.7, 285.74 2.96, 288 14.75 M288 14.75 C290.22 20.33, 288 27.44, 288 44.25 M288 14.75 C288.77 24.68, 288.81 36.81, 288 44.25 M288 44.25 C288.19 53.32, 283.7 58.81, 273.25 59 M288 44.25 C287.78 52.16, 281.27 57.72, 273.25 59 M273.25 59 C213.23 60.23, 151.75 57.81, 14.75 59 M273.25 59 C205.66 59, 137.14 58.42, 14.75 59 M14.75 59 C3.71 57.08, -0.67 52.53, 0 44.25 M14.75 59 C4.59 60.4, 0.63 53.16, 0 44.25 M0 44.25 C-1.8 35.15, -0.81 21.12, 0 14.75 M0 44.25 C-0.65 34.04, 1.02 26.7, 0 14.75 M0 14.75 C-0.82 6.07, 6.4 -0.29, 14.75 0 M0 14.75 C-0.46 3.16, 6.26 0.99, 14.75 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(675.0989721389042 743.8362376361815) rotate(0 135 12.5)"><text x="135" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">get_container_process_pid</text></g><g stroke-linecap="round" transform="translate(687.6227627708798 323.4840727306546) rotate(0 139 42)"><path d="M21 0 M21 0 C76.51 -2.33, 129.26 -2.53, 257 0 M21 0 C85.55 -1.19, 151 -1.13, 257 0 M257 0 C269.75 1.72, 279.34 5.77, 278 21 M257 0 C273.25 -1.82, 277.41 8.38, 278 21 M278 21 C276.91 31.61, 277.64 44.58, 278 63 M278 21 C278.41 34.66, 277.84 48.55, 278 63 M278 63 C276.72 75.31, 269.18 83.25, 257 84 M278 63 C276.85 78.52, 272.8 82.15, 257 84 M257 84 C176.93 84.94, 96.25 83.95, 21 84 M257 84 C187.87 83.82, 119.27 83.31, 21 84 M21 84 C8.74 85.23, -1.47 76.36, 0 63 M21 84 C7.29 83.05, 0.85 75.78, 0 63 M0 63 C1.79 55.96, 0.13 44.56, 0 21 M0 63 C-0.24 54.63, -1.32 45.24, 0 21 M0 21 C0.31 8.62, 7.21 -1.02, 21 0 M0 21 C-2.1 7.36, 6.35 1.51, 21 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(698.1227627708798 328.4840727306546) rotate(0 128.5 37)"><text x="128.5" y="17.666666666666668" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">将exec的process存储在proc</text><text x="128.5" y="42.333333333333336" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">ess.jsonwe文件中</text><text x="128.5" y="67" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">create_process_json_file</text></g><g stroke-linecap="round"><g transform="translate(823.9084378197078 270.9443729945589) rotate(0 -2.217964328346511 28.95299554801412)"><path d="M0.06 -0.19 C-0.69 9.7, -3.75 49.46, -4.49 59.25 M-1.37 -1.34 C-1.73 8.17, -1.45 47.08, -2.14 57.23" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(823.9084378197078 270.9443729945589) rotate(0 -2.217964328346511 28.95299554801412)"><path d="M-11.85 28.03 C-8.39 36.6, -6.79 47.81, -0.91 55.76 M-11.16 28.64 C-9.86 36.25, -7.04 45.03, -1.2 57.19" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(823.9084378197078 270.9443729945589) rotate(0 -2.217964328346511 28.95299554801412)"><path d="M8.38 28.56 C5.75 36.98, 1.26 48.03, -0.91 55.76 M9.07 29.17 C4.96 36.55, 2.35 45.2, -1.2 57.19" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(11.388727823892623 326.4087186540875) rotate(0 142 33.5)"><path d="M16.75 0 M16.75 0 C109.43 -0.1, 199.58 1.56, 267.25 0 M16.75 0 C77.79 2.28, 137.84 1.18, 267.25 0 M267.25 0 C277.56 0.21, 285.69 6.18, 284 16.75 M267.25 0 C279.56 -0.91, 282.67 7.07, 284 16.75 M284 16.75 C282.74 26.82, 284.5 33.95, 284 50.25 M284 16.75 C283.76 27.62, 283.52 37.3, 284 50.25 M284 50.25 C283.06 59.79, 280.35 67.87, 267.25 67 M284 50.25 C281.81 61.54, 278.32 65.42, 267.25 67 M267.25 67 C197.17 65.83, 127.1 68.34, 16.75 67 M267.25 67 C190.26 65.12, 114.54 65.91, 16.75 67 M16.75 67 C3.73 67.95, -0.39 62.84, 0 50.25 M16.75 67 C6.55 67.58, -1.47 60.35, 0 50.25 M0 50.25 C1.57 38.67, 1.26 24.17, 0 16.75 M0 50.25 C0.5 38.43, 0.89 26.8, 0 16.75 M0 16.75 C-0.27 7.25, 5.54 1.2, 16.75 0 M0 16.75 C-0.37 3.55, 5.84 -1.09, 16.75 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(18.388727823892623 347.4087186540875) rotate(0 135 12.5)"><text x="135" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">get_container_process_pid</text></g><g stroke-linecap="round" transform="translate(26.34115527924996 443.98013850620805) rotate(0 131 30)"><path d="M15 0 M15 0 C69.83 1.96, 123.41 2.63, 247 0 M15 0 C64.57 -1.94, 115.42 -1.33, 247 0 M247 0 C258.69 0.6, 262.99 4.21, 262 15 M247 0 C255.67 1.49, 260.86 6.97, 262 15 M262 15 C261.99 25.36, 261.71 33.68, 262 45 M262 15 C261.71 22.27, 261.52 27.46, 262 45 M262 45 C263.93 55.87, 255.1 60.11, 247 60 M262 45 C261.9 53.42, 256.47 59.7, 247 60 M247 60 C172.44 59.29, 100.96 57.48, 15 60 M247 60 C192.31 58.9, 137.07 59.05, 15 60 M15 60 C4.61 61.42, 0.84 55.5, 0 45 M15 60 C3.53 58.94, 1.95 54.97, 0 45 M0 45 C1.03 36.83, 0.9 28.81, 0 15 M0 45 C0.7 37.26, -0.26 29.44, 0 15 M0 15 C-0.05 6.2, 4.68 -1.77, 15 0 M0 15 C0.25 3.91, 5.34 -1.05, 15 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(33.34115527924996 449.48013850620805) rotate(0 124 24.5)"><text x="124" y="17.5" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">file_read_int(shim_pid_f</text><text x="124" y="42" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">ile_name, &shim_pid);</text></g><g stroke-linecap="round"><g transform="translate(153.90737443619605 394.4137188074384) rotate(0 0.6736516382710533 23.784211160763718)"><path d="M0.18 -0.68 C0.78 7.17, 2.23 38.94, 2.53 47.23 M-1.18 1.58 C-0.6 9.54, 1.51 40.64, 1.93 48.25" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(153.90737443619605 394.4137188074384) rotate(0 0.6736516382710533 23.784211160763718)"><path d="M-8.86 27.22 C-7.3 31.05, -2.15 33.59, 0.68 47.39 M-7.23 25.73 C-5.12 33.61, -2.04 40.68, 2.24 47.48" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(153.90737443619605 394.4137188074384) rotate(0 0.6736516382710533 23.784211160763718)"><path d="M7.62 26.19 C5.66 30.27, 7.31 33.03, 0.68 47.39 M9.25 24.71 C6.25 32.95, 4.22 40.34, 2.24 47.48" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(27.29355657668293 551.88487134661) rotate(0 126 30)"><path d="M15 0 M15 0 C84.49 -0.34, 152.29 2.33, 237 0 M15 0 C60.06 1.55, 105.2 1.25, 237 0 M237 0 C246.21 -1.15, 253.29 4.01, 252 15 M237 0 C248.97 0.72, 250.23 6.93, 252 15 M252 15 C250.87 22.75, 251.98 35.47, 252 45 M252 15 C251.38 24.13, 252.63 30.36, 252 45 M252 45 C252.11 54.92, 245.62 59.54, 237 60 M252 45 C251.7 55.34, 245.96 57.91, 237 60 M237 60 C189.4 58.97, 141.84 58.9, 15 60 M237 60 C173.93 58.05, 110.67 58.58, 15 60 M15 60 C5.5 58.72, -0.92 56.7, 0 45 M15 60 C4.97 59.15, 1.8 56.52, 0 45 M0 45 C-0.37 33.58, 0.9 23.5, 0 15 M0 45 C0.7 33.5, 0.46 21.89, 0 15 M0 15 C-1.77 5.22, 4.05 0.3, 15 0 M0 15 C-1.05 4.37, 3.23 -0.58, 15 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(39.79355657668293 557.38487134661) rotate(0 113.5 24.5)"><text x="113.5" y="17.5" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">util_get_process_proc</text><text x="113.5" y="42" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">_info</text></g><g stroke-linecap="round"><g transform="translate(150.0267409029093 613.88487134661) rotate(0 -0.0022104730946921336 33.23373851922196)"><path d="M-0.99 0.13 C-0.72 11.09, 0.93 55.24, 0.99 66.39 M0.69 -0.84 C0.94 10.16, 0.3 55.82, 0.5 67.31" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(150.0267409029093 613.88487134661) rotate(0 -0.0022104730946921336 33.23373851922196)"><path d="M-9.54 40.56 C-9.06 44.82, -5.48 53.13, 2.19 67.91 M-9.12 38.62 C-5.43 49.35, -3.47 59.74, 0.6 67.52" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(150.0267409029093 613.88487134661) rotate(0 -0.0022104730946921336 33.23373851922196)"><path d="M10.98 40.56 C6.47 44.65, 5.06 52.96, 2.19 67.91 M11.4 38.62 C7.91 49.23, 2.7 59.62, 0.6 67.52" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round"><g transform="translate(153.0579642117923 504.98013850620805) rotate(0 0.5731453417457715 21.90952689887297)"><path d="M0.87 -1.17 C0.93 6.1, -0.49 35.98, -0.59 43.72 M-0.14 0.83 C0.4 8.24, 1.78 37.89, 1.73 44.99" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(153.0579642117923 504.98013850620805) rotate(0 0.5731453417457715 21.90952689887297)"><path d="M-8.3 23.15 C-5.43 29.02, -1.12 33.89, 2.73 44.19 M-5.49 24.71 C-5.03 30.27, -2.19 34.63, 1.71 44.39" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(153.0579642117923 504.98013850620805) rotate(0 0.5731453417457715 21.90952689887297)"><path d="M6.83 22.78 C6.41 28.83, 7.43 33.78, 2.73 44.19 M9.63 24.33 C6.43 29.91, 5.61 34.36, 1.71 44.39" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(10.531550089518078 682.7738440377368) rotate(0 144 42.00000000000003)"><path d="M21 0 M21 0 C96.51 1.27, 174.23 3.37, 267 0 M21 0 C101.72 1.34, 182.99 1.51, 267 0 M267 0 C281.63 -1.54, 289.68 7.22, 288 21 M267 0 C281.49 -0.06, 286.62 5.19, 288 21 M288 21 C289.64 32.49, 286.76 47.79, 288 63 M288 21 C287.56 34.75, 287.37 49.74, 288 63 M288 63 C288.3 76.1, 279.19 82.22, 267 84 M288 63 C289.46 77.11, 280 82.51, 267 84 M267 84 C214.59 82.19, 162.02 83.39, 21 84 M267 84 C175.15 84.13, 85.05 84.5, 21 84 M21 84 C6.26 85.56, 1.32 78.26, 0 63 M21 84 C6.67 85.31, -0.48 76.39, 0 63 M0 63 C0.87 48.34, 1.61 34.16, 0 21 M0 63 C0.18 47.4, 0.22 32.3, 0 21 M0 21 C-0.54 5.46, 6.5 -0.42, 21 0 M0 21 C-0.07 5.45, 6.36 -0.82, 21 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(19.531550089518078 687.7738440377368) rotate(0 135 37)"><text x="135" y="17.666666666666668" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">runtime_call_simple(workdir,</text><text x="135" y="42.333333333333336" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">runtime, "start", NULL, 0, </text><text x="135" y="67" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">id, NULL)</text></g><g stroke-linecap="round" transform="translate(10 234.40065983363513) rotate(0 146.19049072265625 24.285714285714306)"><path d="M3.23 2.81 C3.23 2.81, 3.23 2.81, 3.23 2.81 M3.23 2.81 C3.23 2.81, 3.23 2.81, 3.23 2.81 M0.35 12.23 C4.07 6.96, 6.71 3.45, 10.19 0.91 M0.35 12.23 C3.88 7.13, 8.8 3.64, 10.19 0.91 M0.09 18.62 C1.66 14.77, 5.73 12.13, 14.52 2.02 M0.09 18.62 C4.55 14.57, 7.16 9.64, 14.52 2.02 M0.48 24.27 C5.29 17.73, 9.76 14.49, 19.51 2.38 M0.48 24.27 C6.81 15.63, 15.36 7.96, 19.51 2.38 M0.87 29.91 C8.3 22.12, 14.56 14.65, 25.15 1.99 M0.87 29.91 C8.21 22.71, 13.89 15.45, 25.15 1.99 M0.61 36.31 C7.43 26.47, 17.03 16.44, 30.14 2.35 M0.61 36.31 C7.96 27.69, 15.68 18.6, 30.14 2.35 M1.01 41.95 C10.66 31.79, 20.07 20.51, 35.78 1.95 M1.01 41.95 C8.99 32.56, 15.81 24.21, 35.78 1.95 M3.37 45.33 C17.56 29.19, 30.11 13.71, 40.77 2.31 M3.37 45.33 C11.45 36.36, 18.77 26.9, 40.77 2.31 M6.39 47.95 C21.05 29.37, 38.34 13.29, 46.41 1.92 M6.39 47.95 C17.78 33.94, 28.91 20.73, 46.41 1.92 M10.72 49.07 C27.4 30.25, 40.76 12.95, 51.4 2.28 M10.72 49.07 C21.42 37.31, 30.91 24.84, 51.4 2.28 M15.71 49.43 C31.28 33.44, 43.67 16.91, 57.04 1.88 M15.71 49.43 C27.6 35.1, 41.55 19.67, 57.04 1.88 M21.35 49.03 C31.1 35.71, 42.41 24.19, 62.03 2.24 M21.35 49.03 C36.99 32.1, 52.52 13.54, 62.03 2.24 M26.34 49.39 C39.49 34.7, 52.46 20.88, 67.67 1.85 M26.34 49.39 C35.41 39.24, 42.89 29.3, 67.67 1.85 M31.32 49.75 C47.47 32.09, 60.54 15.12, 72.66 2.21 M31.32 49.75 C41.05 37.35, 51.55 25.44, 72.66 2.21 M36.97 49.36 C49.32 35.67, 58.14 24.39, 78.3 1.81 M36.97 49.36 C50.9 34.38, 65.35 17.6, 78.3 1.81 M41.95 49.72 C48.74 41.32, 57.12 30.89, 83.29 2.17 M41.95 49.72 C51.12 36.38, 61.84 25.96, 83.29 2.17 M47.6 49.33 C57.45 39.75, 66.28 28.11, 88.93 1.78 M47.6 49.33 C61.02 33.68, 75.26 17.38, 88.93 1.78 M52.58 49.69 C64.05 36.42, 78.3 22.71, 93.92 2.14 M52.58 49.69 C64.32 36.63, 74.93 23.76, 93.92 2.14 M58.23 49.29 C70.17 33.73, 83.9 19.25, 99.56 1.74 M58.23 49.29 C70.06 35.28, 80.78 23.38, 99.56 1.74 M63.21 49.65 C77.76 32, 89.78 16.01, 104.55 2.1 M63.21 49.65 C76.41 34.29, 90.87 17.43, 104.55 2.1 M68.86 49.26 C77.83 37.28, 88.7 26.58, 109.53 2.46 M68.86 49.26 C82.39 33.74, 96.04 16.26, 109.53 2.46 M73.84 49.62 C87.83 31.09, 104.82 16.63, 115.18 2.07 M73.84 49.62 C88.58 33, 100.89 18.09, 115.18 2.07 M79.49 49.22 C93.76 32.57, 110.37 15.63, 120.16 2.43 M79.49 49.22 C92.44 35, 103.29 20.24, 120.16 2.43 M84.47 49.58 C101.02 30.32, 115.08 14.41, 125.81 2.04 M84.47 49.58 C98.1 34.13, 110.48 19.45, 125.81 2.04 M90.12 49.19 C98.85 39.98, 108.37 29.69, 130.79 2.4 M90.12 49.19 C100.47 36.5, 110.3 25.58, 130.79 2.4 M95.1 49.55 C108.15 33.74, 121.73 21.66, 136.44 2 M95.1 49.55 C104.63 39.16, 113.24 28.44, 136.44 2 M100.75 49.15 C114.53 31.6, 131.28 16.09, 141.42 2.36 M100.75 49.15 C113.88 34.62, 126.06 19.28, 141.42 2.36 M105.73 49.51 C118.07 33.66, 133.83 19.78, 147.07 1.97 M105.73 49.51 C117.24 35.44, 130.63 21.2, 147.07 1.97 M111.38 49.12 C124.43 33.47, 140.64 16.25, 152.05 2.33 M111.38 49.12 C122.84 34.73, 136.08 21.34, 152.05 2.33 M116.36 49.48 C130.53 32.69, 144.46 17.07, 157.7 1.93 M116.36 49.48 C128.95 35.46, 141.25 21.16, 157.7 1.93 M122.01 49.08 C130.94 38.11, 140.72 26.67, 162.68 2.29 M122.01 49.08 C129.76 37.99, 140.25 28.2, 162.68 2.29 M126.99 49.44 C142.1 34.64, 157.45 15.85, 168.33 1.9 M126.99 49.44 C139.15 35.6, 150.5 20.93, 168.33 1.9 M131.98 49.8 C145.28 34.99, 160.56 19.2, 173.31 2.26 M131.98 49.8 C142.57 38.43, 152.06 26.25, 173.31 2.26 M137.62 49.41 C147.5 40.82, 155.92 28.53, 178.96 1.86 M137.62 49.41 C148.53 36.68, 160.86 22.72, 178.96 1.86 M142.61 49.77 C156.02 32.25, 169.07 17.86, 183.94 2.22 M142.61 49.77 C154.57 34.8, 167.81 20.49, 183.94 2.22 M148.26 49.38 C161.7 35.4, 177.6 16.19, 189.59 1.83 M148.26 49.38 C162.13 32.57, 175.24 16.62, 189.59 1.83 M153.24 49.74 C162.45 37.71, 170.13 28.36, 194.57 2.19 M153.24 49.74 C164.58 36.2, 175.51 22.53, 194.57 2.19 M158.89 49.34 C169.77 37.71, 178.59 28.37, 199.56 2.55 M158.89 49.34 C170.66 36.97, 181.61 23.37, 199.56 2.55 M163.87 49.7 C174.25 34.92, 189.94 22.01, 205.2 2.15 M163.87 49.7 C173.87 39.94, 181.67 28.88, 205.2 2.15 M169.52 49.31 C178.91 37.47, 188.2 24.76, 210.19 2.51 M169.52 49.31 C180.12 37.51, 188.17 28.33, 210.19 2.51 M174.5 49.67 C189.49 33, 201.92 16.14, 215.83 2.12 M174.5 49.67 C187.28 33.54, 202.49 17.41, 215.83 2.12 M180.15 49.27 C189.55 39.98, 197.47 27.25, 220.82 2.48 M180.15 49.27 C197.23 31.39, 213.12 10.88, 220.82 2.48 M185.13 49.63 C198.92 37.2, 211.95 19.46, 226.46 2.09 M185.13 49.63 C200.75 30.68, 218.68 11.19, 226.46 2.09 M190.78 49.24 C200.23 38.72, 213.45 26.62, 231.45 2.45 M190.78 49.24 C203.57 34, 217.25 16.76, 231.45 2.45 M195.76 49.6 C202.26 41.83, 210.54 29.65, 237.09 2.05 M195.76 49.6 C208.7 34.86, 221.49 18.33, 237.09 2.05 M201.41 49.2 C214.55 36.85, 225.8 22.33, 242.08 2.41 M201.41 49.2 C212.49 34.56, 225.52 23.09, 242.08 2.41 M206.39 49.56 C214.98 38.16, 223.72 28.59, 247.72 2.02 M206.39 49.56 C215.52 38.77, 226.41 26.6, 247.72 2.02 M212.04 49.17 C227.99 32, 241.59 14.71, 252.71 2.38 M212.04 49.17 C220.42 38.08, 229.72 28.48, 252.71 2.38 M217.02 49.53 C231.34 32, 247.71 14.13, 258.35 1.98 M217.02 49.53 C231.11 33.72, 246.46 15.64, 258.35 1.98 M222.01 49.89 C237.62 34.46, 251.24 15.45, 263.34 2.34 M222.01 49.89 C237.01 32.54, 253.23 14.53, 263.34 2.34 M227.65 49.49 C241.53 34.69, 253.59 21.39, 268.98 1.95 M227.65 49.49 C240.23 34.27, 252.72 20.8, 268.98 1.95 M232.64 49.85 C249.8 30.12, 263.5 13.63, 273.97 2.31 M232.64 49.85 C248.62 32.65, 262.91 14.78, 273.97 2.31 M238.28 49.46 C247.55 36.4, 256.78 27.75, 279.61 1.91 M238.28 49.46 C249.72 36.59, 258.97 25.27, 279.61 1.91 M243.27 49.82 C257.72 33.3, 272.07 18.2, 285.26 1.52 M243.27 49.82 C255.34 37.11, 266.47 22.75, 285.26 1.52 M248.91 49.43 C260.23 38.19, 268.39 28.04, 288.93 3.39 M248.91 49.43 C263.52 32.93, 279.52 14.87, 288.93 3.39 M253.9 49.79 C268.04 31.08, 283.91 12.96, 291.95 6.01 M253.9 49.79 C265.41 35.84, 277.99 22.18, 291.95 6.01 M259.54 49.39 C267.25 41.97, 275.72 31.52, 294.31 9.39 M259.54 49.39 C270.3 37.92, 280.63 23.66, 294.31 9.39 M264.53 49.75 C275.4 35.04, 289.06 21.72, 294.71 15.03 M264.53 49.75 C275.81 37.54, 286.82 23.06, 294.71 15.03 M270.17 49.36 C280.17 39.95, 287 26.88, 294.45 21.43 M270.17 49.36 C278.14 39.54, 287.81 29.51, 294.45 21.43 M275.16 49.72 C280.86 45.6, 284.7 38.68, 294.84 27.07 M275.16 49.72 C280.2 42.52, 288.02 35.73, 294.84 27.07 M280.8 49.32 C283.09 43.47, 288.53 39.29, 294.58 33.47 M280.8 49.32 C285.4 44.36, 289.85 40.1, 294.58 33.47" stroke="#228be6" stroke-width="0.5" fill="none"></path><path d="M12.14 0 M12.14 0 C92.87 0.98, 175.24 0.55, 280.24 0 M12.14 0 C73.27 1.2, 136.07 0.81, 280.24 0 M280.24 0 C287.11 0.43, 293.66 2.41, 292.38 12.14 M280.24 0 C288.25 -2, 294.18 5.65, 292.38 12.14 M292.38 12.14 C292.89 22.63, 290.44 29.2, 292.38 36.43 M292.38 12.14 C292.43 18.55, 292.69 23.34, 292.38 36.43 M292.38 36.43 C290.59 45.65, 286.56 46.96, 280.24 48.57 M292.38 36.43 C290.69 45.55, 289.88 47.84, 280.24 48.57 M280.24 48.57 C185.95 51.28, 88.9 51.95, 12.14 48.57 M280.24 48.57 C219.93 47.79, 160.52 47.26, 12.14 48.57 M12.14 48.57 C6.04 48.2, 0.08 45.13, 0 36.43 M12.14 48.57 C3.44 50.28, 1.97 46.29, 0 36.43 M0 36.43 C0.41 33.04, -1.63 24.79, 0 12.14 M0 36.43 C0.61 30.08, 0.71 24.49, 0 12.14 M0 12.14 C-1.84 4.65, 3.13 1.54, 12.14 0 M0 12.14 C0.59 4.23, 5.49 1.49, 12.14 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(78.19049072265625 246.18637411934944) rotate(0 78 12.5)"><text x="78" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">rt_isula_start</text></g><g stroke-linecap="round"><g transform="translate(155.74189746333582 283.97208840506374) rotate(0 1.281561442504426 22.991011188551813)"><path d="M1.02 0.36 C1.51 7.87, 2.45 37.51, 2.47 45.19 M0.09 -0.5 C0.53 7.17, 1.84 38.98, 1.84 46.48" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(155.74189746333582 283.97208840506374) rotate(0 1.281561442504426 22.991011188551813)"><path d="M-5.9 26.19 C-5.13 29.44, -0.9 35.07, 3.55 47.1 M-6.29 25.17 C-5.06 29.14, -2.64 35.59, 1.56 47.11" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(155.74189746333582 283.97208840506374) rotate(0 1.281561442504426 22.991011188551813)"><path d="M9.73 25.81 C6.48 29.29, 6.7 35.02, 3.55 47.1 M9.34 24.79 C7.09 28.87, 6.02 35.39, 1.56 47.11" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round"><g transform="translate(404.2882171875299 83.26094257144848) rotate(0 -123.4198593636857 74.8395251845869)"><path d="M0.14 0.52 C-41.21 25.21, -206.09 124.06, -246.98 149 M-1.24 -0.26 C-42.34 24.46, -204.11 124.64, -244.84 149.94" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(404.2882171875299 83.26094257144848) rotate(0 -123.4198593636857 74.8395251845869)"><path d="M-225.42 124.56 C-235.34 136.8, -240.7 146.42, -245.27 150.14 M-226.29 125.45 C-230.68 132, -235.74 136.85, -243.85 149.86" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(404.2882171875299 83.26094257144848) rotate(0 -123.4198593636857 74.8395251845869)"><path d="M-214.61 142 C-228.92 147.37, -238.55 150.11, -245.27 150.14 M-215.48 142.9 C-222.34 145.2, -229.98 145.88, -243.85 149.86" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(24.979883466447745 1039.8492765880762) rotate(0 470.6666259765625 679.3333892822263)"><path d="M32 0 M32 0 C288.32 -0.06, 543.9 0.09, 909.33 0 M32 0 C335.97 2.36, 640.78 2.65, 909.33 0 M909.33 0 C931.22 -1.66, 939.37 11.91, 941.33 32 M909.33 0 C931.9 -0.08, 939.12 9.01, 941.33 32 M941.33 32 C940.13 419.03, 939.81 805.08, 941.33 1326.67 M941.33 32 C940.04 315.18, 940.6 598.54, 941.33 1326.67 M941.33 1326.67 C942.19 1347.79, 930.41 1358.47, 909.33 1358.67 M941.33 1326.67 C939.42 1347.63, 930.45 1358.24, 909.33 1358.67 M909.33 1358.67 C572.5 1360.18, 237.72 1360.51, 32 1358.67 M909.33 1358.67 C664.02 1359.13, 418.3 1358.44, 32 1358.67 M32 1358.67 C10.85 1358.03, 0.85 1349.97, 0 1326.67 M32 1358.67 C12.28 1359.55, -0.36 1347.01, 0 1326.67 M0 1326.67 C2.82 827.92, 3.24 328.34, 0 32 M0 1326.67 C-1.53 919.04, -1.61 511.43, 0 32 M0 32 C-1.36 12.6, 8.75 -1.91, 32 0 M0 32 C1.02 9.25, 12.42 -0.53, 32 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g mask="url(#mask-_XK7gBYiKkyROZoCNNEl-)" stroke-linecap="round"><g transform="translate(620.979944501604 582.5160246349513) rotate(0 -61.09130947521555 164.4754069406539)"><path d="M-0.98 -0.99 C1.54 42.4, 37.87 204.63, 14.15 259.79 C-9.56 314.95, -117.08 318.25, -143.27 329.94 M0.71 1.1 C3.22 44.23, 37.81 203.78, 13.63 258.26 C-10.54 312.73, -118.27 316.34, -144.35 327.92" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(620.979944501604 582.5160246349513) rotate(0 -61.09130947521555 164.4754069406539)"><path d="M-121 312.73 C-123.07 313.81, -129.44 317.96, -143.29 327.1 M-119.15 311.64 C-126.85 317.9, -136.54 323.09, -144.78 327.73" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(620.979944501604 582.5160246349513) rotate(0 -61.09130947521555 164.4754069406539)"><path d="M-116.2 332.68 C-119.24 329.32, -126.67 329.07, -143.29 327.1 M-114.35 331.59 C-123.66 331.17, -134.97 329.63, -144.78 327.73" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask id="mask-_XK7gBYiKkyROZoCNNEl-"><rect x="0" y="0" fill="#fff" width="876.979944501604" height="1011.8493681408107"></rect><rect x="552.813440595354" y="816.6827421642482" fill="#000" width="163" height="49" opacity="1"></rect></mask><g transform="translate(552.813440595354 816.6827421642482) rotate(0 7.075194431034447 -69.69131058864298)"><text x="81.5" y="17.5" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">fg=false;</text><text x="81.5" y="42" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">exit_code=NULL</text></g><g stroke-linecap="round" transform="translate(287.64663151332275 914.5159941173732) rotate(0 155.33334350585938 42)"><path d="M21 0 M21 0 C100.05 0.16, 181.14 0.36, 289.67 0 M21 0 C78.86 0.49, 136.58 0.94, 289.67 0 M289.67 0 C302.63 -0.44, 309.28 5.73, 310.67 21 M289.67 0 C302.98 2.11, 310.07 6.91, 310.67 21 M310.67 21 C309.95 34.43, 311.61 46.96, 310.67 63 M310.67 21 C311.46 33.41, 310.67 44.82, 310.67 63 M310.67 63 C311.44 75.63, 303.65 84.16, 289.67 84 M310.67 63 C309.22 77.37, 301.58 86.29, 289.67 84 M289.67 84 C228.08 85.24, 168.52 85.85, 21 84 M289.67 84 C184.31 83.91, 79.62 85.26, 21 84 M21 84 C6.97 85.08, -0.44 78.65, 0 63 M21 84 C9.12 85.24, -1.26 78.31, 0 63 M0 63 C-2.25 50.63, -1.45 37.06, 0 21 M0 63 C-0.94 48.29, 0.58 31.17, 0 21 M0 21 C0.5 6.84, 7.1 -0.6, 21 0 M0 21 C-0.44 6.14, 7.01 1.32, 21 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(381.9799750191821 944.2659941173732) rotate(0 61 12.5)"><text x="61" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">shim_create</text></g><g stroke-linecap="round"><g transform="translate(437.94149785627974 999.5159941173732) rotate(0 0.07628038784321234 33.80897586798295)"><path d="M1.08 1.05 C0.96 11.89, 0.29 55.2, -0.15 66.03 M0.18 0.55 C-0.11 11.43, -1.02 56.01, -0.92 67.06" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(437.94149785627974 999.5159941173732) rotate(0 0.07628038784321234 33.80897586798295)"><path d="M-9.2 39.36 C-8.86 46.72, -5.33 59.91, -0.94 68.14 M-10.41 38.25 C-7.36 46.42, -4.52 55.23, -1.91 66.89" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(437.94149785627974 999.5159941173732) rotate(0 0.07628038784321234 33.80897586798295)"><path d="M11.32 39.53 C4.57 47.01, 1.01 60.15, -0.94 68.14 M10.11 38.41 C7.46 46.73, 4.6 55.49, -1.91 66.89" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(282.3133185250415 1075.849429175967) rotate(0 143.33331298828125 38)"><path d="M19 0 M19 0 C90.47 1.86, 159.08 0.35, 267.67 0 M19 0 C111.91 1.85, 204.44 1.67, 267.67 0 M267.67 0 C279.95 -0.75, 286.68 7.48, 286.67 19 M267.67 0 C279.14 1.89, 286.98 8.28, 286.67 19 M286.67 19 C287.89 31.85, 285.62 47.9, 286.67 57 M286.67 19 C286.3 33.54, 285.56 47.98, 286.67 57 M286.67 57 C285.89 71.45, 280.89 76.14, 267.67 76 M286.67 57 C284.92 68.16, 279.49 78.05, 267.67 76 M267.67 76 C184.68 73.86, 104.79 75.65, 19 76 M267.67 76 C177.92 76.61, 89.46 77.1, 19 76 M19 76 C5.2 77.58, 1.08 68.47, 0 57 M19 76 C4.6 76.6, 2.01 69.13, 0 57 M0 57 C-0.27 43.95, 0.61 29.91, 0 19 M0 57 C0.58 47.52, 0.78 37.38, 0 19 M0 19 C0.62 8.06, 4.75 -1.46, 19 0 M0 19 C2.19 6.95, 4.08 0.09, 19 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(397.64663151332275 1101.599429175967) rotate(0 28 12.5)"><text x="28" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">fork()</text></g><g mask="url(#mask-W6wEgkykYwlEjtvyhCSwL)" stroke-linecap="round"><g transform="translate(951.646692548479 585.1827421642482) rotate(0 -165.31971747182877 184.42643164848909)"><path d="M-0.06 0.45 C-1.42 53.41, 51.44 257.65, -7.05 318.9 C-65.53 380.15, -293.31 359.59, -350.97 367.93 M-1.54 -0.36 C-3.21 53.21, 50.08 258.26, -8.19 319.86 C-66.47 381.45, -293.68 361.38, -351.19 369.21" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(951.646692548479 585.1827421642482) rotate(0 -165.31971747182877 184.42643164848909)"><path d="M-322.54 357.53 C-336.11 364.26, -345.49 365.3, -349.66 367.48 M-323.77 358.14 C-328.08 360.94, -334.39 363.67, -351.08 368.27" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(951.646692548479 585.1827421642482) rotate(0 -165.31971747182877 184.42643164848909)"><path d="M-321.97 378.05 C-335.97 376.62, -345.57 369.5, -349.66 367.48 M-323.19 378.65 C-327.79 377.2, -334.21 375.67, -351.08 368.27" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask id="mask-W6wEgkykYwlEjtvyhCSwL"><rect x="0" y="0" fill="#fff" width="1403.646692548479" height="1053.1827421642483"></rect><rect x="810.979944501604" y="868.1827421642482" fill="#000" width="268" height="74" opacity="1"></rect></mask><g transform="translate(810.979944501604 868.1827421642482) rotate(0 -24.652969424953767 -98.57356835151091)"><text x="134" y="17.666666666666668" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">若设置了--detach,则fg=tru</text><text x="134" y="42.333333333333336" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">e;否则为false;exit_code!=</text><text x="134" y="67" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">NULL</text></g><g mask="url(#mask-nxYkdpV1VimPtYmzvX30G)" stroke-linecap="round"><g transform="translate(372.979944501604 1162.5161161876858) rotate(0 -104.06814577512444 49.719140996932936)"><path d="M-1.18 -0.58 C-35.65 15.75, -173.41 82.12, -207.73 99 M0.41 1.74 C-34.12 18.69, -173.91 83.85, -208.54 100.01" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(372.979944501604 1162.5161161876858) rotate(0 -104.06814577512444 49.719140996932936)"><path d="M-188.3 79.23 C-193.56 84.84, -195.46 86.59, -209.37 99.61 M-187.43 79.68 C-193.92 83.91, -198.12 90.52, -208.58 100.59" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(372.979944501604 1162.5161161876858) rotate(0 -104.06814577512444 49.719140996932936)"><path d="M-179.62 97.83 C-186.99 99.19, -190.84 96.77, -209.37 99.61 M-178.75 98.27 C-187.57 97.4, -194.17 98.88, -208.58 100.59" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask id="mask-nxYkdpV1VimPtYmzvX30G"><rect x="0" y="0" fill="#fff" width="680.979944501604" height="1362.5160551525296"></rect><rect x="200.479944501604" y="1200.0160856701077" fill="#000" width="137" height="25" opacity="1"></rect></mask><g transform="translate(200.479944501604 1200.0160856701073) rotate(0 68.43185422487556 12.219171514511174)"><text x="68.5" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">parent(isulad)</text></g><g mask="url(#mask-TstfDDe_FGH9-OyCRtLUv)" stroke-linecap="round"><g transform="translate(432.97988346644775 1149.1827421642483) rotate(0 105.64669347778892 53.79419904327011)"><path d="M0.87 1.06 C36.11 18.71, 176.3 88.4, 211.43 105.89 M-0.13 0.57 C35.02 18.36, 175.62 89.29, 210.88 107.02" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(432.97988346644775 1149.1827421642483) rotate(0 105.64669347778892 53.79419904327011)"><path d="M182.85 104.77 C188.67 105.98, 197.68 105.28, 210.97 108.97 M181.01 103.86 C193.09 103.67, 204.74 106.54, 210.37 107.64" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(432.97988346644775 1149.1827421642483) rotate(0 105.64669347778892 53.79419904327011)"><path d="M192.09 86.44 C195.15 93.18, 201.4 97.95, 210.97 108.97 M190.24 85.54 C198.92 92.54, 206.99 102.52, 210.37 107.64" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask id="mask-TstfDDe_FGH9-OyCRtLUv"><rect x="0" y="0" fill="#fff" width="743.6465704781665" height="1355.849429175967"></rect><rect x="490.3132269723071" y="1190.0160856701077" fill="#000" width="96" height="25" opacity="1"></rect></mask><g transform="translate(490.3132269723071 1190.0160856701073) rotate(0 48.31334997192954 12.96085553741085)"><text x="48" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">child(shim)</text></g><g stroke-linecap="round" transform="translate(59.646631513322745 1275.8494291759669) rotate(0 132.66665649414062 42)"><path d="M21 0 M21 0 C67.01 2.17, 115.54 1.99, 244.33 0 M21 0 C72.19 2.36, 124.99 1.25, 244.33 0 M244.33 0 C260.1 -0.05, 264.91 8.38, 265.33 21 M244.33 0 C260.43 1.57, 265.84 5.85, 265.33 21 M265.33 21 C266.1 32.66, 264.69 49.68, 265.33 63 M265.33 21 C264.8 34.7, 264.43 50.64, 265.33 63 M265.33 63 C267.06 76.92, 257.57 82.65, 244.33 84 M265.33 63 C263.81 77.85, 256.54 83.4, 244.33 84 M244.33 84 C186.62 86.96, 126.6 85.88, 21 84 M244.33 84 C164.01 84.73, 84.58 85.67, 21 84 M21 84 C5.58 82.38, -0.44 75.94, 0 63 M21 84 C5.14 85.7, -1.22 76.32, 0 63 M0 63 C0.67 53.36, -1.02 41.83, 0 21 M0 63 C0.03 51.55, -0.98 40.52, 0 21 M0 21 C-0.56 7.61, 7.58 0.05, 21 0 M0 21 C1.76 8.39, 7.91 1.8, 21 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(67.31328800746337 1280.8494291759669) rotate(0 125 37)"><text x="125" y="17.666666666666668" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">从读fifo中读取isulad-shimx</text><text x="125" y="42.333333333333336" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">ier写入的内容</text><text x="125" y="67" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">num = util_read_nointr</text></g><g stroke-linecap="round"><g transform="translate(170.3133185250415 1361.182742164248) rotate(0 -1.0833939987327312 25.606997427400188)"><path d="M0.5 0.62 C0.09 8.99, -1.33 41.81, -2.04 50.08 M-0.7 -0.09 C-1.24 8.43, -2.54 42.73, -2.66 51.31" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(170.3133185250415 1361.182742164248) rotate(0 -1.0833939987327312 25.606997427400188)"><path d="M-9.62 28.27 C-9.01 33.34, -6.08 41.52, -4.54 51.98 M-11.14 27.68 C-7.86 33.56, -6.59 41.75, -3.08 51.54" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(170.3133185250415 1361.182742164248) rotate(0 -1.0833939987327312 25.606997427400188)"><path d="M7.73 28.76 C2.59 33.74, -0.22 41.76, -4.54 51.98 M6.21 28.17 C4.42 33.95, 0.61 42, -3.08 51.54" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(67.64663151332275 1414.5160551525294) rotate(0 98.66668701171875 48)"><path d="M123.75 12.25 M123.75 12.25 C137.1 17.1, 148.23 25.18, 172.58 36.75 M123.75 12.25 C140.79 22.27, 158.24 29.29, 172.58 36.75 M172.58 36.75 C196.96 47.36, 199.02 50.32, 172.58 61.25 M172.58 36.75 C199.13 49.5, 195.22 47.52, 172.58 61.25 M172.58 61.25 C157.23 68.16, 142.57 75.51, 123.75 83.75 M172.58 61.25 C158.96 68.31, 142.79 74.04, 123.75 83.75 M123.75 83.75 C99.51 97.96, 97.7 97.32, 74.25 83.75 M123.75 83.75 C99.89 93.74, 100.92 96.85, 74.25 83.75 M74.25 83.75 C54.99 76.55, 36.22 67.36, 24.75 61.25 M74.25 83.75 C58.26 76.74, 43.49 70.01, 24.75 61.25 M24.75 61.25 C0.53 47.95, 0.9 47.62, 24.75 36.75 M24.75 61.25 C0.95 47.24, 0.35 49.8, 24.75 36.75 M24.75 36.75 C43.84 28.41, 64.06 20.38, 74.25 12.25 M24.75 36.75 C35.7 32.04, 48.7 25.44, 74.25 12.25 M74.25 12.25 C100.15 -0.53, 98.71 -0.18, 123.75 12.25 M74.25 12.25 C99.88 -2.18, 99.53 0.04, 123.75 12.25" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(114.8133185250415 1450.2660551525294) rotate(0 51.5 12.5)"><text x="51.5" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">if num > 0</text></g><g mask="url(#mask-L-6tCyqj6x0yvh9jKcA8u)" stroke-linecap="round"><g transform="translate(170.3133185250415 1511.8494291759669) rotate(0 0.405902329029459 41.08292857818299)"><path d="M0.49 -0.92 C0.66 13.07, 1.22 69.12, 1.52 83.08 M-0.7 1.21 C-0.73 14.9, 0.22 67.34, 0.67 81.18" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(170.3133185250415 1511.8494291759669) rotate(0 0.405902329029459 41.08292857818299)"><path d="M-11.8 53.55 C-5.99 64.39, -2.63 71.01, 1.1 81.73 M-9.84 52.91 C-6.45 65.42, -0.94 75.8, 1.24 80.92" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(170.3133185250415 1511.8494291759669) rotate(0 0.405902329029459 41.08292857818299)"><path d="M8.72 53.06 C7.44 64.15, 3.73 70.94, 1.1 81.73 M10.67 52.41 C5.9 65, 3.24 75.57, 1.24 80.92" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask id="mask-L-6tCyqj6x0yvh9jKcA8u"><rect x="0" y="0" fill="#fff" width="271.64663151332275" height="1694.5160856701075"></rect><rect x="159.97997501918212" y="1540.682757423037" fill="#000" width="22" height="25" opacity="1"></rect></mask><g transform="translate(159.97997501918212 1540.682757423037) rotate(0 10.739245834888834 12.249600331112674)"><text x="11" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">no</text></g><g stroke-linecap="round" transform="translate(51.646631513322745 1601.182742164248) rotate(0 136 54.5)"><path d="M27.25 0 M27.25 0 C79.28 1.16, 131.48 0.52, 244.75 0 M27.25 0 C83.9 0.01, 139.99 0.88, 244.75 0 M244.75 0 C263.39 -1.18, 270.96 10.66, 272 27.25 M244.75 0 C263.01 0.46, 273.94 7.69, 272 27.25 M272 27.25 C271.54 43.22, 270.62 59.29, 272 81.75 M272 27.25 C272.72 42.32, 272.27 56.88, 272 81.75 M272 81.75 C271.56 99.81, 262.68 110.01, 244.75 109 M272 81.75 C270.15 98.53, 264.69 110.37, 244.75 109 M244.75 109 C190.03 110.13, 137.93 111.06, 27.25 109 M244.75 109 C192.84 108.41, 140.4 108.09, 27.25 109 M27.25 109 C11.05 108.57, 1.42 101.08, 0 81.75 M27.25 109 C9.98 107.4, 1.6 97.86, 0 81.75 M0 81.75 C0.82 68.31, 2.12 53.6, 0 27.25 M0 81.75 C0.47 67.89, 0.77 52.88, 0 27.25 M0 27.25 C1.07 11.05, 11.06 -1.06, 27.25 0 M0 27.25 C1.83 10.02, 10.93 0.25, 27.25 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(57.646631513322745 1606.182742164248) rotate(0 130 49.5)"><text x="130" y="17.75" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">wait isulad-shim或者shim </text><text x="130" y="42.5" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">parent 进程</text><text x="130" y="67.25" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">util_waitpid_with_timeout</text><text x="130" y="92" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr"></text></g><g stroke-linecap="round"><g transform="translate(175.64663151332275 1706.5161161876856) rotate(0 -1.4345829975511606 31.714170566257167)"><path d="M-0.15 0.32 C-0.55 10.77, -1.79 52.07, -2.08 62.52 M-1.68 -0.55 C-2.22 10.1, -2.63 53.74, -2.72 63.98" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(175.64663151332275 1706.5161161876856) rotate(0 -1.4345829975511606 31.714170566257167)"><path d="M-12.17 36.66 C-9.7 46.34, -5.55 55.75, -3.92 64.91 M-12.87 36.25 C-8.89 43.49, -5.86 52.09, -3.37 64.68" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(175.64663151332275 1706.5161161876856) rotate(0 -1.4345829975511606 31.714170566257167)"><path d="M8.35 36.87 C4.06 46.45, 1.46 55.79, -3.92 64.91 M7.65 36.45 C5.59 43.5, 2.58 52.05, -3.37 64.68" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(44.979944501603995 1763.8494291759669) rotate(0 124 68.00003051757812)"><path d="M156.25 17.25 M156.25 17.25 C176.66 28.46, 194.82 38.33, 216.75 51.75 M156.25 17.25 C170.85 26.93, 185.95 34.34, 216.75 51.75 M216.75 51.75 C247.98 69.11, 248.62 68.06, 216.75 86.25 M216.75 51.75 C248.95 70.84, 249.84 70.12, 216.75 86.25 M216.75 86.25 C204.63 94.31, 190.79 100.88, 156.25 118.75 M216.75 86.25 C195.55 96.63, 176.24 108.51, 156.25 118.75 M156.25 118.75 C124.72 136.61, 126.2 136.11, 93.75 118.75 M156.25 118.75 C122.82 137.58, 123.14 135.28, 93.75 118.75 M93.75 118.75 C68.15 106.33, 44.25 94.91, 31.25 86.25 M93.75 118.75 C74.87 109.76, 56.79 99.57, 31.25 86.25 M31.25 86.25 C1.2 70.22, 0.76 70.58, 31.25 51.75 M31.25 86.25 C-0.58 66.95, -0.7 70.99, 31.25 51.75 M31.25 51.75 C56.43 36.93, 78 26.14, 93.75 17.25 M31.25 51.75 C54.74 38.93, 79.72 24.19, 93.75 17.25 M93.75 17.25 C123.05 -1.54, 123.31 0.02, 156.25 17.25 M93.75 17.25 C126.25 0.01, 124.37 0.65, 156.25 17.25" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(114.479944501604 1819.599459693545) rotate(0 54.5 12.5)"><text x="54.5" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">if ret != 0</text></g><g stroke-linecap="round"><g transform="translate(163.64663151332275 1895.8494291759669) rotate(0 0.11652293213751364 32.57699033206325)"><path d="M0.71 1.15 C0.4 11.65, -0.9 52.4, -0.98 62.83 M-0.38 0.7 C-0.39 11.45, 1.22 54.25, 1.21 64.45" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(163.64663151332275 1895.8494291759669) rotate(0 0.11652293213751364 32.57699033206325)"><path d="M-7.73 34.85 C-5.39 47.19, 0.13 57.1, 1.33 62.9 M-10.62 35.72 C-6.59 46.66, -1.29 56.74, 0.94 64.73" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(163.64663151332275 1895.8494291759669) rotate(0 0.11652293213751364 32.57699033206325)"><path d="M12.79 34.41 C7.51 47.04, 5.41 57.11, 1.33 62.9 M9.9 35.28 C6.53 46.23, 4.44 56.46, 0.94 64.73" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(42.313257489885245 1959.8493681408106) rotate(0 147.5 33.5)"><path d="M16.75 0 M16.75 0 C95.08 0.44, 173.23 -1.11, 278.25 0 M16.75 0 C118.26 -1.19, 219.76 -1.17, 278.25 0 M278.25 0 C290.9 -1, 293.27 3.73, 295 16.75 M278.25 0 C290.22 -0.09, 296.48 4.94, 295 16.75 M295 16.75 C293.82 23.02, 295.82 34.77, 295 50.25 M295 16.75 C294.64 28.9, 293.93 42.93, 295 50.25 M295 50.25 C296.66 60.39, 288.13 67.17, 278.25 67 M295 50.25 C294.16 59.94, 289.27 67.66, 278.25 67 M278.25 67 C211.6 67.24, 144.15 65.2, 16.75 67 M278.25 67 C197.58 68.3, 117.6 67.87, 16.75 67 M16.75 67 C4.33 66.6, -1.96 62.28, 0 50.25 M16.75 67 C3.8 67.85, 0.54 60.37, 0 50.25 M0 50.25 C-0.3 37.53, -1.13 29.46, 0 16.75 M0 50.25 C0.64 40.17, 0.24 31.81, 0 16.75 M0 16.75 C-1.59 5.17, 7.31 1.22, 16.75 0 M0 16.75 C0.99 6.64, 3.47 1.77, 16.75 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(61.813257489885245 1980.8493681408106) rotate(0 128 12.5)"><text x="128" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">show_shim_runtime_errlog</text></g><g stroke-linecap="round"><g transform="translate(148.979944501604 2022.5161161876856) rotate(0 0.47750494991180403 36.14339694603348)"><path d="M-0.49 1.14 C-0.51 13.11, 0.83 60.05, 1 71.6 M1.45 0.69 C1.15 12.35, -0.38 58.14, -0.13 69.97" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(148.979944501604 2022.5161161876856) rotate(0 0.47750494991180403 36.14339694603348)"><path d="M-8.23 41.12 C-4.7 51.45, -2.67 62.69, -0.89 69.48 M-9.97 40.82 C-8.1 48.9, -6.18 54.01, 0.26 70.09" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(148.979944501604 2022.5161161876856) rotate(0 0.47750494991180403 36.14339694603348)"><path d="M12.29 41.32 C7.78 51.48, 1.77 62.65, -0.89 69.48 M10.55 41.01 C7.63 48.97, 4.77 54.03, 0.26 70.09" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(42.313318525041495 2093.1827421642483) rotate(0 109.33334350585938 52)"><path d="M137.5 13.25 M137.5 13.25 C151.59 21.42, 166.09 27.18, 191.17 39.75 M137.5 13.25 C149.11 19.84, 161.68 25.8, 191.17 39.75 M191.17 39.75 C218.52 51.31, 216.83 54.27, 191.17 66.25 M191.17 39.75 C218.67 54, 217.34 52, 191.17 66.25 M191.17 66.25 C175.06 72.64, 161.52 82.68, 137.5 90.75 M191.17 66.25 C177.5 74.44, 161.97 79.55, 137.5 90.75 M137.5 90.75 C109.76 104.56, 109.53 102.75, 82.5 90.75 M137.5 90.75 C110.3 102.4, 110.06 105.33, 82.5 90.75 M82.5 90.75 C65.66 84.95, 50.11 74.07, 27.5 66.25 M82.5 90.75 C68.97 85.22, 56.33 80.48, 27.5 66.25 M27.5 66.25 C1.06 53.93, -0.38 53.03, 27.5 39.75 M27.5 66.25 C1.19 51.37, 1.28 53.62, 27.5 39.75 M27.5 39.75 C41.96 32.63, 57.88 22.93, 82.5 13.25 M27.5 39.75 C40.69 32.84, 55.35 26.45, 82.5 13.25 M82.5 13.25 C109.47 0.97, 109.75 1.5, 137.5 13.25 M82.5 13.25 C108.75 1.53, 111.8 1.48, 137.5 13.25" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(73.14666203090087 2132.9327421642483) rotate(0 78.5 12.5)"><text x="78.5" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">if timeout <= 0</text></g><g stroke-linecap="round"><g transform="translate(147.24918304668677 2201.6278484197146) rotate(0 0.3740707536219361 27.90688780774417)"><path d="M-0.52 0.09 C-0.27 9.58, 0.36 46.39, 0.55 55.77 M1.41 -0.91 C1.53 8.79, -0.39 47, -0.67 56.72" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(147.24918304668677 2201.6278484197146) rotate(0 0.3740707536219361 27.90688780774417)"><path d="M-9.17 28.74 C-9.49 35.1, -6.49 42.06, -1.2 57.69 M-8.66 30.4 C-5.99 37.24, -4.68 45.04, 0.3 57.09" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(147.24918304668677 2201.6278484197146) rotate(0 0.3740707536219361 27.90688780774417)"><path d="M10.27 29.52 C5.57 35.62, 4.18 42.4, -1.2 57.69 M10.79 31.18 C7.86 37.69, 3.58 45.27, 0.3 57.09" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(44.979944501603995 2262.5160856701077) rotate(0 115.33334350585938 27.333328247070312)"><path d="M13.67 0 M13.67 0 C64.76 -1.81, 112.59 -1.78, 217 0 M13.67 0 C93.56 0.44, 175.22 -0.34, 217 0 M217 0 C226.34 0.98, 229.7 6.35, 230.67 13.67 M217 0 C224.78 -0.45, 232.83 6.12, 230.67 13.67 M230.67 13.67 C230.69 25.99, 229.48 36.6, 230.67 41 M230.67 13.67 C230.68 18.86, 230.71 25.83, 230.67 41 M230.67 41 C230.93 49.77, 226.81 53.06, 217 54.67 M230.67 41 C229.88 50.06, 224.79 55.15, 217 54.67 M217 54.67 C149.27 54.13, 83.57 56.29, 13.67 54.67 M217 54.67 C157.64 53.98, 100.07 53.1, 13.67 54.67 M13.67 54.67 C5.97 55.75, -0.77 51.3, 0 41 M13.67 54.67 C3.98 52.46, -1.67 51.16, 0 41 M0 41 C0.96 30.85, 0.27 25.05, 0 13.67 M0 41 C0.34 33.86, 0.16 27.12, 0 13.67 M0 13.67 C-1.62 4.48, 4.35 0.8, 13.67 0 M0 13.67 C0.8 6.72, 2.35 -0.8, 13.67 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(76.31328800746337 2277.349413917178) rotate(0 84 12.5)"><text x="84" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">kill(pid, SIGKILL)</text></g><g mask="url(#mask-Hve2y6rUdI1h2Iiv2-INb)" stroke-linecap="round"><g transform="translate(255.64663151332275 1463.8494291759669) rotate(0 53.271868080898116 187.59706538110038)"><path d="M1 -0.08 C18.33 30.23, 99.16 119.47, 104.14 182.21 C109.12 244.95, 43.07 344.05, 30.87 376.36 M0.06 -1.17 C17.73 29.27, 101.2 120.94, 106.22 183.5 C111.25 246.07, 42.7 341.92, 30.21 374.21" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(255.64663151332275 1463.8494291759669) rotate(0 53.271868080898116 187.59706538110038)"><path d="M34.08 342.49 C32.29 354.12, 32.41 363.45, 30.38 374.42 M35.13 343.49 C33.52 353.05, 31.83 362.31, 30.91 375.11" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(255.64663151332275 1463.8494291759669) rotate(0 53.271868080898116 187.59706538110038)"><path d="M52.25 352.02 C44.77 360.83, 39.23 367.19, 30.38 374.42 M53.3 353.03 C46.44 359.86, 39.47 366.36, 30.91 375.11" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask id="mask-Hve2y6rUdI1h2Iiv2-INb"><rect x="0" y="0" fill="#fff" width="460.979944501604" height="1939.8494291759669"></rect><rect x="344.479944501604" y="1634.0160856701073" fill="#000" width="33" height="25" opacity="1"></rect></mask><g transform="translate(344.479944501604 1634.0160856701073) rotate(0 -35.561444907383134 17.430408886959754)"><text x="16.5" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">yes</text></g><g stroke-linecap="round" transform="translate(519.646692548479 1269.182742164248) rotate(0 138.6666259765625 38.66668701171875)"><path d="M19.33 0 M19.33 0 C103.89 -0.51, 191.96 -2.9, 258 0 M19.33 0 C102.51 -2.6, 185.36 -2.28, 258 0 M258 0 C272.63 1.7, 278.57 6.15, 277.33 19.33 M258 0 C269.2 0.2, 275.16 5.49, 277.33 19.33 M277.33 19.33 C277.43 30.63, 275.33 37.03, 277.33 58 M277.33 19.33 C277.61 28.45, 276.96 38.07, 277.33 58 M277.33 58 C277.73 70.97, 270.37 77.96, 258 77.33 M277.33 58 C278.82 69.56, 269.29 77.53, 258 77.33 M258 77.33 C161.72 77.38, 68.87 77.13, 19.33 77.33 M258 77.33 C206.47 79.08, 155.89 79.55, 19.33 77.33 M19.33 77.33 C5.6 77.91, 1.18 70.8, 0 58 M19.33 77.33 C5.52 76.18, -0.77 71.57, 0 58 M0 58 C1.86 50.14, -1.26 41.29, 0 19.33 M0 58 C-0.33 43.12, 1.01 29.77, 0 19.33 M0 19.33 C0.92 8.22, 7.87 -1.66, 19.33 0 M0 19.33 C1.41 7.53, 5.12 -0.5, 19.33 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(594.8133185250415 1295.3494291759669) rotate(0 63.5 12.5)"><text x="63.5" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">chdir(workdir)</text></g><g stroke-linecap="round"><g transform="translate(658.3133185250415 1350.5161161876856) rotate(0 -0.06027126990255738 25.41792125812401)"><path d="M0.14 -0.49 C-0.05 8.08, -1.08 43.81, -1.04 52.63 M-1.25 -1.8 C-1.09 6.94, 0.95 42.15, 1.13 50.84" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(658.3133185250415 1350.5161161876856) rotate(0 -0.06027126990255738 25.41792125812401)"><path d="M-9.6 25.07 C-4.44 34.87, -1.68 39.88, 0.62 52.71 M-8.34 26.37 C-6.5 32.36, -4.82 36.26, 0.63 50.51" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(658.3133185250415 1350.5161161876856) rotate(0 -0.06027126990255738 25.41792125812401)"><path d="M8.18 24.33 C7.85 34.37, 5.12 39.62, 0.62 52.71 M9.43 25.63 C7.65 31.63, 5.72 35.68, 0.63 50.51" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(559.6466315133227 1414.5161161876856) rotate(0 98.66668701171875 48)"><path d="M123.75 12.25 M123.75 12.25 C137.96 17.11, 151.33 26.65, 172.58 36.75 M123.75 12.25 C136.68 20.05, 151.19 26.3, 172.58 36.75 M172.58 36.75 C199.11 50.42, 195.68 50.23, 172.58 61.25 M172.58 36.75 C198.42 47.67, 196.83 47.76, 172.58 61.25 M172.58 61.25 C156.93 68, 144.34 75.61, 123.75 83.75 M172.58 61.25 C159.67 66.55, 147.95 73.86, 123.75 83.75 M123.75 83.75 C99.75 96.87, 98.65 96.47, 74.25 83.75 M123.75 83.75 C97.11 96.43, 96.76 95.85, 74.25 83.75 M74.25 83.75 C54.45 73.65, 38.49 65.08, 24.75 61.25 M74.25 83.75 C60.07 78.38, 45.64 70.29, 24.75 61.25 M24.75 61.25 C-0.14 48.89, -0.72 47.32, 24.75 36.75 M24.75 61.25 C1.25 46.7, -0.46 51.23, 24.75 36.75 M24.75 36.75 C34.77 29.95, 47.48 23.51, 74.25 12.25 M24.75 36.75 C39.49 30.57, 53.98 21.96, 74.25 12.25 M74.25 12.25 C98.59 -1.21, 98.23 -1.35, 123.75 12.25 M74.25 12.25 C98.96 0.99, 100.38 -0.9, 123.75 12.25" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(635.3133185250415 1450.2661161876856) rotate(0 23 12.5)"><text x="23" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">if fg</text></g><g mask="url(#mask-hhhgW-oNOEe_xpnO91GNy)" stroke-linecap="round"><g transform="translate(654.3133185250415 1505.1828031994044) rotate(0 -0.7338871483990488 51.130687383934855)"><path d="M1.16 -0.84 C1.12 16.38, -0.94 85.95, -1.44 103.11 M0.31 1.33 C0.18 18.21, -2.68 84.25, -2.63 101.21" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(654.3133185250415 1505.1828031994044) rotate(0 -0.7338871483990488 51.130687383934855)"><path d="M-13.65 72.01 C-9.65 76.95, -9.44 83.97, -4.44 101.13 M-11.28 72.55 C-10.66 78.64, -8.82 84.49, -3.3 101.2" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(654.3133185250415 1505.1828031994044) rotate(0 -0.7338871483990488 51.130687383934855)"><path d="M6.86 72.48 C6.76 77.52, 2.86 84.45, -4.44 101.13 M9.24 73.01 C5.5 78.98, 2.98 84.74, -3.3 101.2" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask id="mask-hhhgW-oNOEe_xpnO91GNy"><rect x="0" y="0" fill="#fff" width="755.2777361641914" height="1707.8493681408106"></rect><rect x="642.8311097054666" y="1544.0160856701073" fill="#000" width="22" height="25" opacity="1"></rect></mask><g transform="translate(642.8311097054666 1544.0160856701073) rotate(0 10.748321671175859 12.29740491323173)"><text x="11" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">no</text></g><g stroke-linecap="round" transform="translate(510.3133185250415 1613.182742164248) rotate(0 157.5 32)"><path d="M16 0 M16 0 C79.81 -1.08, 141.31 -1.3, 299 0 M16 0 C84.47 -1.96, 153.92 -0.59, 299 0 M299 0 C311.58 0.81, 315.03 5.81, 315 16 M299 0 C307.74 0.44, 317.01 7.36, 315 16 M315 16 C316.85 28.31, 314.41 37.06, 315 48 M315 16 C314.66 28.54, 315.39 38.84, 315 48 M315 48 C316.93 57.79, 311.37 62.83, 299 64 M315 48 C316.15 58.14, 311.69 64.36, 299 64 M299 64 C226.37 61.67, 156.43 62.7, 16 64 M299 64 C216.06 63.05, 133.77 63.69, 16 64 M16 64 C5.29 62.79, -1.49 58.58, 0 48 M16 64 C3.8 62.07, -0.76 60.96, 0 48 M0 48 C-1.08 40.49, -1.69 34.7, 0 16 M0 48 C1.14 37.22, -0.13 27.65, 0 16 M0 16 C1.36 3.65, 5.98 -0.52, 16 0 M0 16 C-0.35 6.99, 5.66 -2.15, 16 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(523.3133185250415 1632.682742164248) rotate(0 144.5 12.5)"><text x="144.5" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">unsetenv("NOTIFY_SOCKET")</text></g><g stroke-linecap="round"><g transform="translate(655.646692548479 1674.5161161876856) rotate(0 -1.9969944021012793 42.19494547440672)"><path d="M0.01 1.13 C-0.43 15.09, -2.77 70.29, -3.38 83.72 M-1.44 0.67 C-1.92 14.32, -3.54 68.5, -4.01 82.15" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(655.646692548479 1674.5161161876856) rotate(0 -1.9969944021012793 42.19494547440672)"><path d="M-11.49 54.67 C-9.39 60.99, -10.5 66.47, -5.21 81.36 M-14.35 54.21 C-10.42 61.11, -7.99 68.06, -4.85 82.37" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(655.646692548479 1674.5161161876856) rotate(0 -1.9969944021012793 42.19494547440672)"><path d="M9.03 55.33 C6.23 61.49, 0.23 66.82, -5.21 81.36 M6.16 54.86 C4.57 61.75, 1.48 68.52, -4.85 82.37" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(502.97988346644775 1764.5161161876856) rotate(0 143.33331298828125 38)"><path d="M19 0 M19 0 C71.3 -1.15, 121.82 -2.01, 267.67 0 M19 0 C107.85 -1.8, 197.42 -0.38, 267.67 0 M267.67 0 C281.11 0.1, 287.19 7.52, 286.67 19 M267.67 0 C279.86 1.37, 286.44 6.68, 286.67 19 M286.67 19 C285.71 26.48, 288.42 37.35, 286.67 57 M286.67 19 C286.57 27.66, 287.07 36.67, 286.67 57 M286.67 57 C288.21 69.75, 280.84 76.07, 267.67 76 M286.67 57 C288.4 70.69, 282.23 77.29, 267.67 76 M267.67 76 C184.28 77.94, 98.91 78.39, 19 76 M267.67 76 C217.6 76.15, 166.89 77.21, 19 76 M19 76 C5.66 76.6, 1.46 70.3, 0 57 M19 76 C8.26 77.49, 1.64 69.08, 0 57 M0 57 C-1.28 48.23, 0.22 43.27, 0 19 M0 57 C-0.94 43.36, 0.01 29.92, 0 19 M0 19 C-1.42 5.52, 5.34 -0.37, 19 0 M0 19 C0.69 5.97, 6.59 0.59, 19 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(618.313196454729 1790.2661161876856) rotate(0 28 12.5)"><text x="28" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">fork()</text></g><g mask="url(#mask-LhIeWW4KfOyKN4Sq9NUNY)" stroke-linecap="round"><g transform="translate(623.6465704781665 1843.849490211123) rotate(0 -79.94208937451242 50.103210557922694)"><path d="M0.72 0.05 C-25.91 16.56, -133 83.34, -159.78 100.22 M-0.35 -0.97 C-27.13 16.16, -133.89 83.9, -160.61 101.18" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(623.6465704781665 1843.849490211123) rotate(0 -79.94208937451242 50.103210557922694)"><path d="M-142.31 77.71 C-149.53 84.78, -153.15 92.5, -162.03 100.37 M-142.55 77.45 C-146.74 84.29, -152.51 89.42, -160.28 100.26" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(623.6465704781665 1843.849490211123) rotate(0 -79.94208937451242 50.103210557922694)"><path d="M-131.25 95 C-141.98 96.9, -148.95 99.39, -162.03 100.37 M-131.49 94.74 C-138.42 97.28, -146.96 98.09, -160.28 100.26" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask id="mask-LhIeWW4KfOyKN4Sq9NUNY"><rect x="0" y="0" fill="#fff" width="883.6466315133227" height="2045.1828031994044"></rect><rect x="447.64653996058837" y="1882.0161467052635" fill="#000" width="192" height="25" opacity="1"></rect></mask><g transform="translate(447.64653996058837 1882.0161467052635) rotate(0 96.0579411430657 11.936554063782069)"><text x="96" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">parent(shim parent)</text></g><g mask="url(#mask-TL3n9G1A4i1AY3rZ3MZNG)" stroke-linecap="round"><g transform="translate(662.3133185250415 1843.8494291759669) rotate(0 72.83110526815989 52.094522636821466)"><path d="M-1.1 0.12 C22.97 17.56, 119.92 87.84, 144.49 105.06 M0.53 -0.87 C24.96 16.23, 122.51 85.94, 146.76 103.49" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(662.3133185250415 1843.8494291759669) rotate(0 72.83110526815989 52.094522636821466)"><path d="M118.07 93.98 C128.6 99.78, 134.82 101.05, 146.52 102.01 M117.54 95.25 C128.06 99.24, 138.66 100.3, 146.56 102.97" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(662.3133185250415 1843.8494291759669) rotate(0 72.83110526815989 52.094522636821466)"><path d="M130.04 77.31 C136.92 88.15, 139.44 94.57, 146.52 102.01 M129.51 78.59 C135.97 88.37, 142.35 95.29, 146.56 102.97" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask id="mask-TL3n9G1A4i1AY3rZ3MZNG"><rect x="0" y="0" fill="#fff" width="907.646692548479" height="2047.8494291759669"></rect><rect x="654.9800055367602" y="1883.3494291759666" fill="#000" width="160" height="25" opacity="1"></rect></mask><g transform="translate(654.9800055367602 1883.3494291759666) rotate(0 80.16441825644114 12.594522636821466)"><text x="80" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">child(isulad-shim)</text></g><g stroke-linecap="round" transform="translate(388.98000553676025 1951.8494291759669) rotate(0 119.5 29.5)"><path d="M14.75 0 M14.75 0 C92.93 -1.49, 169.79 -1.92, 224.25 0 M14.75 0 C69.99 1.13, 123.97 0.61, 224.25 0 M224.25 0 C234.55 1.01, 240.53 5.35, 239 14.75 M224.25 0 C236.35 2.11, 239.2 6.23, 239 14.75 M239 14.75 C238 26.2, 237.31 37.73, 239 44.25 M239 14.75 C239.07 26.84, 239.36 37.81, 239 44.25 M239 44.25 C238.41 55.28, 235.19 58.55, 224.25 59 M239 44.25 C238.52 55.55, 234.42 57.99, 224.25 59 M224.25 59 C142.52 58.98, 58.18 59.92, 14.75 59 M224.25 59 C151.21 58.65, 75.98 57.69, 14.75 59 M14.75 59 C3.68 58.26, 0.32 54.78, 0 44.25 M14.75 59 C4.34 59.89, -1.43 51.86, 0 44.25 M0 44.25 C-1.63 33.39, 0.96 21.43, 0 14.75 M0 44.25 C-0.4 33.92, -0.2 23.59, 0 14.75 M0 14.75 C-1.14 6.61, 6.81 -0.91, 14.75 0 M0 14.75 C-2.14 7.12, 3.78 1.89, 14.75 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(393.14666203090087 1956.8494291759669) rotate(0 114.5 24.5)"><text x="114.5" y="17.5" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">将shim的pid写入文件中:</text><text x="114.5" y="42" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">file_write_int(fpid, pid)</text></g><g stroke-linecap="round"><g transform="translate(482.3133185250415 2010.5161161876856) rotate(0 -1.452218912290732 35.979882910773085)"><path d="M-0.9 0.59 C-1.42 12.48, -2.79 58.79, -3.11 70.6 M0.83 -0.14 C0.2 12, -3.07 60.49, -3.74 72.1" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(482.3133185250415 2010.5161161876856) rotate(0 -1.452218912290732 35.979882910773085)"><path d="M-11.21 44.35 C-9.93 53.52, -5.98 59.31, -2.24 73.63 M-12.01 43.17 C-9.29 52.67, -6.14 64.05, -4.6 71.22" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(482.3133185250415 2010.5161161876856) rotate(0 -1.452218912290732 35.979882910773085)"><path d="M9.27 45.64 C4.38 54.27, 2.15 59.67, -2.24 73.63 M8.47 44.46 C4.03 53.5, 0.03 64.43, -4.6 71.22" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(351.64663151332275 2082.516116187686) rotate(0 128.66665649414062 34.666656494140625)"><path d="M17.33 0 M17.33 0 C64.7 -0.09, 111.53 -1.92, 240 0 M17.33 0 C97.58 -1.82, 177.18 -0.3, 240 0 M240 0 C252.86 -0.25, 258.18 4.2, 257.33 17.33 M240 0 C250.42 0.42, 258.9 5.31, 257.33 17.33 M257.33 17.33 C256.57 26.55, 259.13 33.22, 257.33 52 M257.33 17.33 C257.47 26.25, 258.33 33.76, 257.33 52 M257.33 52 C256.38 61.97, 253.45 69.89, 240 69.33 M257.33 52 C257.8 62.21, 249.87 71.32, 240 69.33 M240 69.33 C157.76 69.18, 75.75 70.71, 17.33 69.33 M240 69.33 C153.68 69.98, 66.92 70.66, 17.33 69.33 M17.33 69.33 C7.4 70.39, -0.71 63.13, 0 52 M17.33 69.33 C5.26 71.05, 0.77 64.51, 0 52 M0 52 C-1.2 41.47, 1.5 30.35, 0 17.33 M0 52 C1.2 43.75, -0.17 36.67, 0 17.33 M0 17.33 C0.71 4.95, 5.71 -0.13, 17.33 0 M0 17.33 C-0.17 7.29, 4.64 0.14, 17.33 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(362.31328800746337 2104.6827726818265) rotate(0 118 12.5)"><text x="118" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">_exit(EXIT_SUCCESS);</text></g><g stroke-linecap="round" transform="translate(735.646692548479 1950.5160856701075) rotate(0 96.6666259765625 34.00001525878906)"><path d="M17 0 M17 0 C62.05 -0.6, 106.56 0.54, 176.33 0 M17 0 C56.2 -2.01, 95.64 -1.76, 176.33 0 M176.33 0 C189.46 -0.28, 192.39 7.43, 193.33 17 M176.33 0 C188.48 -0.95, 193.25 5.51, 193.33 17 M193.33 17 C193.68 25.7, 193.17 37.59, 193.33 51 M193.33 17 C193.84 25.46, 192.1 35.63, 193.33 51 M193.33 51 C194.15 63.66, 187.77 66.7, 176.33 68 M193.33 51 C195.01 61, 188.37 66.77, 176.33 68 M176.33 68 C118.92 70.78, 64.89 69.53, 17 68 M176.33 68 C113.03 69.34, 51.35 68.19, 17 68 M17 68 C5.39 68.92, -0.3 61.44, 0 51 M17 68 C7.21 66.64, 1.03 61.83, 0 51 M0 51 C1.73 45.76, -0.02 37.52, 0 17 M0 51 C0.49 44.38, -0.58 36.71, 0 17 M0 17 C-1.99 3.77, 4.88 -0.18, 17 0 M0 17 C-1.23 4.32, 7.81 0.05, 17 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(793.8133185250415 1972.0161009288965) rotate(0 38.5 12.5)"><text x="38.5" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">setsid()</text></g><g stroke-linecap="round"><g transform="translate(822.131742264723 2023.8720483910754) rotate(0 -2.012526984208364 29.402566998308316)"><path d="M0.91 0.15 C0.24 9.74, -3.56 48.72, -4.58 58.46 M-0.07 -0.81 C-0.75 8.84, -3.99 49.55, -4.93 59.62" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(822.131742264723 2023.8720483910754) rotate(0 -2.012526984208364 29.402566998308316)"><path d="M-12.43 32.8 C-11.72 38.91, -9.5 46.81, -3.62 58.74 M-13.13 31.59 C-10.93 39.6, -8.33 46.47, -5.53 58.67" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(822.131742264723 2023.8720483910754) rotate(0 -2.012526984208364 29.402566998308316)"><path d="M7.83 34.54 C3.41 40.36, 0.49 47.82, -3.62 58.74 M7.13 33.32 C3.53 40.8, 0.32 47.18, -5.53 58.67" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(663.646692548479 2086.1827421642483) rotate(0 139 29.5)"><path d="M14.75 0 M14.75 0 C100.57 1.32, 184.35 -0.44, 263.25 0 M14.75 0 C93.21 1.86, 171.89 1.45, 263.25 0 M263.25 0 C274.95 0.04, 276.1 6.43, 278 14.75 M263.25 0 C271.43 -2.3, 278.49 3.22, 278 14.75 M278 14.75 C276.69 20.03, 279.47 29.8, 278 44.25 M278 14.75 C277.76 22.64, 278.38 33.17, 278 44.25 M278 44.25 C276.98 52.39, 271.62 60.81, 263.25 59 M278 44.25 C276.06 56.12, 272.31 58.1, 263.25 59 M263.25 59 C212.06 56.31, 163.11 57.84, 14.75 59 M263.25 59 C174.57 58.17, 86.97 59.46, 14.75 59 M14.75 59 C5.78 57.18, 0.84 52.97, 0 44.25 M14.75 59 C5.75 57.94, -0.83 55.11, 0 44.25 M0 44.25 C-1.64 36.63, 1.63 30.85, 0 14.75 M0 44.25 C-0.75 36.84, 0.94 31.08, 0 14.75 M0 14.75 C-1.85 6.4, 5.69 0.53, 14.75 0 M0 14.75 C2.3 6.5, 2.95 -1.96, 14.75 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(675.646692548479 2091.1827421642483) rotate(0 127 24.5)"><text x="127" y="17.5" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">将从父进程继承的fd关闭util</text><text x="127" y="42" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">_check_inherited</text></g><g stroke-linecap="round"><g transform="translate(806.5232733729051 2151.849429175967) rotate(0 -0.8291911096233662 30.383917033821263)"><path d="M-0.31 -0.93 C-1.01 9.03, -2.35 50.06, -3.03 60.56 M1.72 1.2 C0.87 11.22, -2.22 51.87, -3.38 61.7" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(806.5232733729051 2151.849429175967) rotate(0 -0.8291911096233662 30.383917033821263)"><path d="M-12.51 34.56 C-9.04 43.84, -3.49 55.42, -2.22 62.76 M-10.09 33.15 C-7.43 43.01, -5.09 52.75, -3.42 62.5" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(806.5232733729051 2151.849429175967) rotate(0 -0.8291911096233662 30.383917033821263)"><path d="M7.92 36.48 C3.28 44.94, 0.73 55.77, -2.22 62.76 M10.34 35.06 C5.76 44.33, 0.85 53.4, -3.42 62.5" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(650.3133185250415 2217.1828031994046) rotate(0 134.00006103515625 64.66665649414062)"><path d="M32 0 M32 0 C109.65 -0.67, 185.85 0.05, 236 0 M32 0 C93.79 -0.12, 157.3 0.83, 236 0 M236 0 C255.78 -0.39, 268.04 10.71, 268 32 M236 0 C258.99 -2.26, 266.23 9.86, 268 32 M268 32 C268.61 51.29, 269.18 69.78, 268 97.33 M268 32 C267.32 49.74, 268.43 69.32, 268 97.33 M268 97.33 C269.83 118.89, 258.76 129.15, 236 129.33 M268 97.33 C268.25 119.5, 257.64 129.39, 236 129.33 M236 129.33 C181.34 128.93, 126.32 127.73, 32 129.33 M236 129.33 C183.33 128.06, 131.26 128.8, 32 129.33 M32 129.33 C10.7 127.92, 1.81 120.42, 0 97.33 M32 129.33 C11.21 128.05, 1.68 116.83, 0 97.33 M0 97.33 C0.22 78.71, 0.81 64.84, 0 32 M0 97.33 C-0.16 72.82, -1.3 47.3, 0 32 M0 32 C-1.08 12, 11.88 0.72, 32 0 M0 32 C-2.13 11.7, 12.45 -0.51, 32 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(662.8133795601977 2257.349459693545) rotate(0 121.5 24.5)"><text x="121.5" y="17.5" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">execvp(SHIM_BINARY, </text><text x="121.5" y="42" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">(char * const *)params);</text></g><g stroke-linecap="round" transform="translate(1275.2856441679473 376.1824878510967) rotate(0 135.0000000000001 42)"><path d="M21 0 M21 0 C67.6 1.38, 114.72 2.3, 249 0 M21 0 C68.92 0.25, 117.54 0.69, 249 0 M249 0 C263.43 1.78, 271.11 8.49, 270 21 M249 0 C263.88 0.49, 269.15 8.94, 270 21 M270 21 C271.7 35.04, 269.58 47.11, 270 63 M270 21 C270.43 31.44, 270.79 42.42, 270 63 M270 63 C270.04 75.41, 264.85 84.33, 249 84 M270 63 C271.77 77.31, 261.02 86.14, 249 84 M249 84 C166.84 84.59, 85.2 85.1, 21 84 M249 84 C159.42 84.05, 68.86 85.13, 21 84 M21 84 C7.88 83.85, 1.22 78.6, 0 63 M21 84 C7.59 83.16, -0.93 74.78, 0 63 M0 63 C0.19 53.35, 0.46 40.45, 0 21 M0 63 C-0.68 48.85, -0.03 34.12, 0 21 M0 21 C-1.66 8.42, 6.24 -0.91, 21 0 M0 21 C1.93 6.25, 6.17 0.07, 21 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1287.2856441679473 381.1824878510967) rotate(0 123 37)"><text x="123" y="17.666666666666668" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">设置启动isulad-shim超时的</text><text x="123" y="42.333333333333336" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">定时器set_timeout_exit(</text><text x="123" y="67" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">DEFAULT_TIMEOUT);</text></g><g stroke-linecap="round" transform="translate(1278.0634304228302 522.5714064098531) rotate(0 120.83335876464855 33.33332061767578)"><path d="M16.67 0 M16.67 0 C98.52 -0.39, 181.93 -1, 225 0 M16.67 0 C95.25 -0.07, 171.93 -1.26, 225 0 M225 0 C234.63 0.95, 241.95 4.52, 241.67 16.67 M225 0 C234.67 1.37, 242.32 4.87, 241.67 16.67 M241.67 16.67 C243.51 28.57, 240.57 41.92, 241.67 50 M241.67 16.67 C241.4 25.04, 242.81 30.62, 241.67 50 M241.67 50 C239.8 59.74, 234.12 66.09, 225 66.67 M241.67 50 C243 61.06, 233.89 65.31, 225 66.67 M225 66.67 C144.7 68.09, 66.04 69.36, 16.67 66.67 M225 66.67 C142.54 66.78, 61.72 67.01, 16.67 66.67 M16.67 66.67 C5.1 66.95, 0.84 61.9, 0 50 M16.67 66.67 C7.29 68.1, -1.21 62.28, 0 50 M0 50 C-0.97 41.49, 1.34 27.88, 0 16.67 M0 50 C0.55 39.8, 0.8 30.62, 0 16.67 M0 16.67 C0.57 7.36, 7.41 -0.91, 16.67 0 M0 16.67 C-0.2 6.21, 5.38 -1.99, 16.67 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1317.8967891874786 543.4047270275289) rotate(0 81 12.5)"><text x="81" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">set_subreaper()</text></g><g stroke-linecap="round"><g transform="translate(1401.1031044854076 462.7935947236553) rotate(0 -0.4588028452048327 24.6866069230841)"><path d="M0.91 -0.82 C0.57 7.35, -1.45 40.38, -1.83 48.74 M-0.07 1.36 C-0.04 9.75, 0.87 42.55, 0.51 50.19" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1401.1031044854076 462.7935947236553) rotate(0 -0.4588028452048327 24.6866069230841)"><path d="M-9.15 26 C-5.47 34, -1.86 42.47, 2.49 50.63 M-7.97 27.1 C-5.21 33.24, -3.15 41.06, 0.8 49.49" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1401.1031044854076 462.7935947236553) rotate(0 -0.4588028452048327 24.6866069230841)"><path d="M7.58 26.05 C5.97 34.16, 4.3 42.62, 2.49 50.63 M8.76 27.15 C6.74 33.41, 4.03 41.21, 0.8 49.49" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(1275.5475828382723 642.7936201549705) rotate(0 128.88890584309905 39.444427490234375)"><path d="M19.72 0 M19.72 0 C86.59 0.06, 149.32 2.24, 238.06 0 M19.72 0 C77.95 -0.92, 135.96 -1.37, 238.06 0 M238.06 0 C250.09 0.28, 255.95 8.13, 257.78 19.72 M238.06 0 C252.09 1.16, 260.02 7.68, 257.78 19.72 M257.78 19.72 C256.68 27.47, 258.39 37.04, 257.78 59.17 M257.78 19.72 C256.86 30.92, 257.8 39.98, 257.78 59.17 M257.78 59.17 C256.6 73.67, 253.12 77.66, 238.06 78.89 M257.78 59.17 C255.49 70.6, 249.24 77.31, 238.06 78.89 M238.06 78.89 C188.41 78.68, 138.59 77.83, 19.72 78.89 M238.06 78.89 C182.15 78.03, 127.07 78.75, 19.72 78.89 M19.72 78.89 C7.86 79.87, -0.71 71.92, 0 59.17 M19.72 78.89 C6.81 80.8, 0.2 73.39, 0 59.17 M0 59.17 C1.64 43.64, 1.73 31.08, 0 19.72 M0 59.17 C-0.07 45.17, -0.26 29.37, 0 19.72 M0 19.72 C-0.4 6.09, 5 1.97, 19.72 0 M0 19.72 C-0.01 5.79, 7.09 -1.22, 19.72 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1296.9364886813712 669.9880476452049) rotate(0 107.5 12.5)"><text x="107.5" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">解析参数:parse_args</text></g><g stroke-linecap="round"><g transform="translate(1396.6587278578036 596.1269534883038) rotate(0 0.972787512202455 22.44971209776901)"><path d="M-0.85 -0.88 C-0.61 6.88, 0.01 37.99, 0.44 45.78 M0.9 1.27 C1.63 8.72, 2.74 36.7, 2.8 43.78" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1396.6587278578036 596.1269534883038) rotate(0 0.972787512202455 22.44971209776901)"><path d="M-7.05 21.47 C-3.54 28.19, -1.56 31.76, 1.89 45.71 M-5.42 23.46 C-2.9 28.44, -1.78 33.11, 2.9 44.61" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1396.6587278578036 596.1269534883038) rotate(0 0.972787512202455 22.44971209776901)"><path d="M8.53 21.05 C8.69 27.74, 7.32 31.4, 1.89 45.71 M10.16 23.03 C8.74 28.13, 5.92 32.9, 2.9 44.61" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round"><g transform="translate(1403.8590694460809 726.571360633486) rotate(0 0.30498816941859275 35.332977231467794)"><path d="M0.71 -1.01 C0.85 10.84, 1.1 59.55, 0.95 71.67 M-0.38 1.08 C-0.38 12.54, -0.27 58.5, 0.17 69.96" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1403.8590694460809 726.571360633486) rotate(0 0.30498816941859275 35.332977231467794)"><path d="M-12.26 42.52 C-5.82 49.17, -2.41 59.41, 1.73 68.18 M-11.19 41.99 C-6.79 52.32, -3.27 64.25, 0.17 69.62" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1403.8590694460809 726.571360633486) rotate(0 0.30498816941859275 35.332977231467794)"><path d="M8.26 42.15 C8.36 49.05, 5.44 59.39, 1.73 68.18 M9.33 41.63 C5.88 52.28, 1.57 64.35, 0.17 69.62" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g mask="url(#mask-LntXasYDGGulEf1Kota9C)" stroke-linecap="round"><g transform="translate(1400.6267116486088 1165.6346519106912) rotate(0 0.6455569899325901 49.49917619131503)"><path d="M-0.27 0.19 C-0.15 16.66, 0.32 82.15, 0.29 98.83 M1.8 -0.76 C1.82 15.82, 0.03 82.67, -0.5 99.76" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1400.6267116486088 1165.6346519106912) rotate(0 0.6455569899325901 49.49917619131503)"><path d="M-9.65 71.77 C-9.4 78.14, -4.43 85.18, -0.63 98.61 M-9.71 70.98 C-7.36 81.18, -2.39 90.39, 0.47 99.31" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1400.6267116486088 1165.6346519106912) rotate(0 0.6455569899325901 49.49917619131503)"><path d="M10.86 72.35 C5.67 78.52, 5.19 85.41, -0.63 98.61 M10.8 71.56 C6.2 81.75, 4.21 90.77, 0.47 99.31" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask id="mask-LntXasYDGGulEf1Kota9C"><rect x="0" y="0" fill="#fff" width="1500.6267116486088" height="1365.6346519106912"></rect><rect x="1384.1267116486088" y="1203.134651910691" fill="#000" width="33" height="25" opacity="1"></rect></mask><g transform="translate(1384.1267116486088 1203.134651910691) rotate(0 17.14555698993259 11.99917619131503)"><text x="16.5" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">yes</text></g><g stroke-linecap="round" transform="translate(1284.4362209259523 1272.301325843448) rotate(0 137.6190621512277 42)"><path d="M21 0 M21 0 C107.72 2.67, 195.59 1.7, 254.24 0 M21 0 C76.93 -2, 131.78 -2.07, 254.24 0 M254.24 0 C267.76 1.54, 273.47 6.7, 275.24 21 M254.24 0 C267.7 -0.65, 277.53 7.14, 275.24 21 M275.24 21 C274.38 36.69, 273.54 52.93, 275.24 63 M275.24 21 C275.57 36.76, 275.45 50.99, 275.24 63 M275.24 63 C276.86 78.13, 267.85 85.3, 254.24 84 M275.24 63 C276.08 76.83, 266.01 84.58, 254.24 84 M254.24 84 C204.97 83.13, 154.23 81.4, 21 84 M254.24 84 C193.72 83.97, 135.01 85.52, 21 84 M21 84 C8.21 82.8, -0.67 77.37, 0 63 M21 84 C7.44 84.18, -0.15 78.46, 0 63 M0 63 C-1.27 50.5, -2.37 36.16, 0 21 M0 63 C0.82 51.97, 1.26 38.91, 0 21 M0 21 C0.24 6.72, 8.67 1.82, 21 0 M0 21 C-1.83 7.82, 5.07 -0.07, 21 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1293.0552830771799 1277.301325843448) rotate(0 129 37)"><text x="129" y="17.666666666666668" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">打开exit_fd用于感知退出:</text><text x="129" y="42.333333333333336" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">open_no_inherit("exit_fifo</text><text x="129" y="67" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">", O_WRONLY, -1)</text></g><g stroke-linecap="round"><g transform="translate(1399.695146562903 1362.7775090535483) rotate(0 1.4544136639424323 22.194266782244767)"><path d="M1.12 -0.95 C1.64 6.4, 2.49 36.37, 2.66 43.81 M0.24 1.16 C0.76 8.75, 2.08 38.05, 2.33 45.34" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1399.695146562903 1362.7775090535483) rotate(0 1.4544136639424323 22.194266782244767)"><path d="M-7.59 27.04 C-3.9 31.42, -1.95 37.31, 1.03 43.71 M-5.11 24.69 C-4.05 30.86, -0.27 38.91, 2.07 46.26" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1399.695146562903 1362.7775090535483) rotate(0 1.4544136639424323 22.194266782244767)"><path d="M7.39 26.42 C7.19 31.1, 5.26 37.15, 1.03 43.71 M9.87 24.08 C6.14 30.34, 5.15 38.59, 2.07 46.26" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(1254.9124695308628 1409.4441611880347) rotate(0 168 54.5)"><path d="M27.25 0 M27.25 0 C128.3 -1.44, 229.82 -2.77, 308.75 0 M27.25 0 C123.73 1.71, 220.75 1.34, 308.75 0 M308.75 0 C326.47 -0.5, 336.98 9.77, 336 27.25 M308.75 0 C328.71 -0.97, 337.6 9.36, 336 27.25 M336 27.25 C337.15 44.84, 338.38 57.26, 336 81.75 M336 27.25 C335.29 38.89, 335.46 51.76, 336 81.75 M336 81.75 C335.66 100.92, 326.19 109.74, 308.75 109 M336 81.75 C334.17 100.47, 325.8 107.13, 308.75 109 M308.75 109 C232.21 108.67, 152.91 110.45, 27.25 109 M308.75 109 C209.66 108.96, 110.62 109.75, 27.25 109 M27.25 109 C9.09 110.33, -0.96 101.52, 0 81.75 M27.25 109 C10.84 108.19, -1.96 99.13, 0 81.75 M0 81.75 C-1.18 66.51, -0.2 51.11, 0 27.25 M0 81.75 C0.03 68.12, -0.62 56.77, 0 27.25 M0 27.25 C1.72 7.12, 7.76 -0.99, 27.25 0 M0 27.25 C-2.04 10.01, 10.41 -1.12, 27.25 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1261.9124695308628 1426.9441611880347) rotate(0 161 37)"><text x="161" y="17.666666666666668" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">为isulad与runtime之间的io </text><text x="161" y="42.333333333333336" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">copychuan构建main </text><text x="161" y="67" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">loop和epoll::process_io_init(p);</text></g><g stroke-linecap="round"><g transform="translate(1413.5332741178509 1521.148946344285) rotate(0 2.596809273946544 31.688540488321905)"><path d="M-0.53 -1.18 C0.57 9.69, 4.75 53.6, 5.72 64.56 M1.39 0.82 C2.38 11.39, 3.54 52.48, 4.6 62.77" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1413.5332741178509 1521.148946344285) rotate(0 2.596809273946544 31.688540488321905)"><path d="M-9.33 34.72 C-3.71 40.3, -0.97 49.66, 5.95 61.74 M-8.15 35.92 C-3.49 44.17, 0.02 53.44, 4.41 63.75" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1413.5332741178509 1521.148946344285) rotate(0 2.596809273946544 31.688540488321905)"><path d="M11.16 33.46 C12.1 39.24, 10.18 48.88, 5.95 61.74 M12.33 34.66 C10.09 43.46, 6.7 53.15, 4.41 63.75" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(1268.2457302032963 1594.20612407866) rotate(0 161 40)"><path d="M20 0 M20 0 C107.9 -0.95, 193.34 -1.4, 302 0 M20 0 C79.11 -1.22, 138.88 0.01, 302 0 M302 0 C314.65 -1.65, 320.2 6.43, 322 20 M302 0 C317.2 1.7, 321.85 6.44, 322 20 M322 20 C321.86 29.49, 323.41 34.69, 322 60 M322 20 C322.61 33.08, 322.07 45.81, 322 60 M322 60 C321.08 74.32, 314.17 81.24, 302 80 M322 60 C321.81 71.8, 317.54 79.82, 302 80 M302 80 C192.09 79.36, 84.62 79.15, 20 80 M302 80 C241.09 79.83, 182 78.59, 20 80 M20 80 C6.39 78.06, 1.35 75.15, 0 60 M20 80 C6.5 82.24, -1.62 73.79, 0 60 M0 60 C-1.82 45.97, -1.63 29.28, 0 20 M0 60 C0.26 48.45, 0.17 37.64, 0 20 M0 20 C1.52 7.58, 5.56 -0.98, 20 0 M0 20 C-1.1 7.54, 7.52 -1.8, 20 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1307.2457302032963 1621.70612407866) rotate(0 122 12.5)"><text x="122" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">open_io(p, &tid_accept);</text></g><g mask="url(#mask-q0dCKctCJiUTNs4sgMiz5)" stroke-linecap="round"><g transform="translate(1522.5315244160004 1096.2695555914015) rotate(0 45.28467139740631 181.20656351208686)"><path d="M-0.23 -0.52 C14.65 31.46, 78.91 130.38, 90 190.77 C101.08 251.17, 70.15 333.13, 66.29 361.83 M-1.81 1.82 C12.85 34.07, 77.86 131.95, 89.04 192.13 C100.23 252.32, 68.97 334.84, 65.32 362.94" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1522.5315244160004 1096.2695555914015) rotate(0 45.28467139740631 181.20656351208686)"><path d="M60.86 333.14 C62.18 343.19, 63.37 354.76, 66.49 363.01 M61.38 332.92 C62.85 339.74, 62.67 347.31, 65.5 362.32" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1522.5315244160004 1096.2695555914015) rotate(0 45.28467139740631 181.20656351208686)"><path d="M80.85 337.81 C75.05 346.15, 69.07 356.04, 66.49 363.01 M81.36 337.58 C77.97 343.43, 72.89 349.86, 65.5 362.32" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask id="mask-q0dCKctCJiUTNs4sgMiz5"><rect x="0" y="0" fill="#fff" width="1712.5315752786307" height="1558.4917693365187"></rect><rect x="1601.5315752786307" y="1274.8806243169875" fill="#000" width="22" height="25" opacity="1"></rect></mask><g transform="translate(1601.5315752786307 1274.8806243169877) rotate(0 -33.715379465224046 2.5954947865010354)"><text x="11" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">no</text></g><g stroke-linecap="round"><g transform="translate(1599.0031322197585 1618.1728963707114) rotate(0 258.3291835806369 -343.0849661869071)"><path d="M-0.19 -0.03 C86.37 -114.42, 431.99 -571.94, 518.4 -686.14 M-1.75 -1.09 C84.82 -115.3, 431.35 -570.28, 517.97 -684.63" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1599.0031322197585 1618.1728963707114) rotate(0 258.3291835806369 -343.0849661869071)"><path d="M509.04 -654.34 C510.94 -660.87, 514.4 -669.82, 518.47 -683.03 M509.52 -656.57 C513.36 -666.87, 514.96 -677.68, 518.66 -684.22" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1599.0031322197585 1618.1728963707114) rotate(0 258.3291835806369 -343.0849661869071)"><path d="M492.7 -666.75 C498.24 -670.51, 505.33 -676.71, 518.47 -683.03 M493.18 -668.98 C503.37 -674.64, 511.29 -680.66, 518.66 -684.22" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(1831.1428220234227 408.2139067876908) rotate(0 399.16664123535156 253.33337783813477)"><path d="M32 0 M32 0 C301.54 -2.57, 569.75 -2.96, 766.33 0 M766.33 0 C785.91 -0.6, 797.14 11.22, 798.33 32 M798.33 32 C799.5 155.05, 798.8 276.1, 798.33 474.67 M798.33 474.67 C796.6 494.22, 786.5 508.09, 766.33 506.67 M766.33 506.67 C498.2 503.9, 228.73 505.18, 32 506.67 M32 506.67 C10.73 507.62, -0.5 496.55, 0 474.67 M0 474.67 C-0.55 385.51, -0.97 293.91, 0 32 M0 32 C-1.25 9.63, 10.83 1.07, 32 0" stroke="#000000" stroke-width="1.5" fill="none" stroke-dasharray="8 9"></path></g><g stroke-linecap="round" transform="translate(1976.142669435532 436.54734184628455) rotate(0 225 32.5)"><path d="M16.25 0 M16.25 0 C110.57 -0.85, 206.54 -1.77, 433.75 0 M16.25 0 C181.62 -2.66, 348.17 -2.73, 433.75 0 M433.75 0 C443.2 0.33, 448.56 5.14, 450 16.25 M433.75 0 C445.72 0.09, 448.93 7.59, 450 16.25 M450 16.25 C448.76 25.9, 448.42 37.29, 450 48.75 M450 16.25 C450.26 29.78, 450.23 41.06, 450 48.75 M450 48.75 C449.65 57.74, 444.22 65.88, 433.75 65 M450 48.75 C448.7 57.75, 442.85 62.8, 433.75 65 M433.75 65 C271.41 65.15, 108.23 65.23, 16.25 65 M433.75 65 C294.21 63.6, 154 63.34, 16.25 65 M16.25 65 C6.77 63.57, -1.01 60.6, 0 48.75 M16.25 65 C5.83 63.45, -1.65 60.4, 0 48.75 M0 48.75 C0.01 39.66, 1.01 34.31, 0 16.25 M0 48.75 C0.1 40.68, -0.16 31.28, 0 16.25 M0 16.25 C1.28 3.82, 7.4 -0.77, 16.25 0 M0 16.25 C-0.03 7.54, 3.4 -1.77, 16.25 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1983.642669435532 456.54734184628455) rotate(0 217.5 12.5)"><text x="217.5" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">创建io copy线程:start_io_copy_threads(p);</text></g><g stroke-linecap="round"><g transform="translate(2182.8092343769385 509.880700610933) rotate(0 -2.6443452030326853 46.86775347693822)"><path d="M-1.15 -0.99 C-2.12 14.49, -5.15 77.51, -5.74 93.24 M0.45 1.1 C-0.14 16.75, -2.46 79.13, -3.41 94.73" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2182.8092343769385 509.880700610933) rotate(0 -2.6443452030326853 46.86775347693822)"><path d="M-13.99 64.86 C-9.67 75.52, -6.73 83.96, -1.86 95.21 M-12.97 66.82 C-9.77 75.53, -6.41 82.31, -3.94 94.45" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2182.8092343769385 509.880700610933) rotate(0 -2.6443452030326853 46.86775347693822)"><path d="M6.51 65.83 C4.54 75.99, 1.2 84.14, -1.86 95.21 M7.53 67.78 C4.65 76.13, 1.94 82.63, -3.94 94.45" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(1989.4759519062352 576.5473799932572) rotate(0 195.83328247070312 66.66667938232422)"><path d="M245 16.75 M245 16.75 C281.77 29.78, 313.98 41.63, 342.67 50.25 M245 16.75 C283.08 30.11, 319.18 42.42, 342.67 50.25 M342.67 50.25 C391.64 67.3, 392.77 68.77, 342.67 83.75 M342.67 50.25 C390.31 68.86, 390.5 65.59, 342.67 83.75 M342.67 83.75 C305.4 94.05, 264.85 107.57, 245 116.58 M342.67 83.75 C321.98 91.14, 298.96 98.55, 245 116.58 M245 116.58 C196.56 134.02, 197.38 133.87, 147 116.58 M245 116.58 C197.63 133.74, 196.17 133.48, 147 116.58 M147 116.58 C113.62 107.25, 80.3 93.73, 49 83.75 M147 116.58 C119.65 107.1, 90.75 97.47, 49 83.75 M49 83.75 C0.89 67.99, 1.73 65.51, 49 50.25 M49 83.75 C1.4 65.87, -0.63 68.75, 49 50.25 M49 50.25 C77.13 40.29, 102.43 32.73, 147 16.75 M49 50.25 C78.47 39.89, 107.39 30.45, 147 16.75 M147 16.75 C196.79 0.21, 196.99 -0.91, 245 16.75 M147 16.75 C195.69 0.83, 194.64 -1.06, 245 16.75" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2070.3092343769385 630.7140593755814) rotate(0 115 12.5)"><text x="115" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">if (p->state->terminal) </text></g><g mask="url(#mask-7KGyT6UIt9mS3W8Kq8jdt)" stroke-linecap="round"><g transform="translate(2171.9582526622917 712.3923426352203) rotate(0 -111.84402588222395 38.935921198170035)"><path d="M0.1 1.07 C-36.99 13.53, -185.53 63.22, -222.72 75.77 M-1.3 0.59 C-38.56 13.77, -186.66 65, -223.79 77.28" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2171.9582526622917 712.3923426352203) rotate(0 -111.84402588222395 38.935921198170035)"><path d="M-198.61 58.11 C-207 62.96, -212.7 70.23, -222.06 75.79 M-199.64 58.23 C-208.87 64.54, -216.65 71.24, -223.77 77.32" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2171.9582526622917 712.3923426352203) rotate(0 -111.84402588222395 38.935921198170035)"><path d="M-192.01 77.54 C-202.45 76.85, -210.03 78.58, -222.06 75.79 M-193.04 77.66 C-204.55 76.94, -214.71 76.64, -223.77 77.32" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask id="mask-7KGyT6UIt9mS3W8Kq8jdt"><rect x="0" y="0" fill="#fff" width="2494.440553418348" height="888.2139449346635"></rect><rect x="2038.7171022842633" y="737.803143784942" fill="#000" width="44" height="25" opacity="1"></rect></mask><g transform="translate(2038.7171022842635 737.8031437849419) rotate(0 21.39712449580429 13.525120048448457)"><text x="22" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">true</text></g><g stroke-linecap="round" transform="translate(1866.142669435532 786.5473799932572) rotate(0 126 54.5)"><path d="M27.25 0 M27.25 0 C104.69 0.21, 180.79 -1.06, 224.75 0 M27.25 0 C93.7 1.42, 159.63 1.76, 224.75 0 M224.75 0 C243.88 -1.88, 252.66 9.47, 252 27.25 M224.75 0 C243.16 1.52, 253.54 8.86, 252 27.25 M252 27.25 C253.44 45.36, 252.18 66.05, 252 81.75 M252 27.25 C252.43 43.94, 251.92 59.57, 252 81.75 M252 81.75 C253.76 101.42, 242.28 107.16, 224.75 109 M252 81.75 C251.24 102, 244.39 110.23, 224.75 109 M224.75 109 C169.77 105.52, 116.12 105.78, 27.25 109 M224.75 109 C167.91 108.78, 111.63 108.37, 27.25 109 M27.25 109 C7.96 109.43, 0.43 100.52, 0 81.75 M27.25 109 C10.3 107.93, -0.73 97.81, 0 81.75 M0 81.75 C0.33 59.5, -2.12 41.13, 0 27.25 M0 81.75 C-0.59 61.86, -0.34 40.19, 0 27.25 M0 27.25 C-0.06 11.05, 9.37 -1.66, 27.25 0 M0 27.25 C0.41 9.82, 9.94 -1.34, 27.25 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1876.9760282001805 791.5473799932572) rotate(0 115 49.5)"><text x="115" y="17.75" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">开勇socket实现isulad与r</text><text x="115" y="42.5" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">untime的通信:</text><text x="115" y="67.25" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">open_terminal_io(p, </text><text x="115" y="92" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">tid_accept);</text></g><g stroke-linecap="round" transform="translate(2272.809386964829 781.5473799932572) rotate(0 130 50)"><path d="M25 0 M25 0 C81.63 -0.11, 139.44 0.12, 235 0 M25 0 C83.97 0.9, 143.43 0.54, 235 0 M235 0 C250.55 0.43, 260.43 8.94, 260 25 M235 0 C252.89 -1.07, 259.27 6.23, 260 25 M260 25 C261.04 43.22, 258.59 65.32, 260 75 M260 25 C260.22 44.52, 260.46 62.25, 260 75 M260 75 C259.94 93.63, 251.96 98.34, 235 100 M260 75 C260.41 92.4, 252.52 98.66, 235 100 M235 100 C161.55 98.22, 87.57 99.5, 25 100 M235 100 C179.56 100.41, 124.68 99.36, 25 100 M25 100 C8.77 98.73, -0.16 91.7, 0 75 M25 100 C7.38 98.29, -2.15 91.34, 0 75 M0 75 C1.68 58.58, -0.94 43.91, 0 25 M0 75 C1.02 56.43, -0.2 38.47, 0 25 M0 25 C-0.08 8.16, 6.88 1.22, 25 0 M0 25 C-1.59 8.34, 10.59 -0.91, 25 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2280.809386964829 807.0473799932572) rotate(0 122 24.5)"><text x="122" y="17.5" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">利用pipe实现通信:open_g</text><text x="122" y="42" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">eneric_io(p);</text></g><g mask="url(#mask-LUv3oxOqWFjE79MdwSowr)" stroke-linecap="round"><g transform="translate(2204.7739526643422 714.2473090855844) rotate(0 117.56666383628414 33.29057247246328)"><path d="M0.38 0.45 C39.49 11.57, 194.68 56.02, 233.66 66.94 M-0.87 -0.36 C38.71 10.39, 196.95 53.9, 236.01 65.16" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2204.7739526643422 714.2473090855844) rotate(0 117.56666383628414 33.29057247246328)"><path d="M206.84 66.26 C213.65 66.67, 218.6 67.29, 234.48 66.45 M206.12 67.59 C215.35 67.85, 223 66.43, 235.37 65.08" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2204.7739526643422 714.2473090855844) rotate(0 117.56666383628414 33.29057247246328)"><path d="M212.38 46.5 C218.02 51.21, 221.77 56.11, 234.48 66.45 M211.66 47.83 C219.19 53.91, 225.2 58.33, 235.37 65.08" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask id="mask-LUv3oxOqWFjE79MdwSowr"><rect x="0" y="0" fill="#fff" width="2539.1351410193683" height="880.5473799932572"></rect><rect x="2295.9545468418555" y="734.8973445394208" fill="#000" width="52" height="25" opacity="1"></rect></mask><g transform="translate(2295.954546841855 734.8973445394208) rotate(0 26.38606965877119 12.640537018626901)"><text x="26" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">false</text></g><g stroke-linecap="round"><g transform="translate(1420.3004592009356 1675.20612407866) rotate(0 -0.08657229545204714 41.799007109740614)"><path d="M-1.17 1.15 C-1.09 14.6, 0.73 67.99, 0.99 81.73 M0.42 0.71 C0.28 14.82, -0.5 69.37, -0.08 82.89" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1420.3004592009356 1675.20612407866) rotate(0 -0.08657229545204714 41.799007109740614)"><path d="M-8.56 54.35 C-7.88 63.16, -1.2 75.76, -1.91 82.3 M-10.19 55.08 C-7.67 62.99, -3.76 70.53, -0.85 82.9" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1420.3004592009356 1675.20612407866) rotate(0 -0.08657229545204714 41.799007109740614)"><path d="M11.96 54.25 C5.62 62.9, 5.27 75.53, -1.91 82.3 M10.34 54.98 C7.26 62.93, 5.58 70.5, -0.85 82.9" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(1261.5525301978705 1760.6214130946569) rotate(0 157.77777777777771 54.44444444444446)"><path d="M27.22 0 M27.22 0 C78.28 1.66, 133.21 1.78, 288.33 0 M27.22 0 C129.9 -2.66, 233.19 -1.36, 288.33 0 M288.33 0 C304.56 -1.06, 313.73 9.07, 315.56 27.22 M288.33 0 C306.23 0.42, 315.1 8.08, 315.56 27.22 M315.56 27.22 C316.41 41.52, 313.61 53.31, 315.56 81.67 M315.56 27.22 C316.2 40.79, 315.72 52.94, 315.56 81.67 M315.56 81.67 C315.07 101.28, 307.9 109.65, 288.33 108.89 M315.56 81.67 C314.4 100.39, 305.4 110.68, 288.33 108.89 M288.33 108.89 C226.08 108.37, 165.98 110.6, 27.22 108.89 M288.33 108.89 C201.29 107.83, 116.08 108.6, 27.22 108.89 M27.22 108.89 C9.21 108.8, 0.65 101.34, 0 81.67 M27.22 108.89 C8.47 110.69, -0.7 99.51, 0 81.67 M0 81.67 C0.39 61.39, -0.4 41, 0 27.22 M0 81.67 C1.11 66.82, 0.03 51.21, 0 27.22 M0 27.22 C0.78 9.85, 7.21 -1.52, 27.22 0 M0 27.22 C0.18 8.06, 10.92 0.35, 27.22 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1326.3303079756483 1802.5658575391014) rotate(0 93 12.5)"><text x="93" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">create_process(p);</text></g><g stroke-linecap="round" transform="translate(1827.221848018586 1290.297197735499) rotate(0 373.3332824707031 378.9898120470381)"><path d="M32 0 M32 0 C283.39 -2.54, 533.38 -2.58, 714.67 0 M714.67 0 C737 -1.75, 746.69 11.47, 746.67 32 M746.67 32 C745.55 229.85, 746.68 426.15, 746.67 725.98 M746.67 725.98 C745.93 748.75, 736.86 758.41, 714.67 757.98 M714.67 757.98 C571.61 756.06, 428.25 755.04, 32 757.98 M32 757.98 C9.28 757.62, -1.44 748.3, 0 725.98 M0 725.98 C2.46 471.01, 2.3 217.13, 0 32 M0 32 C1.51 9.96, 12.4 0.63, 32 0" stroke="#000000" stroke-width="1.5" fill="none" stroke-dasharray="8 9"></path></g><g stroke-linecap="round" transform="translate(1959.133043697269 1301.6104122503416) rotate(0 134.39996304599237 26.472719993907617)"><path d="M13.24 0 M13.24 0 C86.92 -1.36, 159.68 -0.48, 255.56 0 M13.24 0 C106.29 -1.23, 198.43 -2.14, 255.56 0 M255.56 0 C262.97 -1.39, 267.18 6.21, 268.8 13.24 M255.56 0 C263.28 -0.59, 266.95 6.24, 268.8 13.24 M268.8 13.24 C268.66 24.63, 268 33.79, 268.8 39.71 M268.8 13.24 C269.14 19.37, 269.2 24.76, 268.8 39.71 M268.8 39.71 C269.88 48.95, 263.67 52.27, 255.56 52.95 M268.8 39.71 C267.04 47.55, 262.6 53.51, 255.56 52.95 M255.56 52.95 C183.49 55.3, 114.61 56.19, 13.24 52.95 M255.56 52.95 C177.44 55.54, 98.32 54.78, 13.24 52.95 M13.24 52.95 C5.31 51.29, -1.18 47.24, 0 39.71 M13.24 52.95 C3 54.9, 1.81 49.2, 0 39.71 M0 39.71 C0.08 30.56, -1.22 22.68, 0 13.24 M0 39.71 C-0.51 34.12, -0.6 27.46, 0 13.24 M0 13.24 C-0.34 5.8, 3.44 1.95, 13.24 0 M0 13.24 C-2.25 6.14, 3.98 2.13, 13.24 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2065.0330067432615 1315.355863016409) rotate(0 28.5 12.5)"><text x="28.5" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20.363630764544297px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">fork()</text></g><g mask="url(#mask-Rc4K0TIWjy6S3p32VHwvv)" stroke-linecap="round"><g transform="translate(2048.0312895864267 1359.048191365566) rotate(0 -31.98153403145136 76.89349271115975)"><path d="M0.01 -0.25 C-10.28 25.54, -52.26 128.13, -62.91 153.89 M-1.44 -1.42 C-11.81 24.61, -53.95 129.02, -63.97 155.21" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2048.0312895864267 1359.048191365566) rotate(0 -31.98153403145136 76.89349271115975)"><path d="M-63.58 124.84 C-64.25 129.39, -63.64 137.51, -64.83 156.62 M-63.23 124.64 C-64.14 134.61, -64.66 143.86, -64.44 156.04" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2048.0312895864267 1359.048191365566) rotate(0 -31.98153403145136 76.89349271115975)"><path d="M-44.49 132.38 C-49.01 135.38, -52.31 141.96, -64.83 156.62 M-44.14 132.18 C-50.95 139.7, -57.41 146.61, -64.44 156.04" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask id="mask-Rc4K0TIWjy6S3p32VHwvv"><rect x="0" y="0" fill="#fff" width="2210.7118420415554" height="1613.3612334132815"></rect><rect x="1915.6910133588624" y="1423.7047123894235" fill="#000" width="202" height="25" opacity="1"></rect></mask><g transform="translate(1915.6910133588626 1423.7047123894235) rotate(0 100.35874219611287 12.236971687302002)"><text x="101" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20.363630764544297px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">child:runtime process</text></g><g mask="url(#mask-8ge6qsTvyUMfvKpaRsTeV)" stroke-linecap="round"><g transform="translate(2157.2507277798304 1359.3937295493458) rotate(0 98.62367737808961 61.46057749104648)"><path d="M-0.5 0.55 C32.11 20.68, 162.8 101.34, 195.61 121.77 M1.44 -0.21 C34.41 20.62, 165.42 102.87, 197.75 123.13" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2157.2507277798304 1359.3937295493458) rotate(0 98.62367737808961 61.46057749104648)"><path d="M169.33 115.02 C176.62 118.22, 187.66 121.19, 196.28 123.1 M167.65 116.25 C177.29 119.39, 186.28 121.53, 197.16 122.37" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2157.2507277798304 1359.3937295493458) rotate(0 98.62367737808961 61.46057749104648)"><path d="M180.24 97.64 C184.43 105.96, 192.22 114.11, 196.28 123.1 M178.56 98.86 C184.81 107.53, 190.36 115.15, 197.16 122.37" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask id="mask-8ge6qsTvyUMfvKpaRsTeV"><rect x="0" y="0" fill="#fff" width="2453.9436464239634" height="1581.5057185960716"></rect><rect x="2163.097187101897" y="1395.9497240727087" fill="#000" width="185" height="49" opacity="1"></rect></mask><g transform="translate(2163.0971871018965 1395.949724072709) rotate(0 92.77721805602312 24.90458296768361)"><text x="92.5" y="17.5" font-family="Virgil, Segoe UI Emoji" font-size="20.363630764544297px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">parent:isulad-shim </text><text x="92.5" y="42" font-family="Virgil, Segoe UI Emoji" font-size="20.363630764544297px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">process</text></g><g stroke-linecap="round" transform="translate(1842.607840473729 1516.5598308358458) rotate(0 144.5817784282645 55.49089383338321)"><path d="M27.75 0 M27.75 0 C81.57 1.83, 131.7 2.05, 261.42 0 M27.75 0 C89.55 0.75, 150.65 1.23, 261.42 0 M261.42 0 C278.67 -0.65, 287.51 7.51, 289.16 27.75 M261.42 0 C277.75 -0.76, 288.79 9.15, 289.16 27.75 M289.16 27.75 C288.5 47.47, 290.74 64.81, 289.16 83.24 M289.16 27.75 C289.22 42.86, 289.91 57.36, 289.16 83.24 M289.16 83.24 C290.09 101.36, 278.97 111.7, 261.42 110.98 M289.16 83.24 C288.59 101.18, 278.99 113.17, 261.42 110.98 M261.42 110.98 C179.66 112.39, 94.76 113.94, 27.75 110.98 M261.42 110.98 C174.8 112.15, 87.46 112.08, 27.75 110.98 M27.75 110.98 C7.72 112.68, 0.27 100.59, 0 83.24 M27.75 110.98 C10.08 111.4, -2.29 103.67, 0 83.24 M0 83.24 C1.1 64.96, 1.42 48.53, 0 27.75 M0 83.24 C0.01 69.8, -0.48 57.42, 0 27.75 M0 27.75 C0.01 8.89, 8.52 1.2, 27.75 0 M0 27.75 C-1.13 7.55, 7.66 -0.31, 27.75 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1850.1896189019938 1534.3780077548222) rotate(0 137 37)"><text x="137" y="17.666666666666668" font-family="Virgil, Segoe UI Emoji" font-size="20.363630764544297px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">拼接params,并调用runc二进</text><text x="137" y="42.333333333333336" font-family="Virgil, Segoe UI Emoji" font-size="20.363630764544297px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">制执行命令:exec_runtime_</text><text x="137" y="67" font-family="Virgil, Segoe UI Emoji" font-size="20.363630764544297px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">process(p, exec_fd[1]);</text></g><g stroke-linecap="round" transform="translate(2221.597756095549 1493.933660743115) rotate(0 134.62619108511535 55.49089383338321)"><path d="M27.75 0 M27.75 0 C89.1 0.99, 151.13 0.88, 241.51 0 M27.75 0 C100.35 0.24, 170.38 0.77, 241.51 0 M241.51 0 C260.13 0.61, 269.36 10.6, 269.25 27.75 M241.51 0 C261.99 0.28, 269.35 9.95, 269.25 27.75 M269.25 27.75 C271.17 42.24, 269.42 55.26, 269.25 83.24 M269.25 27.75 C269.78 44.29, 269.57 61.57, 269.25 83.24 M269.25 83.24 C270.63 102.21, 258.36 111.69, 241.51 110.98 M269.25 83.24 C269.68 100.07, 257.71 111.76, 241.51 110.98 M241.51 110.98 C193.04 111.59, 147.68 110.93, 27.75 110.98 M241.51 110.98 C161.67 110.68, 82.74 110.5, 27.75 110.98 M27.75 110.98 C9.73 110.78, 1.6 102.86, 0 83.24 M27.75 110.98 C10.84 110.47, 2.15 101.91, 0 83.24 M0 83.24 C0.37 66.45, -0.9 54.3, 0 27.75 M0 83.24 C0.68 68.25, 0.98 54.6, 0 27.75 M0 27.75 C1.49 8.38, 10.46 1.92, 27.75 0 M0 27.75 C0.92 10.42, 8.89 1.11, 27.75 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2227.2239471806643 1499.024568434251) rotate(0 129.00000000000006 49.5)"><text x="129" y="17.75" font-family="Virgil, Segoe UI Emoji" font-size="20.363630764544297px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">读取runtime写入的信息:</text><text x="129" y="42.5" font-family="Virgil, Segoe UI Emoji" font-size="20.363630764544297px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr"> read_nointr(exec_fd[0],</text><text x="129" y="67.25" font-family="Virgil, Segoe UI Emoji" font-size="20.363630764544297px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">exec_buff, </text><text x="129" y="92" font-family="Virgil, Segoe UI Emoji" font-size="20.363630764544297px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">sizeof(exec_buff) - 1);</text></g><g stroke-linecap="round"><g transform="translate(2349.282735385529 1612.8633802909478) rotate(0 0.08768612112515939 12.03822267685382)"><path d="M-0.58 0.96 C-0.73 4.59, -0.58 18.5, -0.78 22.04 M1.31 0.42 C1.1 4.25, -0.51 19.78, -1.14 23.66" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2349.282735385529 1612.8633802909478) rotate(0 0.08768612112515939 12.03822267685382)"><path d="M-2.69 13.97 C-2.66 16.4, -1.74 20.58, -1.35 23.61 M-4.05 12.93 C-3.11 15.88, -2.42 17.73, -1.63 23.76" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2349.282735385529 1612.8633802909478) rotate(0 0.08768612112515939 12.03822267685382)"><path d="M4.77 14.92 C2.5 17.02, 1.11 20.91, -1.35 23.61 M3.41 13.88 C2.53 16.65, 1.4 18.28, -1.63 23.76" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(2230.648224132641 1644.3981967868378) rotate(0 126.76360150928826 46.836350758451886)"><path d="M23.42 0 M23.42 0 C70.62 -0.56, 121.22 -1.39, 230.11 0 M23.42 0 C66.93 -0.58, 110.73 0.52, 230.11 0 M230.11 0 C245.15 -1.85, 255.21 8.22, 253.53 23.42 M230.11 0 C246.96 -1.52, 253.04 7.94, 253.53 23.42 M253.53 23.42 C254.82 32.62, 252.53 43.71, 253.53 70.25 M253.53 23.42 C252.62 38.98, 254.48 55.54, 253.53 70.25 M253.53 70.25 C252.99 84.5, 244.78 95.15, 230.11 93.67 M253.53 70.25 C251.47 86.91, 245.06 95.09, 230.11 93.67 M230.11 93.67 C169.79 93.28, 111.35 91.61, 23.42 93.67 M230.11 93.67 C158.25 95.34, 87.37 94.82, 23.42 93.67 M23.42 93.67 C5.98 92.1, 0.61 85.42, 0 70.25 M23.42 93.67 C7.96 95.95, 0.94 84.8, 0 70.25 M0 70.25 C-1.27 61.41, 0.03 50.62, 0 23.42 M0 70.25 C0.83 59.89, 0.3 48.49, 0 23.42 M0 23.42 C1.52 8.73, 6.57 -2, 23.42 0 M0 23.42 C-0.7 6.11, 7.95 -1.3, 23.42 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2240.4118256419297 1666.289099858723) rotate(0 117 24.5)"><text x="117" y="17.5" font-family="Virgil, Segoe UI Emoji" font-size="20.363630764544297px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">等待runtime子进程退出:</text><text x="117" y="42" font-family="Virgil, Segoe UI Emoji" font-size="20.363630764544297px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">waitpid(pid, NULL, 0);</text></g><g stroke-linecap="round" transform="translate(2194.1067509886325 1784.6810210230703) rotate(0 180.72722303533067 42.76362460554299)"><path d="M21.38 0 M21.38 0 C119.63 -0.17, 219.11 1.42, 340.07 0 M21.38 0 C121.97 2.05, 220.87 1.39, 340.07 0 M340.07 0 C352.95 -0.52, 361.93 6.08, 361.45 21.38 M340.07 0 C355.86 -0.34, 361.39 6.47, 361.45 21.38 M361.45 21.38 C362.09 34.95, 360.81 52.36, 361.45 64.15 M361.45 21.38 C360.86 35.77, 361.04 49.88, 361.45 64.15 M361.45 64.15 C361.79 77.15, 354.21 86.77, 340.07 85.53 M361.45 64.15 C361.78 79.1, 353.13 85.01, 340.07 85.53 M340.07 85.53 C273.95 86.18, 205.07 87.03, 21.38 85.53 M340.07 85.53 C258.17 84.29, 177.19 84.13, 21.38 85.53 M21.38 85.53 C7.21 86.34, 1.48 76.6, 0 64.15 M21.38 85.53 C7.19 84.15, 1.08 76.41, 0 64.15 M0 64.15 C-1.55 52.05, 0.48 43.05, 0 21.38 M0 64.15 C0.31 49.46, 0.38 33.16, 0 21.38 M0 21.38 C-1.45 8.42, 8.64 -1.71, 21.38 0 M0 21.38 C-2.13 8.16, 7.52 -0.4, 21.38 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2204.333974023963 1802.4991979420467) rotate(0 170.49999999999994 24.5)"><text x="170.5" y="17.5" font-family="Virgil, Segoe UI Emoji" font-size="20.363630764544297px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">获得runtime写入的容器中进程的pid:</text><text x="170.5" y="42" font-family="Virgil, Segoe UI Emoji" font-size="20.363630764544297px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">read_text_file("pid");</text></g><g stroke-linecap="round"><g transform="translate(2360.451236171784 1742.8224639361931) rotate(0 0.22061756550613154 23.608614339690348)"><path d="M0.57 -1.04 C0.73 6.91, 1.09 39.07, 1.04 46.99 M-0.6 1.03 C-0.62 9.2, -0.64 40.55, -0.11 48.26" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2360.451236171784 1742.8224639361931) rotate(0 0.22061756550613154 23.608614339690348)"><path d="M-10.61 25.52 C-7.22 30.96, -5.44 38.86, 0.19 50.08 M-8.42 26.64 C-5.18 33.27, -3.31 42.42, 0.65 47.4" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2360.451236171784 1742.8224639361931) rotate(0 0.22061756550613154 23.608614339690348)"><path d="M5.64 25.06 C4.11 30.71, 0.97 38.75, 0.19 50.08 M7.83 26.17 C5.32 33.06, 1.42 42.37, 0.65 47.4" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round"><g transform="translate(2363.0117205273973 1872.9234210027623) rotate(0 0.5839950477662228 22.017300675115166)"><path d="M-0.79 -0.29 C-0.36 7.36, 1.76 37.78, 1.95 45.53 M1 -1.49 C1.4 5.81, 1.12 35.9, 1.44 43.55" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2363.0117205273973 1872.9234210027623) rotate(0 0.5839950477662228 22.017300675115166)"><path d="M-7.08 23.76 C-4.2 26.48, -1.04 34.91, 1.11 44.13 M-6.88 23.2 C-4.3 29.84, -1.68 35.38, 2.09 44.15" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2363.0117205273973 1872.9234210027623) rotate(0 0.5839950477662228 22.017300675115166)"><path d="M8.4 23.55 C7.43 26.22, 6.73 34.7, 1.11 44.13 M8.6 22.99 C6.46 29.6, 4.36 35.21, 2.09 44.15" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(2218.2037140600096 1917.0446727799992) rotate(0 150.18177688851415 40.21817075997501)"><path d="M20.11 0 M20.11 0 C100.59 -2.4, 176.93 -1.89, 280.25 0 M20.11 0 C79.16 -1.22, 138.65 -1.21, 280.25 0 M280.25 0 C293.9 -1, 302.15 6.54, 300.36 20.11 M280.25 0 C295.19 1.7, 302.66 5.22, 300.36 20.11 M300.36 20.11 C301.59 33.93, 300.74 47.54, 300.36 60.33 M300.36 20.11 C300.17 31.34, 299.76 43.73, 300.36 60.33 M300.36 60.33 C300.55 72.15, 294.56 78.7, 280.25 80.44 M300.36 60.33 C298.45 74.64, 291.9 81.14, 280.25 80.44 M280.25 80.44 C218.75 83.44, 158.97 82.66, 20.11 80.44 M280.25 80.44 C218.8 81.82, 155.42 83.07, 20.11 80.44 M20.11 80.44 C7.85 78.77, -0.94 72.65, 0 60.33 M20.11 80.44 C5.5 80.13, 0.8 75.08, 0 60.33 M0 60.33 C1.75 49.33, 1.85 41.71, 0 20.11 M0 60.33 C-0.2 45.65, 0.54 29.11, 0 20.11 M0 20.11 C1.86 6.96, 7.95 0.42, 20.11 0 M0 20.11 C-2.18 8.31, 5.83 1.45, 20.11 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2224.8854909485235 1932.3173958534076) rotate(0 143.5 24.5)"><text x="143.5" y="17.5" font-family="Virgil, Segoe UI Emoji" font-size="20.363630764544297px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">将其赋值给process:p->ctr_pi</text><text x="143.5" y="42" font-family="Virgil, Segoe UI Emoji" font-size="20.363630764544297px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">d = ctr_pid;</text></g><g stroke-linecap="round"><g transform="translate(1574.999466426789 1808.297391013494) rotate(0 127.63571798438697 -191.61120642804963)"><path d="M1.07 0.82 C43.61 -63.05, 212.79 -318.33, 255.1 -382.52 M0.17 0.2 C42.66 -64.03, 212.34 -320.12, 254.74 -384.04" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1574.999466426789 1808.297391013494) rotate(0 127.63571798438697 -191.61120642804963)"><path d="M249.08 -352.99 C251.5 -361.86, 250.57 -372.26, 256.49 -384.28 M247.58 -354.98 C250.93 -363.98, 252.17 -372.99, 254.33 -384.94" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1574.999466426789 1808.297391013494) rotate(0 127.63571798438697 -191.61120642804963)"><path d="M231.98 -364.33 C239.43 -370.06, 243.46 -377.16, 256.49 -384.28 M230.47 -366.32 C239.25 -371.64, 245.91 -377.05, 254.33 -384.94" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round"><g transform="translate(1414.980540143239 1881.6307040017753) rotate(0 -0.26155997711292045 37.80616365604101)"><path d="M1.1 -0.33 C0.85 12.21, -0.14 63.06, -0.4 75.75 M0.22 -1.55 C-0.33 11.13, -1.32 64.07, -1.62 77.16" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1414.980540143239 1881.6307040017753) rotate(0 -0.26155997711292045 37.80616365604101)"><path d="M-11.85 47.9 C-7.81 58.76, -3.93 72.26, -2.63 75.63 M-11.06 48.68 C-8.89 56.91, -5.31 63.39, -1.92 78.13" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1414.980540143239 1881.6307040017753) rotate(0 -0.26155997711292045 37.80616365604101)"><path d="M8.67 48.33 C4.7 59.21, 0.56 72.54, -2.63 75.63 M9.46 49.11 C6.35 57.2, 4.66 63.56, -1.92 78.13" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(1261.6660924033515 1958.9640169900565) rotate(0 155.33334350585938 42)"><path d="M21 0 M21 0 C80.56 -2.45, 143.52 -1.92, 289.67 0 M21 0 C126.23 -1.4, 231.69 -0.64, 289.67 0 M289.67 0 C302.04 -1.8, 311.46 8.4, 310.67 21 M289.67 0 C302.19 -2.13, 311.46 6.07, 310.67 21 M310.67 21 C310.51 34.18, 310.99 48.05, 310.67 63 M310.67 21 C311.11 33, 311.49 44.63, 310.67 63 M310.67 63 C310.96 76.74, 303.93 83.45, 289.67 84 M310.67 63 C309.16 77.16, 303.55 82.21, 289.67 84 M289.67 84 C201.5 83.79, 115.51 85.51, 21 84 M289.67 84 C235.36 84.11, 180.76 83.86, 21 84 M21 84 C7.32 84.37, -0.13 75.9, 0 63 M21 84 C6.61 81.96, 1 77.93, 0 63 M0 63 C0.78 56.02, 2.09 46.27, 0 21 M0 63 C0.66 51.8, -0.28 39.01, 0 21 M0 21 C0.27 7.02, 8.1 -1.59, 21 0 M0 21 C0.58 5.96, 6.46 0.34, 21 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1272.999435909211 1976.4640169900565) rotate(0 144 24.5)"><text x="144" y="17.5" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">清除超时计时器:released_tim</text><text x="144" y="42" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">eout_exit();</text></g><g stroke-linecap="round"><g transform="translate(1413.0217921508151 2044.8053430224224) rotate(0 -2.128071379236143 103.37967504525125)"><path d="M0.18 -0.07 C-0.39 34.4, -3.23 171.87, -3.96 206.55 M-1.19 -1.15 C-1.79 33.52, -3.85 173.35, -4.43 207.91" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1413.0217921508151 2044.8053430224224) rotate(0 -2.128071379236143 103.37967504525125)"><path d="M-14.37 181.08 C-12.37 187.84, -7.44 193.44, -4.28 208.69 M-13.37 178.89 C-11.75 186.24, -10.81 192.18, -3.91 207.26" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1413.0217921508151 2044.8053430224224) rotate(0 -2.128071379236143 103.37967504525125)"><path d="M6.15 181.4 C2.92 188.11, 2.62 193.63, -4.28 208.69 M7.15 179.21 C4.51 186.31, 1.18 192.18, -3.91 207.26" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(1179.6660313681953 2256.297329978338) rotate(0 239 29)"><path d="M14.5 0 M14.5 0 C140.37 0.68, 265.98 0.32, 463.5 0 M14.5 0 C126.52 1.13, 238.33 0.52, 463.5 0 M463.5 0 C472.65 -1.24, 478.65 6.08, 478 14.5 M463.5 0 C471.37 0.5, 480.23 5.44, 478 14.5 M478 14.5 C479.63 24.38, 478.71 35.94, 478 43.5 M478 14.5 C478.15 20.42, 478.32 26.66, 478 43.5 M478 43.5 C479.12 54.44, 472.1 56.62, 463.5 58 M478 43.5 C478.48 53.26, 474.23 56.23, 463.5 58 M463.5 58 C314.19 60.56, 167.06 60.61, 14.5 58 M463.5 58 C314.6 58.56, 166.2 59.31, 14.5 58 M14.5 58 C4.86 58.31, 1.19 51.6, 0 43.5 M14.5 58 C6.64 60.15, -1.47 52.11, 0 43.5 M0 43.5 C-0.11 33.93, -0.15 23.99, 0 14.5 M0 43.5 C0.85 35.2, 1.06 27.89, 0 14.5 M0 14.5 C-0.87 5.04, 3.23 -0.96, 14.5 0 M0 14.5 C-1.99 6.97, 5.29 2.11, 14.5 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1189.6660313681953 2272.797329978338) rotate(0 229 12.5)"><text x="229" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">process_signal_handle_routine(p, tid_accept);</text></g><g stroke-linecap="round"><g transform="translate(1659.555344112336 2281.519772605291) rotate(0 55.68775084322624 0.17229592370222235)"><path d="M-1.16 1.11 C17.04 1.35, 91.65 1.05, 110.27 0.83 M0.43 0.65 C18.92 0.61, 94.36 -0.91, 112.54 -0.86" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1659.555344112336 2281.519772605291) rotate(0 55.68775084322624 0.17229592370222235)"><path d="M86.31 8.29 C96.78 4.93, 105.91 1, 112.13 -0.38 M84.22 9.19 C92.78 6.13, 103.61 3.17, 112.73 -0.64" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1659.555344112336 2281.519772605291) rotate(0 55.68775084322624 0.17229592370222235)"><path d="M86.1 -12.23 C96.54 -7.63, 105.76 -3.61, 112.13 -0.38 M84.01 -11.33 C92.79 -7.67, 103.69 -3.91, 112.73 -0.64" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(1812.888626583039 2113.741960919093) rotate(0 384.4445292154951 591.111094156901)"><path d="M32 0 M32 0 C177.35 1.2, 321.78 1.74, 736.89 0 M736.89 0 C759.01 1.64, 769.06 9.78, 768.89 32 M768.89 32 C769.76 460.41, 769.91 889.49, 768.89 1150.22 M768.89 1150.22 C768.86 1170.68, 756.44 1183.81, 736.89 1182.22 M736.89 1182.22 C530.29 1180.73, 324.5 1181.16, 32 1182.22 M32 1182.22 C12.47 1183.02, 0.72 1170.12, 0 1150.22 M0 1150.22 C-4.26 784.47, -3.14 417.78, 0 32 M0 32 C-0.32 8.95, 11.14 -0.39, 32 0" stroke="#000000" stroke-width="1.5" fill="none" stroke-dasharray="8 9"></path></g><g stroke-linecap="round" transform="translate(1857.3332066611645 2135.9642255268404) rotate(0 193.5 66.5)"><path d="M32 0 M32 0 C158.07 1.65, 285.55 1.65, 355 0 M32 0 C99.37 0.78, 167.14 0.63, 355 0 M355 0 C376.86 0.42, 388.04 11.45, 387 32 M355 0 C374.75 1.9, 387.21 9.62, 387 32 M387 32 C385.39 57.26, 386.63 82.82, 387 101 M387 32 C385.71 56, 387.3 78.73, 387 101 M387 101 C387.53 122.64, 377.31 134.23, 355 133 M387 101 C385.9 121.22, 377.86 135.05, 355 133 M355 133 C284.38 133.47, 217.73 133.55, 32 133 M355 133 C255.87 134.51, 157.03 134, 32 133 M32 133 C12.28 134.87, -0.96 123.15, 0 101 M32 133 C10.82 133.9, 0.23 124.34, 0 101 M0 101 C1.69 80.29, -0.27 62.76, 0 32 M0 101 C0.16 73.61, 0.28 46.27, 0 32 M0 32 C-0.58 11.94, 10.12 -0.45, 32 0 M0 32 C0.42 10.69, 12.07 0.66, 32 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1866.8332066611645 2165.4642255268404) rotate(0 184 37)"><text x="184" y="17.666666666666668" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">wait任意一个子进程退出,当pid为ctr_pi</text><text x="184" y="42.333333333333336" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">d时ret为0且exit_shim = true:</text><text x="184" y="67" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">reap_container(p->ctr_pid, &status);</text></g><g stroke-linecap="round"><g transform="translate(2029.5553949749662 2268.186439271957) rotate(0 -0.9763343468072776 36.133775445274296)"><path d="M1.02 0.46 C0.57 12.52, -1.65 59.38, -2.6 71.39 M0.09 -0.35 C-0.42 11.95, -2.57 60.83, -2.97 72.62" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2029.5553949749662 2268.186439271957) rotate(0 -0.9763343468072776 36.133775445274296)"><path d="M-11.34 45.87 C-10.5 51.1, -5.88 62.76, -3.27 72.74 M-12.87 44.69 C-9 53.41, -6.95 62.23, -2.53 72.75" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2029.5553949749662 2268.186439271957) rotate(0 -0.9763343468072776 36.133775445274296)"><path d="M9.16 46.68 C3.71 51.84, 2.02 63.25, -3.27 72.74 M7.64 45.5 C4.96 53.79, 0.47 62.36, -2.53 72.75" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(1888.444249955435 2341.519772605291) rotate(0 140.55557250976562 47.77778625488281)"><path d="M176.25 12 M176.25 12 C193.33 17, 215 25.1, 245.86 36 M176.25 12 C191.74 17.07, 207.83 22.2, 245.86 36 M245.86 36 C280.78 48.46, 281.6 48.34, 245.86 60 M245.86 36 C281.49 48.53, 280.44 49.39, 245.86 60 M245.86 60 C218.93 69.88, 192.32 77.33, 176.25 83.56 M245.86 60 C228.6 65.67, 209.64 72, 176.25 83.56 M176.25 83.56 C140.74 95.38, 139.86 97.51, 105.75 83.56 M176.25 83.56 C139.86 96.87, 139.06 95.01, 105.75 83.56 M105.75 83.56 C79.89 73.82, 52.63 64.26, 35.25 60 M105.75 83.56 C83.25 76.43, 60.26 67.56, 35.25 60 M35.25 60 C0.92 48.68, -1.13 47.68, 35.25 36 M35.25 60 C-2.21 47.48, -0.43 48.15, 35.25 36 M35.25 36 C57.94 30.61, 80.85 20.57, 105.75 12 M35.25 36 C57.94 29.45, 79.04 20.89, 105.75 12 M105.75 12 C139.35 -1.41, 141.49 0.65, 176.25 12 M105.75 12 C141.02 -0.33, 141.05 -0.86, 176.25 12" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1961.4998224652006 2377.047558860174) rotate(0 67.5 12.5)"><text x="67.5" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">if ret == 0? </text></g><g mask="url(#mask-bbnZhFWS9Rbhohk7eLPeY)" stroke-linecap="round"><g transform="translate(1897.466340026356 2381.852184694928) rotate(0 -34.41813403104885 -93.0313740481838)"><path d="M0.2 0.08 C-10.87 -12.39, -59.16 -43.9, -66.72 -74.93 C-74.29 -105.97, -48.87 -167.58, -45.19 -186.14 M-1.16 -0.93 C-12.34 -13.19, -60.11 -42.81, -67.63 -73.48 C-75.15 -104.15, -50.14 -166.3, -46.28 -184.95" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1897.466340026356 2381.852184694928) rotate(0 -34.41813403104885 -93.0313740481838)"><path d="M-44.2 -154.89 C-45.08 -164.17, -45.77 -171.42, -46.94 -186.93 M-44.76 -155.82 C-45.45 -161.76, -46.23 -169.89, -45.51 -184.43" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1897.466340026356 2381.852184694928) rotate(0 -34.41813403104885 -93.0313740481838)"><path d="M-63.9 -160.64 C-59.44 -168.39, -54.78 -174.08, -46.94 -186.93 M-64.46 -161.57 C-60.23 -165.91, -56.1 -172.61, -45.51 -184.43" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask id="mask-bbnZhFWS9Rbhohk7eLPeY"><rect x="0" y="0" fill="#fff" width="2064.2661400582147" height="2667.7401692943304"></rect><rect x="1819.6665399944973" y="2294.5753451150563" fill="#000" width="22" height="25" opacity="1"></rect></mask><g transform="translate(1819.6665399944973 2294.575345115057) rotate(0 43.381666000809616 -5.754534468312613)"><text x="11" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">no</text></g><g mask="url(#mask-q9Nk9d9pLVlV43pZNPiRv)" stroke-linecap="round"><g transform="translate(2026.7181602034523 2440.5773127517105) rotate(0 1.2120333094220541 52.872518951391385)"><path d="M0.17 0.9 C0.96 18.6, 2.82 88.27, 3.63 105.42 M-1.2 0.32 C-0.51 17.79, 1.51 86.47, 2.51 103.9" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2026.7181602034523 2440.5773127517105) rotate(0 1.2120333094220541 52.872518951391385)"><path d="M-7.41 75.63 C-4.8 83.7, -5.08 86.86, 3.97 104.81 M-9 75.73 C-4.66 85.37, -1.43 95.37, 3.4 103.94" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2026.7181602034523 2440.5773127517105) rotate(0 1.2120333094220541 52.872518951391385)"><path d="M13.09 74.77 C11.43 83.05, 6.87 86.4, 3.97 104.81 M11.5 74.88 C8.98 84.69, 5.36 94.97, 3.4 103.94" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask id="mask-q9Nk9d9pLVlV43pZNPiRv"><rect x="0" y="0" fill="#fff" width="2130.666438269237" height="2644.853105938624"></rect><rect x="2012.1922992363448" y="2480.2152093451673" fill="#000" width="33" height="25" opacity="1"></rect></mask><g transform="translate(2012.192299236345 2480.215209345167) rotate(0 15.737894276529573 13.23462235793454)"><text x="16.5" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">yes</text></g><g stroke-linecap="round" transform="translate(1892.3331557985337 2545.96417466421) rotate(0 140.55557250976562 47.77778625488281)"><path d="M176.25 12 M176.25 12 C204.04 23.11, 232.03 29.14, 245.86 36 M176.25 12 C199.54 18.87, 223.71 27.48, 245.86 36 M245.86 36 C280.09 47.67, 282.44 47.27, 245.86 60 M245.86 36 C280.24 46.84, 282.12 45.92, 245.86 60 M245.86 60 C230.3 64.15, 209.69 71.42, 176.25 83.56 M245.86 60 C222.81 68.11, 198.5 75.76, 176.25 83.56 M176.25 83.56 C139.26 96.33, 141.38 96.77, 105.75 83.56 M176.25 83.56 C142.77 94.89, 141.24 93.41, 105.75 83.56 M105.75 83.56 C87.93 79.17, 71.54 71.72, 35.25 60 M105.75 83.56 C84.93 77.52, 63.49 71.04, 35.25 60 M35.25 60 C-1.54 49.35, 1.87 47.69, 35.25 36 M35.25 60 C-2.12 46.06, 1.3 47.16, 35.25 36 M35.25 36 C63.87 28.47, 90.03 17.54, 105.75 12 M35.25 36 C57.39 28.4, 79.81 21.55, 105.75 12 M105.75 12 C141.54 -1.08, 141.96 0.42, 176.25 12 M105.75 12 C141.17 -1.26, 139.01 1.95, 176.25 12" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1963.8887283082993 2581.491960919093) rotate(0 69 12.5)"><text x="69" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">if exit_shim? </text></g><g mask="url(#mask-Q11p3sVE8mw_vslY2Q6X2)" stroke-linecap="round"><g transform="translate(2035.1109166221015 2635.9642000955255) rotate(0 4.166440780727612 42.07792280411286)"><path d="M-0.94 0.99 C0.22 14.87, 5.6 68.58, 7.11 82.28 M0.76 0.46 C2.27 14.62, 8.18 70.25, 9.28 83.7" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2035.1109166221015 2635.9642000955255) rotate(0 4.166440780727612 42.07792280411286)"><path d="M-1.99 54.88 C-0.2 65.15, 5.31 75.31, 9.79 84.48 M-2.96 57.55 C0.04 63.29, 2.92 72.28, 8.91 84.46" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2035.1109166221015 2635.9642000955255) rotate(0 4.166440780727612 42.07792280411286)"><path d="M18.44 52.92 C13.8 63.65, 12.91 74.42, 9.79 84.48 M17.47 55.59 C14.82 61.72, 12.05 71.25, 8.91 84.46" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask id="mask-Q11p3sVE8mw_vslY2Q6X2"><rect x="0" y="0" fill="#fff" width="2143.265411306974" height="2818.474552066577"></rect><rect x="2022.6881639645376" y="2664.719376081051" fill="#000" width="33" height="25" opacity="1"></rect></mask><g transform="translate(2022.6881639645378 2664.7193760810505) rotate(0 16.589193438291545 13.322746818587348)"><text x="16.5" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">yes</text></g><g stroke-linecap="round" transform="translate(1921.7775832887685 2727.0753451150567) rotate(0 115 67.5)"><path d="M32 0 M32 0 C94.47 -0.36, 160.99 -3.31, 198 0 M32 0 C94.64 -1.76, 157.03 -2.48, 198 0 M198 0 C219.99 -0.18, 230.92 11.96, 230 32 M198 0 C217.23 -1.6, 230.3 9.5, 230 32 M230 32 C229.18 57.28, 230.44 84.72, 230 103 M230 32 C229.99 52.98, 229.62 74.1, 230 103 M230 103 C229.14 126.17, 218.57 133.55, 198 135 M230 103 C229.46 122.27, 217.95 135.89, 198 135 M198 135 C137.76 134.09, 75.53 135.21, 32 135 M198 135 C151.97 134.63, 106.29 134.24, 32 135 M32 135 C10.21 135.74, 0.39 125.73, 0 103 M32 135 C12.85 135.12, -1.4 126.08, 0 103 M0 103 C1.14 78.39, -0.68 56.66, 0 32 M0 103 C1 78.43, 0.72 54.21, 0 32 M0 32 C-1.55 8.75, 9.5 1.99, 32 0 M0 32 C-0.91 11.06, 12.21 0.52, 32 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1928.7775832887685 2757.5753451150567) rotate(0 108 37)"><text x="108" y="17.666666666666668" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">调用runc </text><text x="108" y="42.333333333333336" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">kill命令杀死容器进程:p</text><text x="108" y="67" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">rocess_kill_all(p);</text></g><g stroke-linecap="round"><g transform="translate(2040.6665399944975 2863.7419863504083) rotate(0 0.33257934987545923 35.79963335288062)"><path d="M-0.61 0.82 C-0.6 12.5, 0.5 58.46, 0.68 69.9 M1.27 0.21 C1.04 12.1, -0.17 59.67, -0.5 71.39" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2040.6665399944975 2863.7419863504083) rotate(0 0.33257934987545923 35.79963335288062)"><path d="M-8.64 42.22 C-6.88 51.84, -4.28 61.9, -2.05 69.75 M-9.37 42.31 C-8.12 48.13, -6.11 55.47, -0.34 72.38" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2040.6665399944975 2863.7419863504083) rotate(0 0.33257934987545923 35.79963335288062)"><path d="M11.88 42.76 C6.44 52.09, 1.85 61.96, -2.05 69.75 M11.15 42.85 C8.21 48.42, 6.03 55.64, -0.34 72.38" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(1907.3332066611645 2935.9642000955255) rotate(0 135 54.5)"><path d="M27.25 0 M27.25 0 C78.11 2.77, 129.68 3.01, 242.75 0 M27.25 0 C93.49 -1.26, 159.48 -2.48, 242.75 0 M242.75 0 C259.6 -0.19, 269.97 10.27, 270 27.25 M242.75 0 C259.89 1.4, 270.94 10.11, 270 27.25 M270 27.25 C271.67 46.31, 269.1 64.08, 270 81.75 M270 27.25 C271.06 48.36, 269.92 71.3, 270 81.75 M270 81.75 C269.99 100.12, 262.12 110.46, 242.75 109 M270 81.75 C267.81 98.45, 259.25 110.71, 242.75 109 M242.75 109 C169.98 109.38, 94.08 111.7, 27.25 109 M242.75 109 C187.65 111.21, 132.79 109.97, 27.25 109 M27.25 109 C10.49 108.36, -1.99 100.21, 0 81.75 M27.25 109 C9.2 108.8, -0.25 99.82, 0 81.75 M0 81.75 C-0.04 61.68, -1.52 45.24, 0 27.25 M0 81.75 C0.05 67.22, 1.29 50.18, 0 27.25 M0 27.25 C-1.31 10.64, 7.33 -1.72, 27.25 0 M0 27.25 C-1.44 7.68, 8.6 -0.7, 27.25 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1916.2776341513988 2940.9642000955255) rotate(0 125.5 49.5)"><text x="125.5" y="17.75" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">等待所有子进程被杀死:DO</text><text x="125.5" y="42.5" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">_RETRY_CALL(120, </text><text x="125.5" y="67.25" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">1000000, nret, </text><text x="125.5" y="92" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">try_wait_all_child);</text></g><g stroke-linecap="round"><g transform="translate(2044.9841797384308 3047.949385280711) rotate(0 -0.5612235787022541 28.518684299644292)"><path d="M0.67 -0.26 C0.32 9.33, -1.51 47.53, -1.8 56.88 M-0.43 -1.45 C-0.46 8.42, 0.63 48.8, 0.37 58.48" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2044.9841797384308 3047.949385280711) rotate(0 -0.5612235787022541 28.518684299644292)"><path d="M-9.88 30.12 C-6.15 39.99, -4.72 47.52, 2.09 60.27 M-10.16 31.14 C-4.99 41.36, -2.8 49.78, 1.17 58.89" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2044.9841797384308 3047.949385280711) rotate(0 -0.5612235787022541 28.518684299644292)"><path d="M9.53 30.04 C6.9 40.05, 1.97 47.61, 2.09 60.27 M9.25 31.07 C7.64 41.49, 3.06 49.95, 1.17 58.89" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(1898.444249955435 3106.7974825662286) rotate(0 163 46.5)"><path d="M23.25 0 M23.25 0 C118.09 -0.68, 214.6 -0.03, 302.75 0 M23.25 0 C103.49 1.88, 184.62 1.25, 302.75 0 M302.75 0 C316.99 -0.37, 327.3 7.73, 326 23.25 M302.75 0 C320.15 -0.93, 323.79 5.54, 326 23.25 M326 23.25 C324.81 36.65, 324.85 47.88, 326 69.75 M326 23.25 C326.76 38.81, 326.32 53.96, 326 69.75 M326 69.75 C326.38 84.9, 319.84 92.05, 302.75 93 M326 69.75 C326.62 87.28, 319.28 93.09, 302.75 93 M302.75 93 C212.09 93.12, 119.16 93.38, 23.25 93 M302.75 93 C216.2 93.4, 128.99 93.28, 23.25 93 M23.25 93 C6.26 94.63, -1.14 84.77, 0 69.75 M23.25 93 C7.49 91.36, -2.02 87.12, 0 69.75 M0 69.75 C1.05 57.24, 1.86 43.45, 0 23.25 M0 69.75 C0.6 53.55, 0.41 38.91, 0 23.25 M0 23.25 C-1.55 9.68, 8.39 -0.39, 23.25 0 M0 23.25 C-1.82 9.4, 7.78 2.05, 23.25 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1909.944249955435 3116.2974825662286) rotate(0 151.5 37)"><text x="151.5" y="17.666666666666668" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">调用runc </text><text x="151.5" y="42.333333333333336" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">delete命令删除容器进程:proces</text><text x="151.5" y="67" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">s_delete(p)</text></g><g stroke-linecap="round"><g transform="translate(2229.0704177187117 3102.36801325136) rotate(0 10.417241149896654 -225.7285473325444)"><path d="M1.2 0.47 C4.16 -74.8, 15.4 -376.65, 18.38 -451.93 M0.36 -0.33 C3.62 -75.35, 17.52 -374.83, 20.47 -450.29" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2229.0704177187117 3102.36801325136) rotate(0 10.417241149896654 -225.7285473325444)"><path d="M30.28 -423.66 C26.71 -432.48, 24.74 -442.03, 22.12 -448.83 M28.65 -421.75 C26.16 -431.44, 23.19 -442.6, 20.46 -450.58" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2229.0704177187117 3102.36801325136) rotate(0 10.417241149896654 -225.7285473325444)"><path d="M9.78 -424.55 C13.42 -433.25, 18.66 -442.5, 22.12 -448.83 M8.15 -422.63 C13.07 -431.95, 17.53 -442.8, 20.46 -450.58" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(2241.5633557032024 2591.3611114744153) rotate(0 114.7619192940848 54.28571973528187)"><path d="M143.75 13.75 M143.75 13.75 C162.3 24.36, 183.85 33.94, 200.77 41.25 M143.75 13.75 C158.2 20.6, 172.56 26.85, 200.77 41.25 M200.77 41.25 C230.86 55.47, 228.06 56.12, 200.77 68.75 M200.77 41.25 C227.51 55.32, 231.58 56.75, 200.77 68.75 M200.77 68.75 C190.4 75.39, 177.68 81.14, 143.75 94.82 M200.77 68.75 C180.85 77.35, 159.54 87.17, 143.75 94.82 M143.75 94.82 C115.16 109.68, 115.98 109.6, 86.25 94.82 M143.75 94.82 C114.05 109.54, 112.82 108.29, 86.25 94.82 M86.25 94.82 C70.29 87.75, 50.85 76.89, 28.75 68.75 M86.25 94.82 C68.27 86.62, 50.46 78.44, 28.75 68.75 M28.75 68.75 C-0.96 54.01, 0.45 53.17, 28.75 41.25 M28.75 68.75 C1.97 52.97, 1.46 57.28, 28.75 41.25 M28.75 41.25 C52.12 31.21, 72.76 21.15, 86.25 13.75 M28.75 41.25 C47.79 32.26, 64.89 23.98, 86.25 13.75 M86.25 13.75 C116.74 -0.99, 115.69 -0.16, 143.75 13.75 M86.25 13.75 C114.13 -0.55, 117.29 -1.91, 143.75 13.75" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2269.825274997287 2633.3968312096977) rotate(0 86.5 12.5)"><text x="86.5" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">if p->exit_fd > 0</text></g><g mask="url(#mask-ULiYVEpD-N41kRq5ttn44)" stroke-linecap="round"><g transform="translate(2355.849069988917 2698.980140928238) rotate(0 3.476666484196187 50.32856572500259)"><path d="M-0.23 0.53 C1.05 17.24, 7.2 82.98, 8.77 99.61 M-1.81 -0.23 C-0.75 16.69, 6.23 83.9, 7.99 100.89" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2355.849069988917 2698.980140928238) rotate(0 3.476666484196187 50.32856572500259)"><path d="M-4.22 74.43 C-0.97 85.57, 5.26 95.46, 7.29 102.71 M-4.24 73.4 C-1.52 80.62, 1.06 87.26, 8.98 100.06" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2355.849069988917 2698.980140928238) rotate(0 3.476666484196187 50.32856572500259)"><path d="M16.19 72.33 C11.5 84.24, 9.8 94.95, 7.29 102.71 M16.17 71.3 C13.75 79.04, 11.19 86.21, 8.98 100.06" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask id="mask-ULiYVEpD-N41kRq5ttn44"><rect x="0" y="0" fill="#fff" width="2464.2957915019097" height="2899.0729332213846"></rect><rect x="2343.5724307454134" y="2736.5265370748116" fill="#000" width="33" height="25" opacity="1"></rect></mask><g transform="translate(2343.5724307454134 2736.526537074811) rotate(0 15.753305727699797 12.782169578429148)"><text x="16.5" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">yes</text></g><g stroke-linecap="round" transform="translate(2272.991839938694 2805.6468366592644) rotate(0 106 42)"><path d="M21 0 M21 0 C72.04 2.18, 126.5 -1.4, 191 0 M21 0 C64.08 0.79, 109.11 0.62, 191 0 M191 0 C204.76 0.07, 213.68 6.47, 212 21 M191 0 C203.67 1.07, 210.55 7.43, 212 21 M212 21 C211.01 32.39, 212.55 45.97, 212 63 M212 21 C211.56 37.58, 211.02 53.6, 212 63 M212 63 C213.16 78.03, 205.47 82.67, 191 84 M212 63 C210.42 75.33, 202.86 83.08, 191 84 M191 84 C144.19 84.29, 96.17 84.05, 21 84 M191 84 C129.01 85.58, 68.22 85.03, 21 84 M21 84 C5.82 83.95, -0.57 76.41, 0 63 M21 84 C8.23 86.17, -1.9 77.57, 0 63 M0 63 C0.86 49.54, -0.53 31.99, 0 21 M0 63 C-0.34 49.59, 0.29 33.65, 0 21 M0 21 C1.49 7.2, 7.24 -1.51, 21 0 M0 21 C-1.73 5.3, 4.7 2.23, 21 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2282.491839938694 2810.6468366592644) rotate(0 96.5 37)"><text x="96.5" y="17.666666666666668" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">write_nointr(p->exit</text><text x="96.5" y="42.333333333333336" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">_fd, &status, </text><text x="96.5" y="67" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">sizeof(int));</text></g><g stroke-linecap="round"><g transform="translate(2373.9442499554348 2891.3611223735506) rotate(0 -0.6950331233085763 32.5151832380252)"><path d="M0.05 0.87 C-0.05 11.73, 0.09 54.16, -0.23 64.75 M-1.38 0.28 C-1.61 10.86, -1.14 52.79, -0.82 63.15" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2373.9442499554348 2891.3611223735506) rotate(0 -0.6950331233085763 32.5151832380252)"><path d="M-10.17 36.29 C-6.49 42.89, -3.43 54.22, -0.96 64.08 M-11.52 36.13 C-8.61 43.65, -5.81 52.89, -1.71 62.59" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2373.9442499554348 2891.3611223735506) rotate(0 -0.6950331233085763 32.5151832380252)"><path d="M10.35 35.9 C7.22 42.64, 3.47 54.1, -0.96 64.08 M9 35.74 C5.56 43.41, 2.02 52.78, -1.71 62.59" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(2277.7538028293184 2956.122998071095) rotate(0 103.33330426897328 42)"><path d="M21 0 M21 0 C56.33 -0.1, 93.27 0.94, 185.67 0 M21 0 C85.31 0.4, 150.81 0.61, 185.67 0 M185.67 0 C198.9 -0.69, 208.54 7.55, 206.67 21 M185.67 0 C199.5 -1.39, 205.54 7.99, 206.67 21 M206.67 21 C207.66 37.63, 208.54 52.79, 206.67 63 M206.67 21 C207.75 37.87, 207.27 53.43, 206.67 63 M206.67 63 C204.97 78.83, 197.86 84.16, 185.67 84 M206.67 63 C206.36 77.9, 198.26 83.26, 185.67 84 M185.67 84 C144.07 82.28, 99.36 81.13, 21 84 M185.67 84 C137.56 82.8, 90.18 83.46, 21 84 M21 84 C6.2 85.43, 1.12 78.06, 0 63 M21 84 C7.86 84.93, -0.13 77.26, 0 63 M0 63 C-0.85 53.53, 1.11 45.27, 0 21 M0 63 C0.45 47.7, -0.39 34.86, 0 21 M0 21 C0.52 5.89, 6.08 0.3, 21 0 M0 21 C-1.75 6.84, 7.45 -2.2, 21 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2285.087107098292 2961.122998071095) rotate(0 96 37)"><text x="96" y="17.666666666666668" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">pthread_timedjoin_</text><text x="96" y="42.333333333333336" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">np(tid_accept, </text><text x="96" y="67" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">NULL, &ts);</text></g><g mask="url(#mask-sqouWLgGXTuSdkOGDLHVp)" stroke-linecap="round"><g transform="translate(2462.5156785268637 2655.1706316508944) rotate(0 28.557515681428868 174.58061438593631)"><path d="M0.18 -0.91 C9.96 29.06, 54.26 121.01, 58.01 179.28 C61.76 237.55, 28.23 320.43, 22.66 348.69 M-1.19 1.22 C8.49 30.86, 52.66 119.14, 57.02 177.28 C61.38 235.42, 30.25 321.27, 24.96 350.07" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2462.5156785268637 2655.1706316508944) rotate(0 28.557515681428868 174.58061438593631)"><path d="M20.55 320.07 C23.03 325.6, 23.55 336.15, 25.41 349.4 M21.47 319.46 C22.17 326.09, 23.16 331.97, 25.55 349.69" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2462.5156785268637 2655.1706316508944) rotate(0 28.557515681428868 174.58061438593631)"><path d="M40.42 325.2 C38.06 329.51, 33.73 338.8, 25.41 349.4 M41.34 324.59 C37.79 330, 34.55 334.8, 25.55 349.69" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask id="mask-sqouWLgGXTuSdkOGDLHVp"><rect x="0" y="0" fill="#fff" width="2620.6109456864615" height="3103.7420602223233"></rect><rect x="2509.6109456864615" y="2821.718265230694" fill="#000" width="22" height="25" opacity="1"></rect></mask><g transform="translate(2509.6109456864615 2821.718265230694) rotate(0 -18.537751478168957 8.032980806136948)"><text x="11" y="18" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">no</text></g><g stroke-linecap="round"><g transform="translate(2376.8013928125774 3045.6468366592644) rotate(0 -0.5525397194394941 30.143871727979786)"><path d="M0.14 -0.2 C0.12 10.07, 0.23 50.26, 0.01 60.73 M-1.25 -1.36 C-1.35 9.1, -0.3 51.31, -0.43 61.64" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2376.8013928125774 3045.6468366592644) rotate(0 -0.5525397194394941 30.143871727979786)"><path d="M-11.27 35.15 C-9.87 41.88, -3.5 52.45, 0.93 63.3 M-10.76 33.42 C-9.02 41.53, -5.95 48.09, -1.38 61.9" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2376.8013928125774 3045.6468366592644) rotate(0 -0.5525397194394941 30.143871727979786)"><path d="M9.25 34.97 C4.32 41.78, 4.36 52.41, 0.93 63.3 M9.76 33.25 C6.1 41.54, 3.77 48.15, -1.38 61.9" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(2292.991927131774 3114.218178037613) rotate(0 97.6190185546875 38.09526715959828)"><path d="M19.05 0 M19.05 0 C76.11 -2.14, 128.59 -0.54, 176.19 0 M19.05 0 C68.78 -1.26, 119.14 0, 176.19 0 M176.19 0 C188.47 -0.48, 194.87 5.26, 195.24 19.05 M176.19 0 C189.16 1.85, 193.13 7.35, 195.24 19.05 M195.24 19.05 C195.18 33.24, 193.92 45.11, 195.24 57.14 M195.24 19.05 C195.79 32.31, 194.04 46.14, 195.24 57.14 M195.24 57.14 C196.98 69.33, 189.56 76.17, 176.19 76.19 M195.24 57.14 C194.29 67.69, 190.01 74, 176.19 76.19 M176.19 76.19 C139.22 76.41, 100.94 77.9, 19.05 76.19 M176.19 76.19 C119.19 76.26, 63.44 75.9, 19.05 76.19 M19.05 76.19 C8.25 75.49, 1.97 70.05, 0 57.14 M19.05 76.19 C4.87 74.28, 0.83 71.93, 0 57.14 M0 57.14 C0.62 44.72, -1.14 27.47, 0 19.05 M0 57.14 C-0.81 44.69, -0.92 30.35, 0 19.05 M0 19.05 C-1.77 8.01, 7.33 0.13, 19.05 0 M0 19.05 C-1.29 4.39, 7.88 -0.81, 19.05 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2302.1109456864615 3127.813445197211) rotate(0 88.5 24.5)"><text x="88.5" y="17.5" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">destroy_io_threa</text><text x="88.5" y="42" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr">d(p, i);</text></g><g stroke-linecap="round"><g transform="translate(655.3958679683622 61.54343801831442) rotate(0 309.0822854324355 -4.1853779366408475)"><path d="M0.63 -0.15 C103.85 -1.49, 515.51 -7.12, 618.66 -8.22 M-0.5 -1.27 C102.57 -2.49, 514.19 -6, 617.43 -7.28" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(655.3958679683622 61.54343801831442) rotate(0 309.0822854324355 -4.1853779366408475)"><path d="M589.1 2.37 C594.44 1.03, 603.84 -1.12, 617.08 -5.39 M588.76 4.17 C595.3 0.59, 600.41 -1.63, 616.73 -6.88" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(655.3958679683622 61.54343801831442) rotate(0 309.0822854324355 -4.1853779366408475)"><path d="M588.89 -18.15 C594.25 -14.4, 603.71 -11.47, 617.08 -5.39 M588.55 -16.35 C595.33 -15.71, 600.49 -13.72, 616.73 -6.88" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask></svg> +\ No newline at end of file +diff --git a/docs/images/k8s_isulad_flow_chart.svg b/docs/images/k8s_isulad_flow_chart.svg +new file mode 100644 +index 00000000..700b5b6d +--- /dev/null ++++ b/docs/images/k8s_isulad_flow_chart.svg +@@ -0,0 +1,16 @@ ++<svg version="1.1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 3462.3616612952683 2251.06399129264" width="3462.3616612952683" height="2251.06399129264"> ++ <!-- svg-source:excalidraw --> ++ ++ <defs> ++ <style class="style-fonts"> ++ @font-face { ++ font-family: "Virgil"; ++ src: url("https://excalidraw.com/Virgil.woff2"); ++ } ++ @font-face { ++ font-family: "Cascadia"; ++ src: url("https://excalidraw.com/Cascadia.woff2"); ++ } ++ </style> ++ </defs> ++ <rect x="0" y="0" width="3462.3616612952683" height="2251.06399129264" fill="#ffffff"></rect><g stroke-linecap="round" transform="translate(10 426.72821970161084) rotate(0 114.66664632161451 28.444442749023438)"><path d="M14.22 0 M14.22 0 C79.83 -1.12, 143.72 1.4, 215.11 0 M14.22 0 C77.56 -0.01, 143.17 0.24, 215.11 0 M215.11 0 C222.69 -0.84, 227.86 3.71, 229.33 14.22 M215.11 0 C223.07 -1.37, 229.04 3.28, 229.33 14.22 M229.33 14.22 C228.93 22.26, 229.24 31.61, 229.33 42.67 M229.33 14.22 C228.2 22.06, 228.6 30.74, 229.33 42.67 M229.33 42.67 C228.13 52.02, 226.01 57.31, 215.11 56.89 M229.33 42.67 C229.34 51.47, 225.64 54.98, 215.11 56.89 M215.11 56.89 C163.47 53.75, 111.67 56.61, 14.22 56.89 M215.11 56.89 C142.48 58.69, 70.15 59.07, 14.22 56.89 M14.22 56.89 C3.3 56.27, -0.37 52.87, 0 42.67 M14.22 56.89 C2.98 56.38, 1.33 50.08, 0 42.67 M0 42.67 C1.38 37.02, -0.48 28.91, 0 14.22 M0 42.67 C0.19 33.27, -0.48 21.49, 0 14.22 M0 14.22 C-1.79 3.31, 4.07 1.46, 14.22 0 M0 14.22 C-1.81 3.23, 4.71 0.84, 14.22 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(89.81668599446584 443.1726624506343) rotate(0 34.84996032714844 12)"><text x="34.84996032714844" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">kubelet</text></g><g stroke-linecap="round" transform="translate(45.64646402994822 289.77870671900155) rotate(0 88.5 30)"><path d="M-0.17 27.68 C-0.17 27.68, -0.17 27.68, -0.17 27.68 M-0.17 27.68 C-0.17 27.68, -0.17 27.68, -0.17 27.68 M1.49 30.75 C8.04 27.97, 8.62 21.39, 17.32 14.82 M2.17 30.3 C5.32 25.8, 9.86 22.15, 16.05 14.84 M3.7 33.81 C7.62 27.65, 13.9 21.78, 23.55 11.17 M4.57 36.1 C10 29.05, 14.1 21.72, 23.86 12.79 M5.78 40.8 C11.64 32.98, 20.16 23.51, 30.81 9.5 M5.4 38.34 C13.66 31.12, 19.56 22.21, 29.98 9.66 M10.08 43.49 C19.08 28.74, 31.27 18.28, 38.37 8.85 M8.15 42.88 C17.24 33.01, 25.22 22.23, 37.98 7.67 M12.19 45.4 C22.73 33.84, 30.89 21.63, 48.26 4.38 M12.02 44.34 C21.59 33.29, 32.01 22.67, 46.35 5.41 M16.75 47.66 C21.54 37.62, 32.64 27.04, 53.16 5.76 M15.94 44.72 C26.45 34.09, 36.55 24.37, 52.61 5.15 M20.03 48.39 C26.48 38.09, 35.5 30.52, 59.52 2.72 M19.33 47.63 C32.89 33.26, 43.22 19.83, 58.53 3.11 M23.1 47.55 C32.91 36.21, 45.43 22.35, 64.83 3.02 M23.46 49.98 C34.26 37.53, 45.29 24.27, 63.73 4.28 M26.41 50.37 C39.19 35.44, 55.17 22.36, 70.16 1.51 M28.2 50.12 C36.49 39.27, 44.32 30.4, 68.3 2.3 M33 50.42 C40 42, 50.15 30.4, 74.04 4.3 M31.44 51.86 C39.82 42.76, 49.46 32.67, 74.88 2.41 M34.33 53.29 C48.04 37.86, 58.88 25.26, 78.32 0.84 M35.08 54.82 C51.01 37.23, 66.64 18.93, 80.74 2.4 M38.19 54.88 C58.16 33.77, 76.23 14.32, 86.04 0.08 M39.17 55.66 C51.16 41.6, 63.44 27.58, 85.83 1.51 M45.91 54.37 C56.94 40.46, 72.43 24.25, 90.68 3.35 M45.26 55.79 C55.53 43.8, 66.23 30.08, 90.2 2.17 M49.57 55.04 C62.04 39.93, 78.46 21.62, 96.61 1.43 M47.85 57.01 C62.35 42, 73.35 27.83, 96.64 1.88 M53.19 56.68 C66.64 44.97, 77.61 28.82, 100.42 4.01 M53.69 58.73 C72.17 36.28, 89.64 16.95, 102.23 1.46 M55.94 58.66 C77.16 38.94, 93.12 18.3, 105.35 2.47 M57.28 59.89 C72.08 42.29, 87.36 23.27, 106.91 2.68 M63.35 58.59 C75.17 42.18, 91.07 26.89, 112.59 1.09 M62.16 60.41 C73.77 47.67, 83.02 35.02, 110.99 2.24 M66.57 59.48 C81.85 43.91, 98.54 23.54, 115.74 4.37 M66.45 61.13 C84.6 40, 102.96 20.29, 115.81 3.91 M72.8 60.53 C84.53 44.07, 102.22 28.07, 122.74 5.18 M71.41 61.47 C90.19 40.01, 109.41 18.36, 121.84 3.53 M75.27 61.84 C90.4 48.02, 101.31 34.1, 126.4 4.12 M77.63 60.89 C87.76 49.34, 99.22 35.06, 127.41 3.14 M83.09 60.49 C92.96 49.43, 101.25 36.54, 130.31 6.18 M83.04 61.84 C93.39 48.48, 103.14 36.3, 130.77 4.47 M87.06 62.15 C103.74 42.92, 116.69 26.57, 135.26 7.05 M86.98 60.18 C105.44 40.57, 122.48 19.7, 134.75 6.08 M92.98 58.52 C106.42 42.92, 123.81 25.56, 140.57 7.06 M93.59 59.35 C105.07 47.08, 117.35 33.23, 140.28 8.05 M97.46 61.13 C116.16 38.17, 133.07 17.87, 141.94 6.76 M97.91 61.11 C114.18 42.67, 127.88 25.6, 144.36 8.48 M103.79 59.25 C117 47.62, 128.74 30.19, 148.63 7.83 M103.86 60.16 C121.56 40.69, 138.65 20.31, 147.48 9.09 M107.19 61.29 C121.69 48.27, 131.87 37.24, 153.35 9.17 M109.74 60.38 C123.28 45.56, 135.6 30.2, 151.7 10.22 M113.31 61.61 C130.88 40.71, 145.63 22.84, 157.48 11.83 M114.86 60.51 C125.37 47.42, 137.16 34.19, 156.04 12.51 M118.79 58.21 C134.54 44.39, 146.93 29.83, 159.57 12.05 M120.33 60.53 C135.49 43.09, 148.59 28.66, 161.17 12.78 M126.48 55.83 C136.41 43.24, 148.71 33.52, 163.96 15.47 M126.45 57.71 C138.2 44.84, 149.25 33, 164.22 17.06 M132.66 56.55 C146.27 43.16, 155.61 28.81, 165.88 16.94 M134.12 55.94 C145.71 42.12, 156.72 29.31, 168.07 17.2 M140.19 56.21 C150.26 45.03, 157.92 35.46, 170.1 20.95 M140.16 54.97 C147.58 46.13, 155.26 38.08, 170.17 20.11 M147.11 54.52 C155.04 43.13, 166.63 34.33, 174.05 23.71 M146.16 53.35 C154.35 44.24, 163.57 34.23, 173.89 22.65 M153.15 53.9 C162.76 43.68, 168.95 33.41, 175.09 24.31 M153.08 52.07 C158.67 45.77, 163.26 40.07, 175.94 25.6 M161.57 47.64 C164.42 44.01, 167.59 42.95, 177.1 32.57 M161.36 47.56 C165.86 43.77, 167.86 41.12, 175.13 32.38" stroke="#fa5252" stroke-width="0.5" fill="none"></path><path d="M125.42 2.19 C137.81 3.67, 150.63 8.43, 158.89 12.16 C167.15 15.89, 172.59 19.89, 174.98 24.59 C177.36 29.29, 177.21 36.01, 173.18 40.38 C169.14 44.75, 160.89 47.62, 150.78 50.82 C140.66 54.01, 125.77 57.95, 112.48 59.53 C99.18 61.11, 84.25 61.45, 71.02 60.3 C57.78 59.15, 43.77 55.85, 33.08 52.63 C22.38 49.41, 12.39 45.15, 6.86 40.98 C1.32 36.81, -1.22 32.24, -0.11 27.61 C1 22.97, 6.1 17.24, 13.5 13.16 C20.9 9.08, 32.44 5.28, 44.28 3.14 C56.13 1.01, 69.57 0.03, 84.57 0.36 C99.58 0.7, 124.87 3.97, 134.31 5.15 C143.75 6.32, 142.01 7.06, 141.2 7.42 M128.18 2.71 C140.47 4.61, 152.92 9.43, 160.8 13.2 C168.69 16.97, 173.57 20.98, 175.48 25.32 C177.38 29.66, 176.81 34.86, 172.21 39.23 C167.6 43.6, 157.78 48.28, 147.86 51.56 C137.95 54.84, 125.7 57.81, 112.74 58.92 C99.79 60.03, 83.73 59.24, 70.14 58.22 C56.54 57.2, 41.96 55.82, 31.17 52.79 C20.38 49.77, 10.49 44.54, 5.38 40.09 C0.26 35.64, -0.93 30.7, 0.48 26.1 C1.89 21.49, 6.26 16.07, 13.83 12.45 C21.4 8.83, 34.08 6.34, 45.9 4.39 C57.73 2.45, 71.17 1.13, 84.77 0.78 C98.38 0.44, 120.51 2.05, 127.53 2.31 C134.56 2.57, 127.49 2, 126.93 2.34" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(108.43753952970428 307.5655032834051) rotate(0 25.629974365234375 12)"><text x="25.629974365234375" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">client</text></g><g stroke-linecap="round"><g transform="translate(135.82823782256173 351.8393160163205) rotate(0 -0.5232778608926765 37.71900260268245)"><path d="M0.14 1.17 C0.24 13.67, 0.18 62.42, 0.14 74.7 M-1.24 0.73 C-1.25 12.9, -0.79 60.82, -0.74 72.85" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(135.82823782256173 351.8393160163205) rotate(0 -0.5232778608926765 37.71900260268245)"><path d="M-9.26 44.96 C-7.65 51.96, -4.59 59.57, -1.68 74.72 M-11.11 44.58 C-7.89 53.35, -3.82 63.71, -0.8 73.65" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(135.82823782256173 351.8393160163205) rotate(0 -0.5232778608926765 37.71900260268245)"><path d="M11.26 44.82 C7.47 51.87, 5.12 59.53, -1.68 74.72 M9.41 44.43 C6.05 53.17, 3.53 63.57, -0.8 73.65" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(391.1009447502379 419.2332643112202) rotate(0 114.66664632161451 28.444442749023438)"><path d="M14.22 0 M14.22 0 C76.48 -1.06, 143.59 -2.67, 215.11 0 M14.22 0 C85.03 0.07, 157.01 -0.29, 215.11 0 M215.11 0 C225.63 -1.33, 231.23 6.23, 229.33 14.22 M215.11 0 C225.86 2.24, 227.39 6.72, 229.33 14.22 M229.33 14.22 C228.4 23.68, 229.22 34.92, 229.33 42.67 M229.33 14.22 C228.64 24.74, 228.87 36.55, 229.33 42.67 M229.33 42.67 C231.19 50.87, 222.96 56.56, 215.11 56.89 M229.33 42.67 C231.25 50.61, 224.29 58.62, 215.11 56.89 M215.11 56.89 C172.73 54.86, 132.68 54.27, 14.22 56.89 M215.11 56.89 C139.23 56.19, 64.82 56.79, 14.22 56.89 M14.22 56.89 C4.56 57.59, 0.54 50.89, 0 42.67 M14.22 56.89 C5.64 58.94, 0.22 50.3, 0 42.67 M0 42.67 C-1.46 36.08, 1.72 31.91, 0 14.22 M0 42.67 C0.97 33.23, 0.76 21.84, 0 14.22 M0 14.22 C0.82 3.06, 3.65 0.78, 14.22 0 M0 14.22 C-1.58 3.15, 3.23 2.02, 14.22 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(477.4776206739034 435.6777070602436) rotate(0 28.28997039794922 12)"><text x="28.28997039794922" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">isulad</text></g><g stroke-linecap="round" transform="translate(415.93419485381173 284.3848010793531) rotate(0 88.5 30)"><path d="M3.68 21.75 C3.68 21.75, 3.68 21.75, 3.68 21.75 M3.68 21.75 C3.68 21.75, 3.68 21.75, 3.68 21.75 M2.94 27.8 C6.81 26.31, 8.75 21.09, 12.97 17.66 M3.37 29.54 C7.57 23.39, 11.2 19.88, 14.51 15.5 M1.13 34.86 C12.26 26.46, 18.18 17.37, 22.99 10.37 M2.47 36.1 C8.56 28.3, 15.33 19.06, 24.55 10.59 M5.92 37.81 C10.54 29.57, 19.14 22.67, 28.91 10.5 M4.73 39.64 C10.54 33.59, 14.06 27.53, 27.31 11.65 M7.83 40.21 C15.83 32.92, 25.45 20.67, 34.03 11.52 M7.71 41.86 C17.42 30.08, 27.58 19.8, 34.1 11.05 M12.09 43.53 C18.06 33.8, 29.86 24.58, 42.35 10.82 M10.7 44.46 C22.11 31.42, 33.93 18.27, 41.45 9.16 M12.59 47.1 C23.7 37.82, 30.69 28.39, 46.66 9 M14.94 46.15 C21.4 38.73, 29.15 28.72, 47.67 8.02 M18.44 48.01 C25.89 39.95, 31.68 29.94, 53.85 7.29 M18.39 49.37 C26.14 38.93, 33.38 29.63, 54.31 5.58 M20.44 51.94 C35.11 34.94, 46.07 20.87, 62.08 4.39 M20.36 49.97 C36.41 33.16, 51.04 15.05, 61.57 3.41 M25.05 49.82 C37.25 35.65, 53.43 19.69, 68.7 2.88 M25.66 50.65 C36.16 39.55, 47.42 26.86, 68.41 3.88 M28.88 53.18 C46.81 31.12, 62.95 11.71, 71.38 1.07 M29.33 53.16 C44.93 35.48, 57.98 19.16, 73.8 2.8 M33.89 52.81 C47.29 40.97, 59.2 23.34, 79.38 0.63 M33.96 53.72 C51.91 33.96, 69.26 13.28, 78.24 1.9 M36.63 55.6 C51.63 42.03, 62.3 30.43, 84.76 1.23 M39.18 54.7 C53.33 39.19, 66.26 23.13, 83.11 2.28 M42.1 56.69 C61.46 33.8, 77.97 13.9, 90.86 1.62 M43.65 55.58 C55.48 41.05, 68.53 26.36, 89.42 2.3 M45.61 55.55 C63.91 38.82, 78.84 21.33, 94.26 0.33 M47.15 57.87 C65.13 37.3, 80.96 19.74, 95.87 1.06 M50.68 56.18 C64.61 39.04, 80.87 24.76, 101.28 0.73 M50.64 58.06 C66.38 40.75, 81.4 24.33, 101.54 2.32 M54.23 59.93 C73.53 40.05, 88.55 19.16, 103.85 1.45 M55.69 59.31 C72.94 38.84, 89.8 19.3, 106.04 1.71 M59.79 61.85 C75.41 43.85, 88.86 27.62, 110.04 3.19 M59.76 60.61 C72.02 46.27, 84.62 32.56, 110.11 2.35 M66.06 60.91 C82.2 40.34, 101.76 22.38, 115.96 3.69 M65.11 59.74 C80.55 42.66, 96.85 24.5, 115.8 2.63 M70.79 61.81 C90.23 40.44, 106.09 19.04, 118.32 2.77 M70.71 59.98 C81.64 47.16, 91.78 35.08, 119.16 4.07 M75.27 60.07 C86.78 46.67, 98.46 35.83, 126.23 4.25 M75.06 59.99 C87.57 46.5, 97.78 34.42, 124.26 4.06 M79.59 61.24 C89.81 48.83, 103.72 35.64, 130.51 5.31 M81.51 61.8 C95.29 44.38, 110.66 26.15, 128.66 6.02 M85.16 60.22 C99.03 49.09, 108.83 34.07, 134.8 6.61 M85.04 60.98 C96.61 50.16, 106.09 37.03, 134.73 6.34 M92.6 61.39 C101.05 47.57, 114.63 35.4, 136.86 5.69 M90.34 61.56 C107.87 40.46, 126.73 20.66, 139.13 6.58 M96.23 62.01 C105.83 46.94, 117.04 36.46, 145.02 6.07 M96.46 62.22 C105.31 51.22, 116.48 39.02, 143.46 7.4 M104.7 61.8 C112.5 45.73, 125.05 35.64, 146.82 6.89 M103.63 60.77 C120.24 39.15, 138.89 18.18, 149.04 8.99 M110.81 59.14 C116.73 47.43, 127.56 39.94, 152.13 9.52 M109.28 59.05 C116.91 48.88, 126.29 39.31, 152.29 10.53 M114.06 58.36 C124.98 46.88, 137.55 34.62, 155.05 13.69 M114.76 59.12 C129.03 41.45, 142.66 24.8, 154.76 11.09 M120.48 55.66 C132.18 46.83, 139.25 34.29, 158.48 15.13 M121.42 57.94 C131.53 45.45, 144.43 30.68, 159.93 14.1 M127.96 56.49 C136.13 45.27, 145.65 31.11, 165.31 16.16 M126.01 57.6 C139.31 42.25, 152.53 26.75, 162.51 14.77 M133.31 57.64 C145.72 44.23, 156.22 28.79, 165.65 16.88 M132.03 56.18 C146.61 41.65, 159.58 24.87, 167.98 16.06 M135.6 58.94 C148.24 45.46, 156.6 33.26, 170.64 18.72 M137.22 56.3 C148.13 45.48, 157.12 34.85, 170.48 19.06 M145.21 54.5 C153.01 43.6, 163.21 31.57, 174.42 21.1 M144.8 55.16 C151.26 45.14, 159.43 35.79, 173.01 22.48 M151.9 52.74 C157.74 46.77, 163.38 36.76, 175.92 25.08 M151.72 53.37 C156.57 47.39, 160.81 41.75, 175.43 27.07 M157.9 52.17 C163.98 43.13, 170.08 36.87, 175.42 30.91 M157.96 51.58 C163.35 47.05, 166.76 42.13, 177.14 30.02" stroke="#fa5252" stroke-width="0.5" fill="none"></path><path d="M102.91 0.28 C116 0.49, 132 3.08, 142.9 5.83 C153.79 8.58, 162.58 12.65, 168.29 16.77 C174 20.89, 177.86 25.76, 177.16 30.56 C176.47 35.35, 171.22 41.48, 164.11 45.51 C156.99 49.55, 145.9 52.32, 134.48 54.74 C123.07 57.17, 109.19 59.56, 95.62 60.04 C82.05 60.52, 65.61 59.37, 53.05 57.63 C40.48 55.88, 28.71 53.03, 20.23 49.56 C11.76 46.09, 4.98 41.4, 2.18 36.81 C-0.61 32.21, -0.35 26.48, 3.47 21.99 C7.3 17.5, 15.63 13.24, 25.15 9.86 C34.67 6.49, 46.05 3.26, 60.6 1.77 C75.15 0.27, 102.12 0.85, 112.45 0.92 C122.77 0.98, 122.71 1.7, 122.56 2.16 M90.43 1.09 C103.67 0.97, 119.97 2.12, 131.93 4.31 C143.89 6.51, 154.79 10.39, 162.2 14.25 C169.61 18.11, 175.07 23, 176.38 27.47 C177.68 31.95, 175.31 36.8, 170.05 41.1 C164.8 45.4, 155.38 50.02, 144.82 53.26 C134.26 56.5, 120.23 59.7, 106.67 60.54 C93.11 61.38, 76.62 59.6, 63.48 58.29 C50.33 56.99, 37.64 55.85, 27.78 52.72 C17.92 49.6, 8.63 44.28, 4.32 39.55 C0.01 34.82, -0.03 28.83, 1.93 24.36 C3.88 19.88, 7.81 16.25, 16.05 12.7 C24.29 9.15, 38.8 5.03, 51.37 3.04 C63.94 1.06, 84.79 1.43, 91.46 0.81 C98.13 0.19, 91.62 -1.05, 91.39 -0.69" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(474.1552782881381 302.1715976437567) rotate(0 30.199966430664062 12)"><text x="30.199966430664062" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">server</text></g><g stroke-linecap="round"><g transform="translate(511.5857950846357 346.9908496633375) rotate(0 0.2695338497335342 36.150957581386905)"><path d="M0.09 0.85 C0.25 13.01, 1.89 60.31, 1.87 72.05 M-1.33 0.25 C-1.32 12.16, 0.78 58.27, 1.39 70.34" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(511.5857950846357 346.9908496633375) rotate(0 0.2695338497335342 36.150957581386905)"><path d="M-8.76 44.15 C-4.79 51.28, -1.73 64.59, 2.78 68.99 M-10.87 42.44 C-5.08 52.87, -1.86 62.96, 0.81 70.32" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(511.5857950846357 346.9908496633375) rotate(0 0.2695338497335342 36.150957581386905)"><path d="M11.74 43.19 C8.06 50.69, 3.47 64.36, 2.78 68.99 M9.63 41.48 C7.94 52.08, 3.7 62.51, 0.81 70.32" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round"><g transform="translate(224.07059178207874 311.596909344623) rotate(0 93.38666777561366 0.7448040874400874)"><path d="M0.63 -0.52 C31.77 -0.22, 155.94 0.5, 187.27 0.76 M-0.5 1.82 C30.42 2.32, 154.66 1.67, 186.13 1.67" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(224.07059178207874 311.596909344623) rotate(0 93.38666777561366 0.7448040874400874)"><path d="M157.1 11.39 C165.54 9.99, 175.45 6, 184.36 0.04 M158.92 12.61 C166.55 8.26, 174.11 5.76, 185.51 1.75" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(224.07059178207874 311.596909344623) rotate(0 93.38666777561366 0.7448040874400874)"><path d="M157.04 -9.13 C165.33 -3.64, 175.26 -0.74, 184.36 0.04 M158.87 -7.91 C166.59 -6.54, 174.17 -3.31, 185.51 1.75" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round"><g transform="translate(401.5251483339257 337.29388921411874) rotate(0 -85.75627581758204 -0.1390905062639447)"><path d="M0.68 -1.13 C-27.75 -1.36, -142.6 -0.73, -171.05 -0.7 M-0.42 0.9 C-29 0.79, -143.86 0.85, -172.19 0.73" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(401.5251483339257 337.29388921411874) rotate(0 -85.75627581758204 -0.1390905062639447)"><path d="M-145.86 -10.12 C-151.68 -5.53, -159.43 -4.51, -171.76 1.7 M-143.4 -9.72 C-149.85 -7.33, -157.81 -4.67, -171.65 1.41" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(401.5251483339257 337.29388921411874) rotate(0 -85.75627581758204 -0.1390905062639447)"><path d="M-145.89 10.4 C-151.82 9.61, -159.57 5.25, -171.76 1.7 M-143.44 10.8 C-150.02 8.44, -157.97 6.35, -171.65 1.41" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g transform="translate(299.5251094933715 282.26357971039) rotate(0 19.259979248046875 12)"><text x="0" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="start" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">grpc</text></g><g stroke-linecap="round"><g transform="translate(622.8107707281383 446.47327052541954) rotate(0 63.58800273071233 -1.1187277300982856)"><path d="M-0.23 0.29 C20.95 0.32, 105.62 -0.67, 126.83 -0.84 M-1.81 -0.61 C19.74 -0.81, 107.1 -2.19, 128.99 -2.52" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(622.8107707281383 446.47327052541954) rotate(0 63.58800273071233 -1.1187277300982856)"><path d="M101.44 6.4 C110.32 3.82, 118.35 1.12, 129.16 -2.62 M100.06 8.67 C110.9 3.14, 121.15 0.7, 129.45 -2.69" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(622.8107707281383 446.47327052541954) rotate(0 63.58800273071233 -1.1187277300982856)"><path d="M101.12 -14.12 C109.91 -10.94, 118.03 -7.88, 129.16 -2.62 M99.74 -11.85 C110.77 -9.68, 121.15 -4.4, 129.45 -2.69" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(752.2684809768084 406.1001728922614) rotate(0 67.5 40)"><path d="M40.61 3.26 C50.07 0.15, 63.38 0.09, 74.35 0.87 C85.33 1.65, 97.47 4.15, 106.49 7.92 C115.51 11.7, 123.84 17.57, 128.47 23.52 C133.1 29.48, 135.39 36.98, 134.25 43.65 C133.11 50.32, 128.26 58.09, 121.64 63.53 C115.03 68.97, 104.85 73.62, 94.55 76.28 C84.25 78.94, 71.06 79.97, 59.85 79.48 C48.63 78.98, 36.12 77.23, 27.26 73.3 C18.4 69.38, 11.29 62.12, 6.69 55.93 C2.1 49.75, -1.47 42.91, -0.3 36.2 C0.87 29.48, 6.35 21.14, 13.71 15.62 C21.07 10.11, 38.06 5.38, 43.85 3.11 C49.65 0.84, 48.02 1.51, 48.51 1.99 M41.88 1.78 C51.71 -1.21, 66.66 -2.21, 77.91 -0.83 C89.17 0.55, 100.84 5.45, 109.4 10.05 C117.96 14.66, 125.19 20.77, 129.26 26.81 C133.32 32.84, 135.34 39.97, 133.76 46.26 C132.19 52.56, 126.61 59.22, 119.81 64.58 C113.01 69.95, 103.56 75.86, 92.96 78.47 C82.37 81.07, 67.59 81.57, 56.23 80.23 C44.88 78.89, 33.46 74.68, 24.83 70.42 C16.2 66.16, 8.63 60.77, 4.46 54.67 C0.29 48.57, -1.93 40.52, -0.21 33.82 C1.51 27.12, 7.72 19.4, 14.78 14.46 C21.84 9.52, 37.28 5.94, 42.15 4.18 C47.01 2.41, 43.59 3.35, 43.95 3.87" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(800.8687827071658 434.3159016447995) rotate(0 18.66999053955078 12)"><text x="18.66999053955078" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">CRI</text></g><g stroke-linecap="round"><g transform="translate(886.4478777233116 436.18043731646776) rotate(0 75.043757982746 -43.77942354772479)"><path d="M1.02 -1.01 C26.03 -15.57, 125.34 -72.41, 149.98 -86.88 M0.1 1.07 C25.05 -13.87, 124.95 -74.02, 149.63 -88.63" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(886.4478777233116 436.18043731646776) rotate(0 75.043757982746 -43.77942354772479)"><path d="M129.01 -63.48 C137.4 -72.33, 141.46 -80.02, 149.52 -88.49 M130.63 -66.12 C137.22 -72.42, 141.94 -78.58, 149.36 -88.15" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(886.4478777233116 436.18043731646776) rotate(0 75.043757982746 -43.77942354772479)"><path d="M118.48 -81.1 C130.06 -84.81, 137.24 -87.28, 149.52 -88.49 M120.1 -83.74 C129.72 -84.9, 137.53 -85.89, 149.36 -88.15" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round"><g transform="translate(852.2269438322405 484.5232437664665) rotate(0 87.21825058140189 52.17678262877824)"><path d="M0.79 -0.51 C29.88 17.38, 145.72 88.35, 174.7 106.18 M-0.26 -1.82 C28.64 15.8, 144.64 86.55, 173.63 104.26" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(852.2269438322405 484.5232437664665) rotate(0 87.21825058140189 52.17678262877824)"><path d="M143.37 97.96 C154.04 99.51, 163.73 101.86, 171.64 102.78 M144.75 98.16 C150.44 98.85, 156.23 100.9, 173.36 105.05" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(852.2269438322405 484.5232437664665) rotate(0 87.21825058140189 52.17678262877824)"><path d="M154.06 80.45 C161.21 87.56, 167.55 95.4, 171.64 102.78 M155.44 80.65 C158.81 84.91, 162.42 90.54, 173.36 105.05" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round"><g transform="translate(820.2890502975711 490.9175062069397) rotate(0 106.97556665302295 759.7548438040928)"><path d="M-0.31 0.73 C35.38 254.02, 178.79 1266.28, 214.26 1519.43 M1.73 0.07 C37.36 253.03, 178.97 1264.7, 213.93 1517.64" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(820.2890502975711 490.9175062069397) rotate(0 106.97556665302295 759.7548438040928)"><path d="M201.1 1493.1 C205.22 1498.46, 209.93 1506.67, 215.5 1516.24 M199 1491.86 C203.03 1498.78, 208.46 1506.41, 214.91 1518.28" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(820.2890502975711 490.9175062069397) rotate(0 106.97556665302295 759.7548438040928)"><path d="M221.43 1490.27 C219.25 1496.46, 217.67 1505.54, 215.5 1516.24 M219.33 1489.03 C217.11 1496.95, 216.33 1505.44, 214.91 1518.28" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(1036.50769415288 296.5461966460997) rotate(0 136.5 51)"><path d="M6.13 6.66 C6.13 6.66, 6.13 6.66, 6.13 6.66 M6.13 6.66 C6.13 6.66, 6.13 6.66, 6.13 6.66 M3.25 16.07 C7.28 11.13, 10.16 7.39, 15.05 2.49 M3.25 16.07 C6.05 12.17, 9.22 8.14, 15.05 2.49 M3.64 21.71 C9.26 15.47, 17.68 6.02, 21.35 1.34 M3.64 21.71 C6.92 17.51, 10.84 11.74, 21.35 1.34 M3.38 28.11 C6.95 22.46, 13.03 14.91, 27 0.94 M3.38 28.11 C12.16 17.91, 21.49 6.86, 27 0.94 M3.12 34.51 C11.49 22.83, 23.66 11.13, 31.98 1.3 M3.12 34.51 C12.71 21.73, 22.94 10.76, 31.98 1.3 M3.51 40.15 C8.6 32.67, 18.03 24.33, 37.63 0.91 M3.51 40.15 C13.79 27.48, 24.01 15.17, 37.63 0.91 M3.25 46.55 C14.63 30.12, 28.08 18.78, 42.61 1.27 M3.25 46.55 C16.2 31.5, 30.04 15.86, 42.61 1.27 M3.65 52.19 C20.1 34.31, 34.82 14.96, 48.26 0.87 M3.65 52.19 C12.8 40.51, 22.56 29.57, 48.26 0.87 M3.38 58.59 C19.94 36.28, 39.28 17.68, 53.24 1.23 M3.38 58.59 C16.02 43.56, 28.51 29.28, 53.24 1.23 M3.12 64.99 C21.67 47.37, 36.27 25.64, 58.89 0.84 M3.12 64.99 C23.01 42.6, 42.18 19.78, 58.89 0.84 M3.52 70.63 C22.15 52.48, 39.06 30.93, 63.87 1.2 M3.52 70.63 C24.99 45.78, 46.04 21.5, 63.87 1.2 M3.26 77.03 C28.83 50.57, 50.08 18.53, 69.52 0.81 M3.26 77.03 C29.21 48.55, 54.33 19.35, 69.52 0.81 M2.34 84.18 C29.2 52.59, 56.53 21.93, 74.5 1.17 M2.34 84.18 C29.99 51.96, 56.92 19.68, 74.5 1.17 M4.04 88.32 C22.53 68.95, 38.39 50.1, 80.15 0.77 M4.04 88.32 C22.18 67.85, 38.53 47.74, 80.15 0.77 M5.1 93.21 C26.47 68.17, 50.74 42.2, 85.13 1.13 M5.1 93.21 C27.9 67.97, 49.92 42.73, 85.13 1.13 M7.46 96.58 C38 60.77, 67.8 27.4, 90.12 1.49 M7.46 96.58 C28.27 74.12, 49.11 50.13, 90.12 1.49 M11.13 98.45 C35.41 71.66, 62.1 43.06, 95.76 1.1 M11.13 98.45 C34.09 74.12, 55.42 48.31, 95.76 1.1 M14.81 100.32 C41.94 69.67, 64.74 41.57, 100.75 1.46 M14.81 100.32 C35.51 77.1, 56.18 53.58, 100.75 1.46 M19.14 101.44 C43.01 72.5, 70.27 44.63, 106.39 1.06 M19.14 101.44 C49.69 67, 78.7 31.9, 106.39 1.06 M24.13 101.8 C52.23 66.82, 85.17 31.24, 111.38 1.42 M24.13 101.8 C45.07 78.48, 65.58 53.84, 111.38 1.42 M27.8 103.67 C62.07 62.16, 96 25.97, 117.02 1.03 M27.8 103.67 C52.9 75.24, 76.94 47.85, 117.02 1.03 M32.79 104.03 C52.06 82.51, 71.11 60.86, 122.01 1.39 M32.79 104.03 C60.83 68.96, 89.21 35.71, 122.01 1.39 M38.43 103.63 C60.05 76.44, 80.5 51.28, 127.65 0.99 M38.43 103.63 C63.86 73.95, 88.64 44.59, 127.65 0.99 M43.42 103.99 C69.22 74.72, 91.03 48.87, 132.64 1.35 M43.42 103.99 C63.63 80.32, 83.52 58.43, 132.64 1.35 M49.06 103.6 C69.9 79.16, 88.66 52.96, 138.29 0.96 M49.06 103.6 C69.17 80.64, 90.37 57.32, 138.29 0.96 M54.05 103.96 C75.2 81.51, 96.13 54.44, 143.27 1.32 M54.05 103.96 C78.77 75.94, 102.72 46.87, 143.27 1.32 M59.69 103.56 C78.45 81, 100.92 55.29, 148.92 0.92 M59.69 103.56 C81.87 79.44, 101.82 56.7, 148.92 0.92 M64.68 103.92 C86.79 75.69, 112.75 48.3, 153.9 1.28 M64.68 103.92 C90.27 73.63, 116.32 43.28, 153.9 1.28 M70.32 103.53 C88.78 81.27, 109.7 61.96, 159.55 0.89 M70.32 103.53 C89 81.92, 108.53 60.44, 159.55 0.89 M75.31 103.89 C107.23 68.01, 134.68 32.64, 164.53 1.25 M75.31 103.89 C102.67 69.88, 132.83 36.84, 164.53 1.25 M80.95 103.5 C104.02 76.84, 127.27 48.95, 170.18 0.85 M80.95 103.5 C116.98 61.96, 152.84 22.59, 170.18 0.85 M85.94 103.86 C104.1 81.77, 121.02 63.76, 175.16 1.21 M85.94 103.86 C107.21 79.81, 128.25 54.78, 175.16 1.21 M90.93 104.22 C109.39 82.44, 128.17 61.81, 180.81 0.82 M90.93 104.22 C121.92 67.99, 155.13 30.17, 180.81 0.82 M96.57 103.82 C120.73 77.38, 143.04 50.55, 185.79 1.18 M96.57 103.82 C127.41 67.33, 161.17 28.87, 185.79 1.18 M101.56 104.18 C133.59 68.73, 161.53 32.12, 191.44 0.79 M101.56 104.18 C120.42 81.3, 140.77 59.49, 191.44 0.79 M107.2 103.79 C133.21 74.63, 158.01 43.9, 196.42 1.15 M107.2 103.79 C132.18 76.38, 154.82 49.84, 196.42 1.15 M112.19 104.15 C136.02 75.95, 163.51 45.96, 201.41 1.51 M112.19 104.15 C133.45 80.64, 153.86 55.91, 201.41 1.51 M117.83 103.75 C145.71 72.84, 172.01 42.72, 207.05 1.11 M117.83 103.75 C148.23 68.18, 178.21 34.43, 207.05 1.11 M122.82 104.11 C155.2 63.93, 187.49 25.34, 212.04 1.47 M122.82 104.11 C140.52 82.54, 158.97 62.84, 212.04 1.47 M128.46 103.72 C152.57 71.78, 180.1 43.42, 217.68 1.08 M128.46 103.72 C149.65 80.42, 169.03 56.8, 217.68 1.08 M133.45 104.08 C155.77 80.16, 176.55 53.64, 222.67 1.44 M133.45 104.08 C158.34 75.76, 184.91 46.3, 222.67 1.44 M139.09 103.68 C164.53 74.83, 188.59 43.05, 228.31 1.04 M139.09 103.68 C158.67 80.59, 179.19 58.15, 228.31 1.04 M144.08 104.04 C166.19 77.61, 187.14 52.66, 233.3 1.4 M144.08 104.04 C173.42 72.35, 202.31 39.12, 233.3 1.4 M149.72 103.65 C176.45 72.04, 206.73 38.39, 238.94 1.01 M149.72 103.65 C170.47 77.6, 194.56 52.99, 238.94 1.01 M154.71 104.01 C177.66 79.05, 196.66 54.38, 243.93 1.37 M154.71 104.01 C185.95 67.42, 219.03 29.69, 243.93 1.37 M160.35 103.61 C179.43 78.22, 200.87 52.7, 248.92 1.73 M160.35 103.61 C184.7 75.75, 209.86 48.23, 248.92 1.73 M165.34 103.97 C201.07 62.33, 235.57 24.38, 253.9 2.09 M165.34 103.97 C197.62 68.81, 227.89 32.53, 253.9 2.09 M170.98 103.58 C192.84 78.38, 216.4 51.12, 258.89 2.45 M170.98 103.58 C195.79 73.51, 223.61 43.45, 258.89 2.45 M175.97 103.94 C210.2 65.44, 243.69 28.42, 263.22 3.56 M175.97 103.94 C192.22 83.34, 210.04 63.63, 263.22 3.56 M181.61 103.54 C205.72 76.44, 226.77 48.7, 266.24 6.19 M181.61 103.54 C213.78 67.77, 245.3 30.84, 266.24 6.19 M186.6 103.9 C217.39 67.56, 249.5 28.74, 268.6 9.57 M186.6 103.9 C216.51 71.82, 246.47 37.2, 268.6 9.57 M192.24 103.51 C217.6 74.95, 244.91 43.24, 271.62 12.19 M192.24 103.51 C222.78 67.06, 254.7 31.97, 271.62 12.19 M197.23 103.87 C223.02 75.53, 245.22 46.9, 273.33 16.32 M197.23 103.87 C227.2 68.66, 258.13 33.99, 273.33 16.32 M202.21 104.23 C221.53 78.23, 243.77 54.16, 274.38 21.21 M202.21 104.23 C221 83.25, 240.11 60.51, 274.38 21.21 M207.86 103.84 C233.56 75.52, 257.09 48.56, 274.77 26.86 M207.86 103.84 C223.99 87.04, 239.62 68.16, 274.77 26.86 M212.84 104.2 C228.08 82.98, 244.68 63.86, 274.51 33.25 M212.84 104.2 C235.74 77.93, 258.58 51.58, 274.51 33.25 M218.49 103.8 C234.56 88.57, 246.41 71.37, 274.25 39.65 M218.49 103.8 C237.47 81.65, 255.94 58.3, 274.25 39.65 M223.47 104.16 C240.44 82.3, 258.45 62.47, 274.65 45.29 M223.47 104.16 C240.41 86.23, 257.74 65.28, 274.65 45.29 M229.12 103.77 C236.72 93.11, 248.92 83.8, 274.38 51.69 M229.12 103.77 C245.11 86.28, 259.67 68.21, 274.38 51.69 M234.1 104.13 C243.27 92.16, 250.41 85.03, 274.78 57.34 M234.1 104.13 C249.06 87.02, 264.06 69.02, 274.78 57.34 M239.75 103.73 C250.28 89.31, 264.16 77.83, 274.52 63.73 M239.75 103.73 C251.62 89.44, 264.17 75.71, 274.52 63.73 M244.73 104.09 C249.39 95.31, 258.93 87.91, 274.91 69.38 M244.73 104.09 C250.39 97.05, 256.28 90.63, 274.91 69.38 M249.72 104.45 C255.71 95.75, 266.42 86, 273.34 77.28 M249.72 104.45 C255.69 96.34, 263.3 90.6, 273.34 77.28 M256.68 102.55 C260.2 98.4, 263.19 94.4, 272.42 84.44 M256.68 102.55 C261.82 96.87, 266.96 90.35, 272.42 84.44" stroke="#fd7e14" stroke-width="0.5" fill="none"></path><path d="M25.5 0 M25.5 0 C91.21 -1.81, 158.81 0.11, 247.5 0 M25.5 0 C98.5 1.27, 171.58 0.44, 247.5 0 M247.5 0 C263.09 -0.25, 271.2 7.84, 273 25.5 M247.5 0 C264.6 -0.58, 273.14 10.68, 273 25.5 M273 25.5 C271.74 36.24, 271.64 49.06, 273 76.5 M273 25.5 C272.68 40.97, 272.78 57.41, 273 76.5 M273 76.5 C271.44 91.94, 263.79 101.76, 247.5 102 M273 76.5 C273.25 94.97, 265.32 101.51, 247.5 102 M247.5 102 C173.65 101.45, 103.2 102.51, 25.5 102 M247.5 102 C179.04 102.93, 111.55 102.62, 25.5 102 M25.5 102 C8.94 102.54, 1.29 92.87, 0 76.5 M25.5 102 C9.73 101.04, 1.75 92.14, 0 76.5 M0 76.5 C1.21 59.95, 1.19 44.96, 0 25.5 M0 76.5 C0.98 60.3, -0.01 45.44, 0 25.5 M0 25.5 C1.57 7.99, 9.55 1.84, 25.5 0 M0 25.5 C0.25 8.71, 10.45 -1.03, 25.5 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1041.607807067919 335.5461966460997) rotate(0 131.39988708496094 12)"><text x="131.39988708496094" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">PodSandboxManagerService</text></g><g stroke-linecap="round" transform="translate(1038.4673463587392 590.5095653167916) rotate(0 137 64)"><path d="M32 0 M32 0 C107.31 -0.06, 181.23 1.68, 242 0 M32 0 C92.14 1.77, 152.35 2.07, 242 0 M242 0 C262.13 0.52, 272.17 8.96, 274 32 M242 0 C261.05 0.09, 271.81 10.44, 274 32 M274 32 C273.9 55.49, 273.62 79.92, 274 96 M274 32 C273.58 51, 274.16 70.09, 274 96 M274 96 C274.82 115.86, 261.38 127.54, 242 128 M274 96 C275.16 118.22, 265.63 128.24, 242 128 M242 128 C177.28 129.89, 114.17 129.83, 32 128 M242 128 C161.73 128.13, 83.14 128.53, 32 128 M32 128 C9.82 126.56, -0.67 116.28, 0 96 M32 128 C12.03 129.49, 0.15 116.19, 0 96 M0 96 C-0.93 82.86, 0.52 67.95, 0 32 M0 96 C0.48 75.94, -0.19 57.25, 0 32 M0 32 C1.73 9.35, 10.34 0.95, 32 0 M0 32 C-1.4 12.44, 10.92 0.79, 32 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1081.9874421839345 642.5095653167916) rotate(0 93.47990417480469 12)"><text x="93.47990417480469" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">runtime service impl</text></g><g stroke-linecap="round" transform="translate(994.2448092818399 2014.8452431416083) rotate(0 163 44.5)"><path d="M22.25 0 M22.25 0 C86.76 1.09, 151.24 0.66, 303.75 0 M22.25 0 C109.02 -0.94, 196.5 -0.97, 303.75 0 M303.75 0 C320.16 -0.03, 326.93 5.7, 326 22.25 M303.75 0 C316.67 0.45, 325.14 6.44, 326 22.25 M326 22.25 C327.04 33.69, 325.98 41.54, 326 66.75 M326 22.25 C325.7 32.27, 326.43 41.41, 326 66.75 M326 66.75 C326.15 80.31, 320.06 89.13, 303.75 89 M326 66.75 C325.12 80.21, 319.96 90.99, 303.75 89 M303.75 89 C236.85 91.2, 171.99 90.55, 22.25 89 M303.75 89 C222.04 88.26, 141.63 88.81, 22.25 89 M22.25 89 C9.31 87.48, 1.06 81.22, 0 66.75 M22.25 89 C8.93 89.08, 0.14 81.05, 0 66.75 M0 66.75 C0.05 53.81, 1.61 35.8, 0 22.25 M0 66.75 C-1.19 58.07, -0.49 48.86, 0 22.25 M0 22.25 C-0.86 8.92, 6.02 1.42, 22.25 0 M0 22.25 C-0.9 7.48, 6.83 0.6, 22.25 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1054.1048938155313 2035.3452431416083) rotate(0 103.1399154663086 24)"><text x="103.1399154663086" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge"></text><text x="103.1399154663086" y="24" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">ImageManagerService</text></g><g stroke-linecap="round"><g transform="translate(1313.9198500972193 356.26875697452743) rotate(0 52.46276486258148 -23.004790023833557)"><path d="M0.9 0.24 C18.26 -7.09, 87.84 -36.84, 105.02 -44.39 M-0.09 -0.69 C17.12 -8.33, 87.36 -39.05, 104.55 -46.25" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1313.9198500972193 356.26875697452743) rotate(0 52.46276486258148 -23.004790023833557)"><path d="M83.1 -24.15 C91.7 -33.71, 98.58 -41.86, 103.25 -46.78 M83.4 -26.02 C90.97 -32.54, 97.05 -39.66, 104.99 -46.27" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1313.9198500972193 356.26875697452743) rotate(0 52.46276486258148 -23.004790023833557)"><path d="M75 -43 C86.52 -45.66, 96.39 -46.86, 103.25 -46.78 M75.3 -44.87 C85.72 -44.89, 94.61 -45.47, 104.99 -46.27" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(1425.952151060651 132.73223013534036) rotate(0 256.23700864765306 124.97508700007081)"><path d="M32 0 M32 0 C194.38 2.09, 355.07 2.36, 480.47 0 M480.47 0 C500.47 -1.54, 513.79 8.75, 512.47 32 M512.47 32 C514.72 88.69, 514.66 145.95, 512.47 217.95 M512.47 217.95 C512.2 237.43, 500.17 248.43, 480.47 249.95 M480.47 249.95 C370.76 250.33, 260.02 250.34, 32 249.95 M32 249.95 C10.38 251.64, 0.84 240.86, 0 217.95 M0 217.95 C-0.66 160.51, 1.35 106.1, 0 32 M0 32 C-0.58 8.98, 12.26 -1.66, 32 0" stroke="#000000" stroke-width="1.5" fill="none" stroke-dasharray="8 9"></path></g><g stroke-linecap="round" transform="translate(1461.2128342550404 165.72081685521715) rotate(0 232 40)"><path d="M20 0 M20 0 C155.15 0.9, 291.72 1.09, 444 0 M20 0 C136.88 1.03, 253.96 1.22, 444 0 M444 0 C458.53 -1.18, 462.55 6.13, 464 20 M444 0 C455.97 1.49, 463.38 8.06, 464 20 M464 20 C464.44 30.86, 462.49 40.03, 464 60 M464 20 C463.24 33.08, 464.62 46.28, 464 60 M464 60 C463.8 74.67, 456.75 80.26, 444 80 M464 60 C463.18 71.11, 456.47 79.83, 444 80 M444 80 C323.64 81.02, 203.25 80.74, 20 80 M444 80 C321.46 77.67, 199.51 78.28, 20 80 M20 80 C6.94 78.22, -0.89 72.23, 0 60 M20 80 C5.32 78.92, 0.32 72.6, 0 60 M0 60 C1.93 45.73, -0.54 31.37, 0 20 M0 60 C-0.92 47.04, 0.47 34.2, 0 20 M0 20 C-0.67 6.48, 5.6 -0.8, 20 0 M0 20 C0.25 5.15, 6.28 2.14, 20 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1526.5929612081654 193.72081685521715) rotate(0 166.619873046875 12)"><text x="166.619873046875" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">service_executor_t:callback函数</text></g><g stroke-linecap="round" transform="translate(1447.9781393583967 285.46042023426224) rotate(0 238.6888606061773 34.84857190029257)"><path d="M17.42 0 M17.42 0 C134.3 0.66, 251.28 1.35, 459.95 0 M17.42 0 C150.21 0, 282.95 -0.4, 459.95 0 M459.95 0 C469.57 -1.06, 475.6 7.1, 477.38 17.42 M459.95 0 C471.88 -2.05, 476.35 4.54, 477.38 17.42 M477.38 17.42 C476.58 26.54, 476.97 35.61, 477.38 52.27 M477.38 17.42 C476.33 31.38, 478.01 45.84, 477.38 52.27 M477.38 52.27 C476.16 63.77, 473.13 69.7, 459.95 69.7 M477.38 52.27 C476.61 63.68, 470.34 68.77, 459.95 69.7 M459.95 69.7 C323.32 68.35, 185.7 69.29, 17.42 69.7 M459.95 69.7 C345.79 70.6, 232.49 70.72, 17.42 69.7 M17.42 69.7 C4.45 67.95, 1.53 62.64, 0 52.27 M17.42 69.7 C6.24 67.94, 0.59 63.13, 0 52.27 M0 52.27 C0.91 42.68, 0.62 30.74, 0 17.42 M0 52.27 C-0.06 40.96, -1.07 30.09, 0 17.42 M0 17.42 C0.49 5.26, 4.95 -1.08, 17.42 0 M0 17.42 C-0.7 6.55, 5.46 0, 17.42 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1576.2270967052964 308.30899213455484) rotate(0 110.43990325927734 12)"><text x="110.43990325927734" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">Network::PluginManager</text></g><g stroke-linecap="round"><g transform="translate(1929.0843006785108 212.93503904600257) rotate(0 105.15624147941708 1.7990164303899476)"><path d="M1.1 -0.87 C36.12 -0.18, 175.48 2.33, 210.09 3.24 M0.22 1.28 C35.18 2.21, 175.1 4.25, 209.7 4.47" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1929.0843006785108 212.93503904600257) rotate(0 105.15624147941708 1.7990164303899476)"><path d="M179.94 16.19 C188.93 10.74, 199.59 7.05, 209.64 6.08 M181.08 13.82 C187.88 12.8, 195.58 10.38, 210.61 5.37" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1929.0843006785108 212.93503904600257) rotate(0 105.15624147941708 1.7990164303899476)"><path d="M180.18 -4.33 C188.94 -3.67, 199.54 -1.23, 209.64 6.08 M181.31 -6.7 C187.91 -2.78, 195.55 -0.26, 210.61 5.37" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(2154.397132316625 10) rotate(0 648.9822644893216 533.9221007324232)"><path d="M32 0 M32 0 C292.77 3.36, 554.57 3.33, 1265.96 0 M32 0 C514.67 1.14, 996.79 1.56, 1265.96 0 M1265.96 0 C1287.28 -0.63, 1299.29 9.37, 1297.96 32 M1265.96 0 C1288.71 -1.75, 1300.01 12.47, 1297.96 32 M1297.96 32 C1298.41 413.76, 1297.49 793.52, 1297.96 1035.84 M1297.96 32 C1295.65 275.25, 1295.59 518.45, 1297.96 1035.84 M1297.96 1035.84 C1299.4 1056.99, 1286.4 1066.27, 1265.96 1067.84 M1297.96 1035.84 C1298.77 1058.71, 1288.84 1066.64, 1265.96 1067.84 M1265.96 1067.84 C863.91 1072.11, 462.76 1071.29, 32 1067.84 M1265.96 1067.84 C1018.28 1067.47, 770.15 1067.55, 32 1067.84 M32 1067.84 C9.51 1068.33, -1.37 1056.24, 0 1035.84 M32 1067.84 C10.54 1069.94, 1.59 1054.98, 0 1035.84 M0 1035.84 C1.95 831.07, 3.14 626.75, 0 32 M0 1035.84 C-1.88 694.32, -2.22 352.4, 0 32 M0 32 C-1.64 9.64, 10.95 0.06, 32 0 M0 32 C-1.98 8.76, 12.22 1.67, 32 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g stroke-linecap="round" transform="translate(2231.3024347645787 86.69472207020567) rotate(0 192.46679021440877 47.09913122726334)"><path d="M5.61 6.2 C5.61 6.2, 5.61 6.2, 5.61 6.2 M5.61 6.2 C5.61 6.2, 5.61 6.2, 5.61 6.2 M3.38 14.87 C7.57 13.08, 6.98 7.3, 14.54 2.04 M3.38 14.87 C5.98 12.42, 9.12 9.38, 14.54 2.04 M0.5 24.28 C8.56 16.1, 14.89 6.37, 20.18 1.64 M0.5 24.28 C4.5 17.82, 10.92 13.75, 20.18 1.64 M0.89 29.93 C7.29 22.25, 11.76 18.14, 25.17 2 M0.89 29.93 C9.53 20.11, 19.63 10.17, 25.17 2 M0.63 36.32 C7.24 27.52, 15.16 18.33, 30.81 1.61 M0.63 36.32 C6.88 29.06, 13.98 19.1, 30.81 1.61 M0.37 42.72 C11.96 28.02, 24.94 16.95, 35.8 1.97 M0.37 42.72 C13.95 28.03, 27.16 11.41, 35.8 1.97 M0.76 48.36 C11.62 35.79, 27.25 19.24, 41.44 1.57 M0.76 48.36 C9.35 37.65, 18.4 28.44, 41.44 1.57 M0.5 54.76 C14.34 38.79, 26.57 26.49, 46.43 1.93 M0.5 54.76 C12.71 40.01, 26.99 24.98, 46.43 1.93 M0.9 60.4 C19.9 39.37, 34.89 20.32, 52.07 1.54 M0.9 60.4 C16.19 43.39, 30.92 27.66, 52.07 1.54 M0.64 66.8 C12.47 51.36, 25.17 37.9, 57.06 1.9 M0.64 66.8 C17.21 45.53, 35.02 25.59, 57.06 1.9 M1.69 71.69 C15.55 56.53, 30.62 41.43, 62.7 1.5 M1.69 71.69 C18.14 51.12, 35.57 32.72, 62.7 1.5 M2.08 77.33 C26.45 49, 52.1 22.94, 67.69 1.86 M2.08 77.33 C25.35 50.63, 48.03 24.31, 67.69 1.86 M3.13 82.22 C30.77 51.36, 57.8 21.37, 73.33 1.47 M3.13 82.22 C27.98 53.29, 55.1 21.82, 73.33 1.47 M4.84 86.36 C24.81 63.34, 47.58 39.02, 78.32 1.83 M4.84 86.36 C26.9 60.97, 50.44 34.26, 78.32 1.83 M7.86 88.98 C31.38 61.7, 54.75 35.82, 83.96 1.43 M7.86 88.98 C29.41 64.65, 49.63 41.58, 83.96 1.43 M10.22 92.36 C37.38 63.14, 66.96 30.99, 88.95 1.79 M10.22 92.36 C37.63 59.93, 66.35 28.09, 88.95 1.79 M14.55 93.47 C34.41 71.36, 51.73 50.7, 94.59 1.4 M14.55 93.47 C34.16 70.77, 55.07 45.92, 94.59 1.4 M18.88 94.59 C48.54 60.55, 77.97 27.36, 99.58 1.76 M18.88 94.59 C43.97 64.96, 69.34 35.62, 99.58 1.76 M23.87 94.95 C47.96 68.83, 68.41 42.82, 104.56 2.12 M23.87 94.95 C48.43 66.1, 74.87 37.18, 104.56 2.12 M29.51 94.55 C57.27 60.32, 88.19 25.62, 110.21 1.73 M29.51 94.55 C62.23 57.3, 92.84 21.26, 110.21 1.73 M34.5 94.91 C57.76 68.35, 81.37 42.88, 115.19 2.09 M34.5 94.91 C65.14 59.8, 97.68 22.71, 115.19 2.09 M40.14 94.52 C66.63 62.4, 96.78 30.82, 120.84 1.69 M40.14 94.52 C64.55 66.65, 88.57 38.09, 120.84 1.69 M45.13 94.88 C61.88 76.22, 77.55 58.41, 125.82 2.05 M45.13 94.88 C72.93 62.3, 101.43 30.31, 125.82 2.05 M50.77 94.49 C77.22 62.15, 106.49 29.33, 131.47 1.66 M50.77 94.49 C80.92 60.27, 110.09 27.1, 131.47 1.66 M55.76 94.85 C80.81 69.22, 105.13 40.84, 136.45 2.02 M55.76 94.85 C73.55 72.75, 93.36 52.62, 136.45 2.02 M61.4 94.45 C93.93 58.14, 127.11 19.82, 142.1 1.62 M61.4 94.45 C87.45 64.24, 114.96 33.02, 142.1 1.62 M66.39 94.81 C92.29 63.77, 119.96 32.82, 147.08 1.98 M66.39 94.81 C97.62 59.33, 128.07 22.74, 147.08 1.98 M72.03 94.42 C97.75 61.93, 127.16 29.7, 152.73 1.59 M72.03 94.42 C90.26 73.57, 109.15 52.19, 152.73 1.59 M77.02 94.78 C110.77 56.89, 141.54 22.89, 157.71 1.95 M77.02 94.78 C101.35 65.12, 127.39 37.41, 157.71 1.95 M82.66 94.38 C105.49 69.81, 125.9 45.61, 163.36 1.55 M82.66 94.38 C106.2 67.43, 129.77 39.75, 163.36 1.55 M87.65 94.74 C110.25 71.49, 127.5 46.41, 168.34 1.91 M87.65 94.74 C109.35 69.89, 130.32 46.31, 168.34 1.91 M93.29 94.35 C108.68 73.26, 124.02 58.57, 173.99 1.52 M93.29 94.35 C109.96 74.19, 127.64 54.99, 173.99 1.52 M98.28 94.71 C114.12 74.91, 131.93 57.93, 178.97 1.88 M98.28 94.71 C114.97 73.79, 132.67 54.6, 178.97 1.88 M103.92 94.31 C124.68 72.65, 142.82 49.59, 184.62 1.48 M103.92 94.31 C128.8 66.02, 154.85 36.51, 184.62 1.48 M108.91 94.67 C130.33 69.98, 152.07 48.43, 189.6 1.84 M108.91 94.67 C133.1 67.4, 156.15 39.78, 189.6 1.84 M113.9 95.03 C136.43 71.66, 155.82 46.68, 194.59 2.2 M113.9 95.03 C143.33 62.27, 172.7 29.04, 194.59 2.2 M119.54 94.64 C144.39 63.64, 173.35 31.45, 200.23 1.81 M119.54 94.64 C149.38 62.05, 177.26 29.22, 200.23 1.81 M124.53 95 C154.84 62.03, 183.36 28.49, 205.22 2.17 M124.53 95 C154.43 59.36, 185.15 25.01, 205.22 2.17 M130.17 94.6 C152.62 71.76, 170.75 47.86, 210.86 1.77 M130.17 94.6 C147.42 74.44, 164.72 54.13, 210.86 1.77 M135.16 94.96 C153.13 71.69, 174.6 48.13, 215.85 2.13 M135.16 94.96 C153.11 75.24, 168.51 56.21, 215.85 2.13 M140.8 94.57 C169.05 60.27, 196.04 27.29, 221.49 1.74 M140.8 94.57 C159.28 74.62, 176.29 53.92, 221.49 1.74 M145.79 94.93 C165.26 72.22, 182.6 51.74, 226.48 2.1 M145.79 94.93 C173.94 62.56, 201.23 30.63, 226.48 2.1 M151.43 94.54 C172.25 67.69, 195.81 46.1, 232.12 1.71 M151.43 94.54 C173.16 71.51, 193.8 46.5, 232.12 1.71 M156.42 94.9 C173.77 73.93, 191.47 54.53, 237.11 2.07 M156.42 94.9 C180.9 66.27, 206.84 36.64, 237.11 2.07 M162.06 94.5 C192.49 63.21, 219.57 27.43, 242.75 1.67 M162.06 94.5 C189.66 63.31, 215.86 32.98, 242.75 1.67 M167.05 94.86 C196.72 58.21, 227.78 25.28, 247.74 2.03 M167.05 94.86 C188.68 68.74, 212.25 43.56, 247.74 2.03 M172.69 94.47 C194.73 67.83, 217.82 44.37, 253.38 1.64 M172.69 94.47 C193.14 71.73, 212.89 49.1, 253.38 1.64 M177.68 94.83 C194.47 73.4, 214.67 52.6, 258.37 2 M177.68 94.83 C200.12 68.79, 222.64 42.95, 258.37 2 M183.32 94.43 C217.47 56.45, 245.82 20.04, 264.01 1.6 M183.32 94.43 C205.06 68.99, 227.04 43.91, 264.01 1.6 M188.31 94.79 C212.03 67.05, 239.46 36.58, 269 1.96 M188.31 94.79 C216.33 63.23, 243.77 29.57, 269 1.96 M193.95 94.4 C228.06 56.55, 257.97 20.29, 274.64 1.57 M193.95 94.4 C216.21 67.06, 239.69 41.84, 274.64 1.57 M198.94 94.76 C231.99 57.82, 261.18 19.83, 279.63 1.93 M198.94 94.76 C230.59 59.66, 261.69 22.79, 279.63 1.93 M203.92 95.12 C226.44 70.57, 248.21 43.15, 284.62 2.29 M203.92 95.12 C225.53 70.66, 248.79 46.01, 284.62 2.29 M209.57 94.72 C229.2 70.01, 249.92 47.73, 290.26 1.89 M209.57 94.72 C236.6 64.44, 261.14 33.97, 290.26 1.89 M214.55 95.08 C241.94 64.1, 266.83 35.13, 295.25 2.25 M214.55 95.08 C242.26 62.28, 270.85 31.3, 295.25 2.25 M220.2 94.69 C239.18 70.67, 263.6 45.81, 300.89 1.86 M220.2 94.69 C248.18 64.46, 275.42 31.65, 300.89 1.86 M225.18 95.05 C252.75 62.23, 283.41 29.77, 305.88 2.22 M225.18 95.05 C255.65 59.07, 286.53 24.26, 305.88 2.22 M230.83 94.65 C248.75 75.36, 265.76 56.99, 311.52 1.82 M230.83 94.65 C251.31 72.02, 272 48.15, 311.52 1.82 M235.81 95.01 C258.83 69.25, 280.12 43.63, 316.51 2.18 M235.81 95.01 C253.46 75.14, 271.17 54.99, 316.51 2.18 M241.46 94.62 C265.82 68.19, 288.1 40.69, 322.15 1.79 M241.46 94.62 C264.7 65.88, 289.57 38.32, 322.15 1.79 M246.44 94.98 C275.55 63.66, 301.49 32.82, 327.14 2.15 M246.44 94.98 C276.88 61.8, 306.32 26.99, 327.14 2.15 M252.09 94.58 C279.57 64.02, 302.06 37.38, 332.78 1.76 M252.09 94.58 C268.46 75.12, 286.17 54.3, 332.78 1.76 M257.07 94.94 C277.43 72.42, 297.61 49.39, 337.77 2.12 M257.07 94.94 C284.26 64.17, 312.2 32.89, 337.77 2.12 M262.72 94.55 C286.02 66.41, 308.79 38.76, 343.41 1.72 M262.72 94.55 C287.72 67.48, 311.49 38.39, 343.41 1.72 M267.7 94.91 C289.91 70.91, 308.62 44.57, 348.4 2.08 M267.7 94.91 C298.5 59.66, 328.34 25.92, 348.4 2.08 M273.35 94.52 C292.88 72.53, 315.93 47.3, 354.04 1.69 M273.35 94.52 C301.59 60.29, 331.09 27.43, 354.04 1.69 M278.33 94.88 C307.88 64.38, 334.63 32.41, 359.03 2.05 M278.33 94.88 C296.94 73.05, 316.65 50.27, 359.03 2.05 M283.32 95.24 C301.77 72.97, 323.89 47.81, 364.67 1.65 M283.32 95.24 C314.95 57.49, 346.97 21.2, 364.67 1.65 M288.96 94.84 C316.82 60.1, 344.8 27.69, 369 2.77 M288.96 94.84 C308.89 74.18, 327.48 50.22, 369 2.77 M293.95 95.2 C315.54 66.91, 340.09 40.01, 373.33 3.88 M293.95 95.2 C315.39 70.05, 336.57 45.93, 373.33 3.88 M299.59 94.81 C321.68 71.47, 342.83 42.89, 377.67 5 M299.59 94.81 C317.99 73.03, 337.38 49.83, 377.67 5 M304.58 95.17 C326 74.42, 344.83 50.69, 380.68 7.62 M304.58 95.17 C327.04 70.04, 348.13 45.15, 380.68 7.62 M310.22 94.77 C335.53 62.13, 365.99 31.71, 383.05 11 M310.22 94.77 C329.37 73.16, 346.52 53.08, 383.05 11 M315.21 95.13 C330.11 76.81, 342.1 62.86, 385.41 14.38 M315.21 95.13 C344.19 64.86, 370.03 32.33, 385.41 14.38 M320.85 94.74 C336.55 77.61, 351.45 58.7, 385.8 20.02 M320.85 94.74 C346.05 66.3, 369.78 37.63, 385.8 20.02 M325.84 95.1 C349.76 68.55, 374.98 40.66, 386.86 24.91 M325.84 95.1 C347.24 70.52, 370.03 45.78, 386.86 24.91 M331.48 94.7 C344.92 82.29, 353.02 68.66, 386.59 31.31 M331.48 94.7 C348.53 73.88, 366.01 53.12, 386.59 31.31 M336.47 95.06 C353.94 75.18, 368.69 55.98, 386.33 37.71 M336.47 95.06 C347.82 83.46, 357.52 70.61, 386.33 37.71 M342.11 94.67 C353.56 81.23, 363.24 68.73, 386.73 43.35 M342.11 94.67 C355.07 79.3, 368.1 65.14, 386.73 43.35 M347.1 95.03 C359.81 81.67, 369 67.58, 386.47 49.75 M347.1 95.03 C354.85 84.75, 363.63 76.68, 386.47 49.75 M352.74 94.63 C363.2 83.25, 372.07 72.86, 386.2 56.14 M352.74 94.63 C361.27 84.98, 369.37 75.99, 386.2 56.14 M357.73 94.99 C368.36 82.12, 380.57 71.05, 386.6 61.79 M357.73 94.99 C364.33 87.23, 372.98 76.64, 386.6 61.79 M362.72 95.35 C368.06 88.08, 370.8 85.75, 386.34 68.19 M362.72 95.35 C368.7 88.94, 375.53 80.28, 386.34 68.19 M369.67 93.45 C375.3 86.86, 376.76 82.16, 384.76 76.09 M369.67 93.45 C374.16 88.14, 379.09 81.92, 384.76 76.09" stroke="#228be6" stroke-width="0.5" fill="none"></path><path d="M23.55 0 M23.55 0 C119.35 1.35, 214.53 1.67, 361.38 0 M23.55 0 C94.98 0.51, 167.56 1.43, 361.38 0 M361.38 0 C378.18 -0.9, 383.52 8.91, 384.93 23.55 M361.38 0 C378.72 -0.57, 386.38 8.79, 384.93 23.55 M384.93 23.55 C386.27 39.89, 385.31 58.88, 384.93 70.65 M384.93 23.55 C384.7 33.85, 383.61 44.85, 384.93 70.65 M384.93 70.65 C385.27 86.15, 376.87 93.29, 361.38 94.2 M384.93 70.65 C386.07 87.95, 377.32 93.07, 361.38 94.2 M361.38 94.2 C243.72 95.61, 127.13 96.64, 23.55 94.2 M361.38 94.2 C265.4 92.81, 169.11 92.98, 23.55 94.2 M23.55 94.2 C9.28 92.52, 1.15 86.03, 0 70.65 M23.55 94.2 C6.76 95.05, -1.53 85.31, 0 70.65 M0 70.65 C-0.9 59.45, 0.44 47.24, 0 23.55 M0 70.65 C0.6 58.68, -0.23 47.97, 0 23.55 M0 23.55 C1.19 9.6, 8.18 -0.83, 23.55 0 M0 23.55 C-1.13 8.1, 6.29 -0.94, 23.55 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2349.449286319319 121.79385329746901) rotate(0 74.31993865966797 12)"><text x="74.31993865966797" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">RunPodSandbox</text></g><g stroke-linecap="round"><g transform="translate(2410.395439080794 182.0559079628299) rotate(0 -0.3978628610452688 35.637740280205804)"><path d="M-0.62 0.17 C-0.86 12.18, -0.87 58.95, -1.13 71.07 M1.26 -0.79 C0.83 11.42, -1.88 60.1, -2.05 72.07" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2410.395439080794 182.0559079628299) rotate(0 -0.3978628610452688 35.637740280205804)"><path d="M-10.96 43.57 C-10.16 51.31, -4.07 63.84, -3.45 72.66 M-11.35 43.98 C-8.18 50.54, -7.22 57.41, -1.81 73.01" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2410.395439080794 182.0559079628299) rotate(0 -0.3978628610452688 35.637740280205804)"><path d="M9.55 44.35 C3.59 51.73, 2.92 64, -3.45 72.66 M9.15 44.76 C7.26 50.99, 3.15 57.67, -1.81 73.01" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(2236.9717530693324 262.1534242822961) rotate(0 192.46679021440877 47.09913122726334)"><path d="M23.55 0 M23.55 0 C103.17 3.04, 183.57 0.78, 361.38 0 M23.55 0 C133.34 0.53, 243.71 1.26, 361.38 0 M361.38 0 C378.77 1.08, 383.38 8.21, 384.93 23.55 M361.38 0 C378.69 -0.53, 384.35 7.83, 384.93 23.55 M384.93 23.55 C385.9 37.55, 383.36 46.07, 384.93 70.65 M384.93 23.55 C384.37 42.62, 385.42 60.92, 384.93 70.65 M384.93 70.65 C385.85 85.32, 378.78 92.24, 361.38 94.2 M384.93 70.65 C386.71 86.34, 376.78 94.55, 361.38 94.2 M361.38 94.2 C275.62 95.03, 189.65 94.32, 23.55 94.2 M361.38 94.2 C276.94 91.66, 192.4 91.53, 23.55 94.2 M23.55 94.2 C7.19 94.29, -0.5 86.47, 0 70.65 M23.55 94.2 C10.03 92.52, -0.61 85.88, 0 70.65 M0 70.65 C0.62 59.45, 1.6 49.01, 0 23.55 M0 70.65 C0.15 58.93, -0.52 48.42, 0 23.55 M0 23.55 C-0.24 8.06, 9.12 0.71, 23.55 0 M0 23.55 C-0.49 8.46, 8.62 -0.82, 23.55 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2265.248678171436 285.25255550955944) rotate(0 164.1898651123047 24)"><text x="164.1898651123047" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">1. Pull the image for the sandbox</text><text x="164.1898651123047" y="24" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">EnsureSandboxImageExists</text></g><g stroke-linecap="round" transform="translate(2226.3599103366214 436.0131733110304) rotate(0 192.46679021440877 47.09913122726334)"><path d="M23.55 0 M23.55 0 C113 1.52, 200.09 1.03, 361.38 0 M23.55 0 C145.52 0.87, 267.68 1.06, 361.38 0 M361.38 0 C377.84 1.3, 383.46 6.06, 384.93 23.55 M361.38 0 C376.79 -0.09, 383.67 9.16, 384.93 23.55 M384.93 23.55 C384.2 32.98, 384.8 46.01, 384.93 70.65 M384.93 23.55 C384.8 37.04, 384.91 51.29, 384.93 70.65 M384.93 70.65 C384.48 86.42, 375.54 93.05, 361.38 94.2 M384.93 70.65 C383.77 86.62, 376.1 94.2, 361.38 94.2 M361.38 94.2 C245.77 96.65, 130.35 95.78, 23.55 94.2 M361.38 94.2 C244.27 95.87, 127.91 95.96, 23.55 94.2 M23.55 94.2 C7.47 92.73, -0.27 86.22, 0 70.65 M23.55 94.2 C7.06 92.05, -1.72 88.1, 0 70.65 M0 70.65 C0.43 54.83, -0.48 38.38, 0 23.55 M0 70.65 C-0.35 56.78, -0.06 44.59, 0 23.55 M0 23.55 C0.56 9.46, 6.51 1.52, 23.55 0 M0 23.55 C-1.58 5.92, 9.39 0.03, 23.55 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2256.0268348283735 459.11230453829376) rotate(0 162.79986572265625 24)"><text x="162.79986572265625" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">2. Create the sandbox container</text><text x="162.79986572265625" y="24" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">CreateSandboxContainer</text></g><g stroke-linecap="round"><g transform="translate(2411.2292163736784 357.3516867368228) rotate(0 -0.5197654683884139 35.93610327942316)"><path d="M0.16 0.33 C0.28 12.27, -0.13 59.47, -0.2 71.29 M-1.22 -0.54 C-1.09 11.56, -0.75 60.29, -0.64 72.41" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2411.2292163736784 357.3516867368228) rotate(0 -0.5197654683884139 35.93610327942316)"><path d="M-10.57 45.91 C-10.21 52.06, -7.91 55.12, 0.7 72.44 M-10.73 43.8 C-7.72 52.03, -5.93 59.36, -1.22 72.08" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2411.2292163736784 357.3516867368228) rotate(0 -0.5197654683884139 35.93610327942316)"><path d="M9.95 45.75 C5.75 51.96, 3.48 55.05, 0.7 72.44 M9.79 43.64 C7.36 51.94, 3.7 59.31, -1.22 72.08" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round"><g transform="translate(2612.747244879103 489.0724535186148) rotate(0 52.02919541353094 0.2697081558499548)"><path d="M0.45 -0.13 C17.9 -0.21, 87.6 0.17, 104.83 0.16 M-0.77 -1.25 C16.52 -1.16, 86.13 1.41, 103.97 1.78" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2612.747244879103 489.0724535186148) rotate(0 52.02919541353094 0.2697081558499548)"><path d="M75.27 11.49 C84.55 6.78, 92.79 6.98, 102.22 1.96 M74.62 10.75 C82.26 8.31, 89.12 7.06, 104.56 2.33" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2612.747244879103 489.0724535186148) rotate(0 52.02919541353094 0.2697081558499548)"><path d="M75.88 -9.03 C85.08 -7.36, 93.12 -0.8, 102.22 1.96 M75.22 -9.76 C82.65 -7.02, 89.36 -3.09, 104.56 2.33" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round"><g transform="translate(1334.6647761794493 660.3642618020249) rotate(0 36.17624263005973 3.2688330011344533)"><path d="M-0.35 0.67 C11.18 1.9, 58.6 5.69, 70.3 6.57 M1.67 -0.03 C13.49 0.93, 60.95 3.59, 72.7 4.81" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1334.6647761794493 660.3642618020249) rotate(0 36.17624263005973 3.2688330011344533)"><path d="M44.92 11.91 C56.45 8.01, 66.84 5.55, 72.94 6.75 M44.68 12.47 C52.48 9.88, 62.64 7.38, 72.18 4.29" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1334.6647761794493 660.3642618020249) rotate(0 36.17624263005973 3.2688330011344533)"><path d="M46.49 -8.55 C57.26 -4.36, 67.03 1.27, 72.94 6.75 M46.25 -8 C53.67 -4.2, 63.34 -0.32, 72.18 4.29" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(1415.837811384943 474.7019440187243) rotate(0 274.0148711964812 217.8413300086212)"><path d="M32 0 M32 0 C224.49 -0.15, 414.48 -0.72, 516.03 0 M516.03 0 C535.38 -0.8, 546.41 11.27, 548.03 32 M548.03 32 C548.83 115.99, 548.15 199.91, 548.03 403.68 M548.03 403.68 C549.97 426.17, 536.07 434.05, 516.03 435.68 M516.03 435.68 C379.41 436.56, 244.68 436.69, 32 435.68 M32 435.68 C10.09 434.08, -0.9 425.18, 0 403.68 M0 403.68 C2.17 299.16, 1.58 196.24, 0 32 M0 32 C0.43 11.78, 12.46 1.13, 32 0" stroke="#000000" stroke-width="1.5" fill="none" stroke-dasharray="8 9"></path></g><g stroke-linecap="round" transform="translate(1499.246278621079 492.14992563423976) rotate(0 204 32.5)"><path d="M4.33 3.76 C4.33 3.76, 4.33 3.76, 4.33 3.76 M4.33 3.76 C4.33 3.76, 4.33 3.76, 4.33 3.76 M3.41 10.91 C5.57 8.34, 5.59 7.04, 10.62 2.61 M3.41 10.91 C5.04 9.51, 6.58 7.47, 10.62 2.61 M2.49 18.07 C5.45 12.75, 8.32 11.18, 17.58 0.71 M2.49 18.07 C5.89 15.05, 9.74 11.42, 17.58 0.71 M2.23 24.46 C8.02 19.41, 15.58 9.27, 22.57 1.07 M2.23 24.46 C6.66 19.07, 12.12 13.81, 22.57 1.07 M2.62 30.11 C10.33 20.4, 16.36 13.59, 28.21 0.67 M2.62 30.11 C11.01 20.28, 19.01 10.26, 28.21 0.67 M3.02 35.75 C12.55 25.81, 21.75 13.99, 33.2 1.03 M3.02 35.75 C12.16 25.39, 23.07 12.84, 33.2 1.03 M2.76 42.15 C12.11 32.97, 18.48 23.97, 38.84 0.64 M2.76 42.15 C11.55 32.29, 18.89 23.46, 38.84 0.64 M3.15 47.79 C14.91 36.23, 26.54 21.25, 43.83 1 M3.15 47.79 C16.89 32.91, 29.15 17.62, 43.83 1 M1.58 55.7 C13.88 41.58, 23.1 29.75, 48.81 1.36 M1.58 55.7 C15.57 39.89, 29.41 22.66, 48.81 1.36 M3.94 59.08 C21.1 42.47, 33.74 22.84, 54.46 0.96 M3.94 59.08 C13.67 46.36, 25.01 34.49, 54.46 0.96 M6.3 62.45 C20.12 48.34, 34.92 31.57, 59.44 1.32 M6.3 62.45 C17.3 48.28, 29.37 35.52, 59.44 1.32 M9.32 65.08 C32.84 41.15, 52.57 14.2, 65.09 0.93 M9.32 65.08 C23.88 48.19, 37.78 33.05, 65.09 0.93 M13.65 66.19 C26.43 52.33, 36.55 36.64, 70.07 1.29 M13.65 66.19 C28.75 49.41, 42.63 32.98, 70.07 1.29 M19.3 65.8 C32.87 51.77, 42.39 35.73, 75.72 0.89 M19.3 65.8 C34.25 50.56, 46.46 33.48, 75.72 0.89 M24.28 66.16 C44.32 44.11, 66.42 21.15, 80.7 1.25 M24.28 66.16 C38.08 51.21, 52.24 33.88, 80.7 1.25 M29.93 65.76 C44.02 49.74, 57.19 33.92, 86.35 0.86 M29.93 65.76 C45.19 47.38, 60.99 29.95, 86.35 0.86 M34.91 66.12 C47.48 47.74, 63.17 34.11, 91.33 1.22 M34.91 66.12 C51.51 47.6, 68.31 28.22, 91.33 1.22 M40.56 65.73 C54.97 50.6, 67.25 35.46, 96.98 0.83 M40.56 65.73 C61.7 42.28, 81.27 17.91, 96.98 0.83 M45.54 66.09 C58.6 53.26, 73.43 37.4, 101.96 1.19 M45.54 66.09 C61.32 49.07, 74.59 32.79, 101.96 1.19 M51.19 65.7 C68.48 45.35, 87.5 24.25, 107.61 0.79 M51.19 65.7 C62.73 51.66, 73.57 37.59, 107.61 0.79 M56.17 66.06 C74.66 41.62, 97.44 18.62, 112.59 1.15 M56.17 66.06 C71.05 49.46, 84.82 33.14, 112.59 1.15 M61.82 65.66 C78.47 47.22, 97.78 23.7, 118.24 0.76 M61.82 65.66 C75.71 50.37, 89.66 34.67, 118.24 0.76 M66.8 66.02 C79.36 50.67, 94.27 34.13, 123.22 1.12 M66.8 66.02 C88.2 42.87, 108.29 19.48, 123.22 1.12 M71.79 66.38 C88.77 49.26, 103.62 27.23, 128.87 0.72 M71.79 66.38 C94.51 40.02, 115.31 14.1, 128.87 0.72 M77.43 65.99 C88.37 52.78, 102.75 40.69, 133.85 1.08 M77.43 65.99 C96.37 45.12, 115.33 22.06, 133.85 1.08 M82.42 66.35 C102.38 44.76, 117.41 24.49, 139.5 0.69 M82.42 66.35 C100.1 45.39, 117.22 24.84, 139.5 0.69 M88.06 65.95 C107.62 40.34, 127.33 17.96, 144.48 1.05 M88.06 65.95 C102.41 49.43, 115.63 33.01, 144.48 1.05 M93.05 66.31 C113.73 45.02, 132.35 23.14, 149.47 1.41 M93.05 66.31 C112.17 44.23, 130.51 22.93, 149.47 1.41 M98.69 65.92 C115.07 42.68, 136.57 22.33, 155.11 1.01 M98.69 65.92 C115.14 49.39, 130.33 31.17, 155.11 1.01 M103.68 66.28 C121.56 44.84, 138.68 25.13, 160.1 1.37 M103.68 66.28 C122.63 44.79, 142.05 21.54, 160.1 1.37 M109.32 65.88 C125.38 46.33, 140.23 30.31, 165.74 0.98 M109.32 65.88 C128.16 45.8, 145.68 24.01, 165.74 0.98 M114.31 66.24 C132.26 46.85, 145.95 28.34, 170.73 1.34 M114.31 66.24 C132.13 47.05, 148.06 28.08, 170.73 1.34 M119.95 65.85 C138.74 45.43, 156.25 27.85, 176.38 0.94 M119.95 65.85 C140.34 42.36, 160.65 20.24, 176.38 0.94 M124.94 66.21 C142.35 47.33, 158.36 31.44, 181.36 1.3 M124.94 66.21 C140.24 48.68, 155.08 31.37, 181.36 1.3 M130.58 65.81 C144.07 48.37, 158.67 30.03, 187.01 0.91 M130.58 65.81 C149.25 42.74, 168.48 20.99, 187.01 0.91 M135.57 66.17 C159.26 41.69, 179.31 15.59, 191.99 1.27 M135.57 66.17 C150.49 49.41, 165.44 31.12, 191.99 1.27 M141.21 65.78 C162.29 43.42, 178.75 19.67, 197.64 0.88 M141.21 65.78 C155.07 50.61, 168.53 35.75, 197.64 0.88 M146.2 66.14 C165.7 41.84, 189.16 15.74, 202.62 1.24 M146.2 66.14 C167.67 42.65, 189.96 16.67, 202.62 1.24 M151.84 65.75 C167.86 46.73, 185.37 27.29, 208.27 0.84 M151.84 65.75 C165.31 50.43, 178.32 36.18, 208.27 0.84 M156.83 66.11 C178.03 42.46, 200.84 15.06, 213.25 1.2 M156.83 66.11 C168.17 51.84, 180.35 38.18, 213.25 1.2 M162.47 65.71 C183.96 42.46, 204.63 18.33, 218.9 0.81 M162.47 65.71 C179.49 45.59, 197.63 26.31, 218.9 0.81 M167.46 66.07 C179.79 50.21, 192.15 34, 223.88 1.17 M167.46 66.07 C184.02 46.68, 200.28 28.08, 223.88 1.17 M172.45 66.43 C193.47 42.51, 214.64 17.53, 229.53 0.77 M172.45 66.43 C188.11 49.05, 201.6 32.86, 229.53 0.77 M178.09 66.04 C190.79 52.8, 200.03 39.47, 234.51 1.13 M178.09 66.04 C191.55 51.19, 206.44 33.94, 234.51 1.13 M183.08 66.4 C195.88 50.92, 211.29 35.51, 240.16 0.74 M183.08 66.4 C199.62 47.22, 216.96 27.02, 240.16 0.74 M188.72 66 C211.97 40.93, 231.56 17.7, 245.14 1.1 M188.72 66 C211.52 40.93, 233.02 13.8, 245.14 1.1 M193.71 66.36 C210.78 48.38, 228.56 26.58, 250.13 1.46 M193.71 66.36 C206.21 49.93, 221.38 35.46, 250.13 1.46 M199.35 65.97 C214.96 47.87, 230.93 31.82, 255.77 1.06 M199.35 65.97 C221.62 40.19, 243.14 16.26, 255.77 1.06 M204.34 66.33 C226.7 41.51, 249.45 14.75, 260.76 1.42 M204.34 66.33 C221.79 47.13, 238.82 26.84, 260.76 1.42 M209.98 65.93 C224.83 45.48, 245.22 26.98, 266.4 1.03 M209.98 65.93 C220 52.83, 232.84 38.4, 266.4 1.03 M214.97 66.29 C227.88 50.7, 242.92 32.13, 271.39 1.39 M214.97 66.29 C233.72 44.04, 253.08 22.15, 271.39 1.39 M220.61 65.9 C234.51 47.09, 251.81 32.08, 277.03 0.99 M220.61 65.9 C236.1 48.32, 253.2 28.9, 277.03 0.99 M225.6 66.26 C239.31 51.01, 251.36 40.23, 282.02 1.35 M225.6 66.26 C247.57 41.78, 267.11 17.48, 282.02 1.35 M231.24 65.86 C243.44 51.54, 258.17 35.95, 287.66 0.96 M231.24 65.86 C242.25 51.36, 254.95 38.41, 287.66 0.96 M236.23 66.22 C256.1 43.3, 274.64 21.18, 292.65 1.32 M236.23 66.22 C252.23 47.35, 267.5 30.13, 292.65 1.32 M241.87 65.83 C256.68 48.81, 273.65 29.89, 298.29 0.92 M241.87 65.83 C257.27 49.27, 270.28 31.2, 298.29 0.92 M246.86 66.19 C260.35 50.46, 271.69 38.96, 303.28 1.28 M246.86 66.19 C264.5 45.76, 280.79 27.88, 303.28 1.28 M252.5 65.8 C275.12 39.73, 296.47 16.43, 308.92 0.89 M252.5 65.8 C266.69 50.89, 279.5 35.89, 308.92 0.89 M257.49 66.16 C278.58 42.66, 300.94 17.8, 313.91 1.25 M257.49 66.16 C274.01 45.99, 293.35 24.82, 313.91 1.25 M263.13 65.76 C280.9 46.39, 299.09 23.69, 319.55 0.86 M263.13 65.76 C276.8 50.87, 290.04 35.82, 319.55 0.86 M268.12 66.12 C283.74 46.54, 301.47 29.12, 324.54 1.22 M268.12 66.12 C280.92 51.64, 291.59 38.04, 324.54 1.22 M273.11 66.48 C289.14 47.98, 306.02 30.06, 330.18 0.82 M273.11 66.48 C287.89 47.86, 303.45 31.05, 330.18 0.82 M278.75 66.09 C299.91 41.14, 322.58 13.72, 335.17 1.18 M278.75 66.09 C294.08 48.83, 308.7 29.61, 335.17 1.18 M283.74 66.45 C304.18 41.73, 323.23 21.18, 340.16 1.54 M283.74 66.45 C298.36 50.61, 312.57 33.36, 340.16 1.54 M289.38 66.05 C302.16 50.12, 314.05 34.8, 345.8 1.15 M289.38 66.05 C300.42 51.88, 313.64 39.1, 345.8 1.15 M294.37 66.41 C307.19 50.37, 320.36 36.25, 350.79 1.51 M294.37 66.41 C306.92 52.82, 320.74 36.24, 350.79 1.51 M300.01 66.02 C318.11 44.41, 337.31 23.75, 356.43 1.11 M300.01 66.02 C318.73 44.67, 336.6 23.1, 356.43 1.11 M305 66.38 C318.96 50.02, 332.09 33.1, 361.42 1.47 M305 66.38 C322.92 45.36, 342.45 23.02, 361.42 1.47 M310.64 65.98 C326.39 47.57, 342.27 30.4, 367.06 1.08 M310.64 65.98 C328.67 43.74, 348.34 22.2, 367.06 1.08 M315.63 66.34 C328.33 52.29, 339.77 39.18, 372.05 1.44 M315.63 66.34 C327.99 52.87, 337.94 39.46, 372.05 1.44 M321.27 65.95 C338.17 46.65, 355.93 26.94, 377.69 1.04 M321.27 65.95 C341.69 41.69, 363.48 17.49, 377.69 1.04 M326.26 66.31 C345.83 43.09, 365.28 21.63, 382.68 1.4 M326.26 66.31 C337.58 52.77, 350.3 39.04, 382.68 1.4 M331.9 65.91 C349.49 46.99, 365.5 30.73, 388.32 1.01 M331.9 65.91 C353.37 41.83, 375.04 16.12, 388.32 1.01 M336.89 66.27 C355.13 45.8, 370.1 27.34, 393.31 1.37 M336.89 66.27 C349.92 51.6, 360.73 39.21, 393.31 1.37 M342.53 65.88 C360.68 46.09, 376 24.62, 397.64 2.48 M342.53 65.88 C359.54 47.06, 375.39 27.68, 397.64 2.48 M347.52 66.24 C368.39 39.93, 389.82 16.38, 402.63 2.84 M347.52 66.24 C368.04 41.21, 391.14 16.26, 402.63 2.84 M353.16 65.84 C374.78 41.84, 393.68 17.84, 404.99 6.22 M353.16 65.84 C367.57 48.95, 383.53 31.57, 404.99 6.22 M358.15 66.2 C377.18 43.2, 393.29 26, 407.35 9.6 M358.15 66.2 C371.28 52.13, 382.11 38.63, 407.35 9.6 M363.13 66.57 C375.44 52.55, 386.53 40.88, 407.75 15.24 M363.13 66.57 C377.53 50.51, 390.67 34.82, 407.75 15.24 M368.78 66.17 C377.59 58.39, 385.24 48.55, 407.48 21.64 M368.78 66.17 C382.47 49.64, 397.01 35.11, 407.48 21.64 M373.76 66.53 C384.76 51.22, 399.11 35.35, 407.88 27.29 M373.76 66.53 C385.43 52.77, 396.09 38.2, 407.88 27.29 M379.41 66.14 C388.29 53.64, 398.97 41.31, 407.62 33.68 M379.41 66.14 C385.94 57.95, 392.28 51.64, 407.62 33.68 M384.39 66.5 C394.01 56.05, 400.33 46.71, 408.01 39.33 M384.39 66.5 C393.52 56.95, 402.21 45.47, 408.01 39.33 M390.04 66.1 C395.06 59.68, 400.14 56.49, 407.75 45.72 M390.04 66.1 C395.28 60.21, 399.67 53.7, 407.75 45.72 M394.37 67.22 C396.48 61.6, 402.31 60.72, 408.15 51.37 M394.37 67.22 C399.78 61.7, 406.23 54.75, 408.15 51.37" stroke="#868e96" stroke-width="0.5" fill="none"></path><path d="M16.25 0 M16.25 0 C109.24 1.04, 205.07 0.11, 391.75 0 M16.25 0 C104.81 -1.06, 192.84 -1.07, 391.75 0 M391.75 0 C402.09 -1.43, 408.33 7.16, 408 16.25 M391.75 0 C402.06 0.51, 409.27 3.39, 408 16.25 M408 16.25 C407.36 24.63, 406.66 31.74, 408 48.75 M408 16.25 C406.8 25.47, 408.74 36.8, 408 48.75 M408 48.75 C409.04 60.18, 401.84 64.71, 391.75 65 M408 48.75 C407.9 60.99, 402.74 67.26, 391.75 65 M391.75 65 C256.88 62.98, 119.07 63.55, 16.25 65 M391.75 65 C246.46 67.25, 100.49 66.9, 16.25 65 M16.25 65 C6.52 66.77, 0.96 60.38, 0 48.75 M16.25 65 C6.81 65.01, -0.53 59.72, 0 48.75 M0 48.75 C-1.24 39.7, 2.08 31.2, 0 16.25 M0 48.75 C0.41 39.97, 0.7 31.35, 0 16.25 M0 16.25 C-0.36 4.72, 5.29 -1.33, 16.25 0 M0 16.25 C1.69 5.2, 3.15 2.24, 16.25 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1590.1763933671728 512.6499256342397) rotate(0 113.06988525390625 12)"><text x="113.06988525390625" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">RuntimeVersionerService</text></g><g stroke-linecap="round" transform="translate(1501.7445487207042 575.9128890043116) rotate(0 202 29.5)"><path d="M3.93 3.41 C3.93 3.41, 3.93 3.41, 3.93 3.41 M3.93 3.41 C3.93 3.41, 3.93 3.41, 3.93 3.41 M3.01 10.57 C4.93 7.1, 7.61 4.32, 10.23 2.26 M3.01 10.57 C4.76 8.65, 5.54 6.84, 10.23 2.26 M2.75 16.96 C6.15 12.94, 9.47 7.09, 16.52 1.11 M2.75 16.96 C7.05 13.49, 9.42 9.11, 16.52 1.11 M2.49 23.36 C10.89 13.82, 14.12 8.71, 21.51 1.47 M2.49 23.36 C7.15 18.7, 10.72 13.2, 21.51 1.47 M2.88 29 C7.24 23.04, 15.73 16.02, 27.15 1.08 M2.88 29 C7.89 22.26, 14.12 16.33, 27.15 1.08 M3.27 34.65 C10.12 26.32, 19.45 16.56, 32.14 1.44 M3.27 34.65 C13.76 22.82, 21.66 12.43, 32.14 1.44 M3.01 41.04 C14.56 27.47, 25.99 11.23, 37.78 1.05 M3.01 41.04 C15.82 26.03, 27.83 12.34, 37.78 1.05 M3.41 46.69 C12.3 36.23, 19.96 27.42, 42.77 1.41 M3.41 46.69 C12.33 35.55, 21.21 25.33, 42.77 1.41 M3.15 53.09 C16.36 36.71, 29.06 22.67, 48.41 1.01 M3.15 53.09 C20.6 32.02, 38.21 12.33, 48.41 1.01 M6.17 55.71 C19.41 37.63, 32.25 22.21, 53.4 1.37 M6.17 55.71 C18.31 41.95, 29.51 29.79, 53.4 1.37 M8.53 59.09 C28.09 38.62, 42.42 17.26, 58.39 1.73 M8.53 59.09 C24.68 40.95, 42.25 21.82, 58.39 1.73 M12.86 60.2 C24.41 44.52, 37.85 30.83, 64.03 1.34 M12.86 60.2 C26.42 45.43, 38.38 29.33, 64.03 1.34 M18.5 59.81 C31.62 41.02, 47.4 23.56, 69.02 1.7 M18.5 59.81 C34.41 41.02, 49.06 23.9, 69.02 1.7 M23.49 60.17 C37.58 40.18, 53.81 22.59, 74.66 1.3 M23.49 60.17 C32.61 47.09, 44.48 35.72, 74.66 1.3 M28.48 60.53 C45.61 43.85, 57.68 27.42, 79.65 1.66 M28.48 60.53 C39.54 47.24, 48.72 36.92, 79.65 1.66 M34.12 60.13 C45.84 46.35, 58.64 31.68, 85.29 1.27 M34.12 60.13 C44.19 48.73, 54.58 36.9, 85.29 1.27 M39.11 60.49 C58.99 39.61, 78 15.85, 90.28 1.63 M39.11 60.49 C58.23 38.77, 74.93 18.5, 90.28 1.63 M44.75 60.1 C64.91 37.46, 84.11 16.5, 95.92 1.23 M44.75 60.1 C64.23 36.59, 83.75 12.82, 95.92 1.23 M49.74 60.46 C62.25 48.18, 70.6 34.01, 100.91 1.59 M49.74 60.46 C61.24 46.3, 73.86 32.76, 100.91 1.59 M55.38 60.07 C75.88 37.35, 91.64 17.69, 106.55 1.2 M55.38 60.07 C71.06 40.37, 89.07 22.64, 106.55 1.2 M60.37 60.43 C77.82 41.62, 90.52 23.99, 111.54 1.56 M60.37 60.43 C71.87 46.49, 83.54 33.47, 111.54 1.56 M66.01 60.03 C78.42 44.41, 88.64 32.36, 117.18 1.16 M66.01 60.03 C84.48 39.53, 101.53 18.64, 117.18 1.16 M71 60.39 C93.09 35.7, 112.84 14.29, 122.17 1.52 M71 60.39 C81.12 48.94, 91.56 35.26, 122.17 1.52 M76.64 60 C90.38 40.6, 108.15 27.17, 127.81 1.13 M76.64 60 C87.98 44.96, 100.57 30.32, 127.81 1.13 M81.63 60.36 C97.99 44.98, 111.28 27.82, 132.8 1.49 M81.63 60.36 C101.35 37.12, 119.73 15.38, 132.8 1.49 M87.27 59.96 C97.76 45.33, 111.59 31.02, 138.44 1.09 M87.27 59.96 C106.19 37.12, 126.96 15.85, 138.44 1.09 M92.26 60.32 C111.01 37.41, 130.92 17.14, 143.43 1.46 M92.26 60.32 C103.56 46.17, 116.15 32.46, 143.43 1.46 M97.9 59.93 C113.57 42.94, 128.88 24.15, 148.42 1.82 M97.9 59.93 C114.26 40.89, 132.31 21.63, 148.42 1.82 M102.89 60.29 C124.28 37.71, 141.6 14.31, 154.06 1.42 M102.89 60.29 C113.63 47.73, 123.35 37.01, 154.06 1.42 M108.53 59.89 C119.47 46.78, 133.73 32.14, 159.05 1.78 M108.53 59.89 C124.34 41.03, 140.09 22.28, 159.05 1.78 M113.52 60.25 C132.25 38.82, 152.04 15.64, 164.69 1.39 M113.52 60.25 C123.5 47.4, 133.79 35.48, 164.69 1.39 M119.16 59.86 C135.58 38.84, 155.14 16.28, 169.68 1.75 M119.16 59.86 C134.5 41.87, 150.23 23.04, 169.68 1.75 M124.15 60.22 C141.69 43.12, 157.12 22.32, 175.32 1.35 M124.15 60.22 C137.88 45.38, 151.32 29.42, 175.32 1.35 M129.13 60.58 C146.26 44.41, 159.43 25.08, 180.31 1.71 M129.13 60.58 C146.2 40.93, 164 20.25, 180.31 1.71 M134.78 60.18 C153.83 35.98, 176.49 14.3, 185.95 1.32 M134.78 60.18 C152.01 40.24, 170.66 21.09, 185.95 1.32 M139.76 60.54 C152.1 45.42, 165.83 30.96, 190.94 1.68 M139.76 60.54 C151.74 46.01, 164.13 33.44, 190.94 1.68 M145.41 60.15 C155.88 44.81, 169.16 32.52, 196.58 1.28 M145.41 60.15 C159.65 44.92, 173.42 27.47, 196.58 1.28 M150.39 60.51 C170.18 37.28, 189.8 16.46, 201.57 1.64 M150.39 60.51 C162.11 46.87, 172.58 34.32, 201.57 1.64 M156.04 60.12 C174.06 39.6, 190.17 21.05, 207.21 1.25 M156.04 60.12 C167.91 47.27, 179.42 33.23, 207.21 1.25 M161.02 60.48 C175.68 42.54, 187.15 28.54, 212.2 1.61 M161.02 60.48 C177.39 41.55, 195.48 21.34, 212.2 1.61 M166.67 60.08 C188.02 36.2, 207.12 12.47, 217.84 1.21 M166.67 60.08 C181.28 44.4, 194.17 28.65, 217.84 1.21 M171.65 60.44 C189.94 36.79, 211.75 13.62, 222.83 1.57 M171.65 60.44 C182.45 47.61, 194.71 34.46, 222.83 1.57 M177.3 60.05 C196.51 37.36, 213.04 18.13, 228.47 1.18 M177.3 60.05 C196.4 37.91, 215.73 14.8, 228.47 1.18 M182.28 60.41 C195.49 43.79, 208.71 26.88, 233.46 1.54 M182.28 60.41 C201.48 38.47, 217.96 18.87, 233.46 1.54 M187.93 60.01 C200.19 47.58, 210.93 34.73, 239.1 1.14 M187.93 60.01 C202.1 43.49, 216.46 27, 239.1 1.14 M192.91 60.37 C203.86 47.34, 216.02 33.92, 244.09 1.5 M192.91 60.37 C212.64 35.76, 233.34 13.47, 244.09 1.5 M198.56 59.98 C209.97 46.46, 220.22 34.06, 249.07 1.86 M198.56 59.98 C215.74 42.41, 230.37 23.56, 249.07 1.86 M203.54 60.34 C215.66 48.08, 229.47 34.46, 254.72 1.47 M203.54 60.34 C214.72 48.25, 227.39 33.01, 254.72 1.47 M209.19 59.94 C222.16 44.39, 233.77 33.38, 259.7 1.83 M209.19 59.94 C220.42 46.79, 231.51 34.6, 259.7 1.83 M214.17 60.3 C232.02 42.81, 244.33 25.87, 265.35 1.44 M214.17 60.3 C232.46 39.33, 248.73 19.99, 265.35 1.44 M219.16 60.66 C238.57 36.7, 260.01 15, 270.33 1.8 M219.16 60.66 C230.62 47.38, 241.76 34.55, 270.33 1.8 M224.8 60.27 C243.74 41.38, 260.52 20.09, 275.98 1.4 M224.8 60.27 C237.12 46.37, 246.82 34.19, 275.98 1.4 M229.79 60.63 C241.78 49.99, 253.16 35.85, 280.96 1.76 M229.79 60.63 C240.38 46.85, 252.62 33.98, 280.96 1.76 M235.44 60.23 C247.29 47.77, 254.59 36.66, 286.61 1.37 M235.44 60.23 C255.91 37.81, 275.73 14.03, 286.61 1.37 M240.42 60.59 C256.99 40.3, 273.78 22.49, 291.59 1.73 M240.42 60.59 C255.6 43.31, 270.41 27.03, 291.59 1.73 M246.07 60.2 C259.56 45.74, 273.16 29.09, 297.24 1.33 M246.07 60.2 C262.57 38.93, 280.37 18.11, 297.24 1.33 M251.05 60.56 C270.22 39.79, 288.02 16.24, 302.22 1.69 M251.05 60.56 C268.87 40.67, 288.38 18.1, 302.22 1.69 M256.7 60.17 C270.03 45.5, 283.72 26.14, 307.87 1.3 M256.7 60.17 C268.3 46.73, 281.27 31.88, 307.87 1.3 M261.68 60.53 C283.71 36.72, 302.78 14.59, 312.85 1.66 M261.68 60.53 C276.49 44.22, 289.84 29.33, 312.85 1.66 M267.33 60.13 C283.53 40.76, 296.51 26.03, 318.5 1.26 M267.33 60.13 C287.07 38.4, 305.48 15.62, 318.5 1.26 M272.31 60.49 C291.31 40.91, 306.42 19.97, 323.48 1.62 M272.31 60.49 C291.63 37.33, 312.03 13.98, 323.48 1.62 M277.96 60.1 C297.4 39.4, 313.26 18.63, 329.13 1.23 M277.96 60.1 C294.68 41.26, 312.39 23.32, 329.13 1.23 M282.94 60.46 C302.5 37.05, 321.25 17.53, 334.12 1.59 M282.94 60.46 C292.94 46.8, 305.13 36.12, 334.12 1.59 M288.59 60.06 C309.46 37.37, 327.14 15.39, 339.76 1.19 M288.59 60.06 C300.16 46.22, 310.81 33.67, 339.76 1.19 M293.57 60.42 C306.5 43.16, 320.93 28.87, 344.75 1.55 M293.57 60.42 C310.31 43.23, 324.27 25.35, 344.75 1.55 M299.22 60.03 C316.22 40.93, 328.77 23.81, 349.73 1.91 M299.22 60.03 C312.32 44.46, 324.66 30.27, 349.73 1.91 M304.2 60.39 C321.22 41.37, 336.32 23.81, 355.38 1.52 M304.2 60.39 C322.37 38.63, 340.81 18.1, 355.38 1.52 M309.85 59.99 C326.53 39.27, 345.14 19.58, 360.36 1.88 M309.85 59.99 C328.67 38.43, 346.22 17.72, 360.36 1.88 M314.83 60.35 C332.29 41.12, 349.69 22.59, 366.01 1.49 M314.83 60.35 C335.14 37.9, 354.28 14.46, 366.01 1.49 M319.82 60.71 C333.53 45.36, 345.67 27.55, 370.99 1.85 M319.82 60.71 C336.23 40.11, 354.46 20.44, 370.99 1.85 M325.46 60.32 C341.58 42.55, 355.53 27.79, 376.64 1.45 M325.46 60.32 C340.79 43.3, 356.74 26.24, 376.64 1.45 M330.45 60.68 C341.91 48.04, 350.25 35.05, 381.62 1.81 M330.45 60.68 C351.26 37.56, 370.83 14.68, 381.62 1.81 M336.09 60.28 C353.38 43.74, 369.16 23.76, 387.27 1.42 M336.09 60.28 C351.67 41.91, 367.28 24.85, 387.27 1.42 M341.08 60.64 C351.73 47.56, 361.95 33.34, 392.91 1.02 M341.08 60.64 C355.91 43.55, 372.61 24.28, 392.91 1.02 M346.72 60.25 C356.76 45.23, 370.06 32.62, 397.24 2.14 M346.72 60.25 C363.39 40.27, 382.12 19.52, 397.24 2.14 M351.71 60.61 C365.2 44.13, 383.53 28.72, 400.91 4.01 M351.71 60.61 C366.98 44.32, 382.09 27.3, 400.91 4.01 M357.35 60.21 C370.27 43.72, 382.9 29.74, 403.28 7.39 M357.35 60.21 C369.61 46.74, 381.49 34.11, 403.28 7.39 M362.34 60.57 C377.78 43.8, 394.19 22.59, 405.64 10.76 M362.34 60.57 C377.88 42.35, 393.96 23.65, 405.64 10.76 M367.98 60.18 C382.92 44.69, 394.82 29.97, 406.03 16.41 M367.98 60.18 C382.08 43.98, 395.73 27.05, 406.03 16.41 M372.97 60.54 C384.56 48.4, 397.17 31.98, 405.77 22.8 M372.97 60.54 C386.14 47.04, 398.85 31.7, 405.77 22.8 M378.61 60.15 C389.07 46.62, 399.98 38.91, 406.17 28.45 M378.61 60.15 C388.32 46.94, 400.38 36.29, 406.17 28.45 M383.6 60.51 C388.67 52.68, 392.64 48.26, 405.91 34.85 M383.6 60.51 C391.36 52.07, 399.91 43.63, 405.91 34.85 M389.9 59.36 C392.63 54.9, 400.73 49.72, 406.3 40.49 M389.9 59.36 C393.96 54.01, 397.3 50.21, 406.3 40.49 M396.2 58.21 C399.91 55.36, 402.39 51.99, 404.73 48.4 M396.2 58.21 C398.85 54.42, 401.3 51.99, 404.73 48.4" stroke="#12b886" stroke-width="0.5" fill="none"></path><path d="M14.75 0 M14.75 0 C135.47 2.74, 256.04 1.28, 389.25 0 M14.75 0 C95.4 -1.5, 174.7 -1.31, 389.25 0 M389.25 0 C399.04 -1.4, 405.62 6.79, 404 14.75 M389.25 0 C399.19 -1.5, 405.01 6.3, 404 14.75 M404 14.75 C405.13 25.01, 402.73 36.58, 404 44.25 M404 14.75 C404.49 25.29, 403.43 37.58, 404 44.25 M404 44.25 C403.64 53.45, 399.34 57.66, 389.25 59 M404 44.25 C403.14 54.36, 397.44 56.96, 389.25 59 M389.25 59 C313.28 61.86, 237 60.77, 14.75 59 M389.25 59 C282.31 61, 175.66 60.37, 14.75 59 M14.75 59 C3.99 60.77, -1.58 52.76, 0 44.25 M14.75 59 C4.17 61.02, -0.43 56.12, 0 44.25 M0 44.25 C-0.5 34.95, 1.66 24.41, 0 14.75 M0 44.25 C-0.41 34.29, -0.22 23.64, 0 14.75 M0 14.75 C-0.96 3.49, 5.5 1.89, 14.75 0 M0 14.75 C1.96 4.13, 4.16 1, 14.75 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1583.9346503442393 593.4128890043116) rotate(0 119.80989837646484 12)"><text x="119.80989837646484" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">ContainerManagerService</text></g><g stroke-linecap="round" transform="translate(1504.4597237670987 662.3667855446823) rotate(0 204 32.5)"><path d="M4.33 3.76 C4.33 3.76, 4.33 3.76, 4.33 3.76 M4.33 3.76 C4.33 3.76, 4.33 3.76, 4.33 3.76 M3.41 10.91 C5.18 8, 8 5.79, 10.62 2.61 M3.41 10.91 C6.33 8.05, 8.73 4.2, 10.62 2.61 M2.49 18.07 C4.58 11.73, 10.46 9.34, 17.58 0.71 M2.49 18.07 C6.03 14.98, 8.32 11.58, 17.58 0.71 M2.23 24.46 C8.67 16.48, 15.2 7.85, 22.57 1.07 M2.23 24.46 C7.55 17.88, 12.91 13.52, 22.57 1.07 M2.62 30.11 C8.52 22.69, 14.89 13.67, 28.21 0.67 M2.62 30.11 C9.54 22.67, 16.33 14.54, 28.21 0.67 M3.02 35.75 C10.7 24.06, 21.86 15.98, 33.2 1.03 M3.02 35.75 C15.83 21.13, 27.65 8.13, 33.2 1.03 M2.76 42.15 C16.42 24.79, 29.8 10.83, 38.84 0.64 M2.76 42.15 C10.88 30.58, 21.23 19.71, 38.84 0.64 M3.15 47.79 C14.22 35.67, 24.86 23.19, 43.83 1 M3.15 47.79 C14.23 35.02, 25.32 20.1, 43.83 1 M1.58 55.7 C19.54 36.19, 37.08 15.09, 48.81 1.36 M1.58 55.7 C15.17 38.75, 31.11 20.68, 48.81 1.36 M3.94 59.08 C17.65 41.55, 32.54 26.7, 54.46 0.96 M3.94 59.08 C15.99 45.28, 26.71 31.77, 54.46 0.96 M6.3 62.45 C22.98 46.92, 38.25 26.2, 59.44 1.32 M6.3 62.45 C26.59 41.65, 43.67 18.42, 59.44 1.32 M9.32 65.08 C24.26 46.09, 43.75 25.48, 65.09 0.93 M9.32 65.08 C29.09 42.02, 49.86 19.11, 65.09 0.93 M13.65 66.19 C26.45 49.93, 39.75 38.73, 70.07 1.29 M13.65 66.19 C24.51 51.61, 37.03 39.13, 70.07 1.29 M19.3 65.8 C30.28 52.71, 43.62 36.25, 75.72 0.89 M19.3 65.8 C34.33 48.12, 49.72 31.49, 75.72 0.89 M24.28 66.16 C38.76 50.9, 50.77 34.93, 80.7 1.25 M24.28 66.16 C38.48 47.5, 54.43 30.12, 80.7 1.25 M29.93 65.76 C53.44 42.51, 73.36 15.22, 86.35 0.86 M29.93 65.76 C51.55 40.8, 74.05 15.73, 86.35 0.86 M34.91 66.12 C50.36 45.62, 67.18 30.88, 91.33 1.22 M34.91 66.12 C48.93 50.66, 61.51 34.94, 91.33 1.22 M40.56 65.73 C57.32 45.21, 76.51 25.51, 96.98 0.83 M40.56 65.73 C61.5 41.79, 83.31 17.88, 96.98 0.83 M45.54 66.09 C60.73 49.48, 73.85 35.2, 101.96 1.19 M45.54 66.09 C66.98 40.07, 88.97 16.05, 101.96 1.19 M51.19 65.7 C72.7 41.67, 91.31 19.08, 107.61 0.79 M51.19 65.7 C70.64 45.84, 87.3 23.72, 107.61 0.79 M56.17 66.06 C74.67 45.65, 88.42 26.24, 112.59 1.15 M56.17 66.06 C75.11 45.55, 92.64 24.54, 112.59 1.15 M61.82 65.66 C78.37 46.01, 96.66 25.52, 118.24 0.76 M61.82 65.66 C72.77 52.28, 85.17 38.93, 118.24 0.76 M66.8 66.02 C83.24 43.43, 104.38 25.31, 123.22 1.12 M66.8 66.02 C89.1 40.31, 110.45 15.79, 123.22 1.12 M71.79 66.38 C88.83 45.92, 104.46 31.16, 128.87 0.72 M71.79 66.38 C94.49 40.43, 116.98 15.28, 128.87 0.72 M77.43 65.99 C95.42 44.28, 113.31 24.81, 133.85 1.08 M77.43 65.99 C92.07 50.64, 107.47 34.11, 133.85 1.08 M82.42 66.35 C97.21 47.41, 115.96 27.68, 139.5 0.69 M82.42 66.35 C99.84 45.92, 120.29 25.08, 139.5 0.69 M88.06 65.95 C110.28 39.59, 133.08 13.94, 144.48 1.05 M88.06 65.95 C107.14 42.55, 127.56 20.43, 144.48 1.05 M93.05 66.31 C108.74 46.39, 127.35 29.63, 149.47 1.41 M93.05 66.31 C108.69 48, 123.59 31.88, 149.47 1.41 M98.69 65.92 C117.08 41.72, 137.33 19.75, 155.11 1.01 M98.69 65.92 C110.82 50.4, 124.45 34.8, 155.11 1.01 M103.68 66.28 C119.41 47.45, 136.53 29.2, 160.1 1.37 M103.68 66.28 C121.19 45.38, 139.95 25.76, 160.1 1.37 M109.32 65.88 C126.37 46.22, 144.44 26.41, 165.74 0.98 M109.32 65.88 C121.47 50.54, 134.57 36.2, 165.74 0.98 M114.31 66.24 C131.76 47.28, 149.9 28.71, 170.73 1.34 M114.31 66.24 C128.31 49.87, 141.34 34.9, 170.73 1.34 M119.95 65.85 C139.41 42.41, 156.41 23.73, 176.38 0.94 M119.95 65.85 C140.24 42.42, 161.43 18.56, 176.38 0.94 M124.94 66.21 C142.21 45.93, 162.45 23.02, 181.36 1.3 M124.94 66.21 C137.78 52.54, 150.62 36.92, 181.36 1.3 M130.58 65.81 C149.64 41.87, 168.27 23.67, 187.01 0.91 M130.58 65.81 C143.69 50.27, 157.1 35.38, 187.01 0.91 M135.57 66.17 C159.59 41.82, 180.15 14.76, 191.99 1.27 M135.57 66.17 C151.41 47.31, 168.68 27.62, 191.99 1.27 M141.21 65.78 C163.18 39.23, 183.94 16.12, 197.64 0.88 M141.21 65.78 C162.79 42.94, 182.08 18.73, 197.64 0.88 M146.2 66.14 C163.22 45.29, 182.63 24.83, 202.62 1.24 M146.2 66.14 C164.33 46.33, 181.76 24.34, 202.62 1.24 M151.84 65.75 C165.63 52.97, 180.4 35.71, 208.27 0.84 M151.84 65.75 C164.18 52.7, 175.81 37.88, 208.27 0.84 M156.83 66.11 C176.67 41.43, 198.93 16.33, 213.25 1.2 M156.83 66.11 C169.7 51.55, 183.43 36.66, 213.25 1.2 M162.47 65.71 C185.15 41.69, 203.21 21.12, 218.9 0.81 M162.47 65.71 C185.45 39.49, 208 14.72, 218.9 0.81 M167.46 66.07 C189.39 40.78, 211.93 17.15, 223.88 1.17 M167.46 66.07 C188.57 41.37, 209.01 18.93, 223.88 1.17 M172.45 66.43 C192.44 45.53, 208.48 24.15, 229.53 0.77 M172.45 66.43 C188.75 49.3, 203.06 31.76, 229.53 0.77 M178.09 66.04 C188.56 52.81, 204.72 39.53, 234.51 1.13 M178.09 66.04 C195.47 45.94, 215.45 23.58, 234.51 1.13 M183.08 66.4 C196 48.29, 212.09 34.01, 240.16 0.74 M183.08 66.4 C201.41 47.02, 219.44 25.89, 240.16 0.74 M188.72 66 C208.93 45.92, 228.39 21.04, 245.14 1.1 M188.72 66 C206.84 43.42, 226.75 22.48, 245.14 1.1 M193.71 66.36 C213.17 45.16, 231.61 25.44, 250.13 1.46 M193.71 66.36 C212.55 43.19, 233.39 21.21, 250.13 1.46 M199.35 65.97 C214.57 52.34, 225.43 33.43, 255.77 1.06 M199.35 65.97 C215.13 46.34, 231.66 28.17, 255.77 1.06 M204.34 66.33 C216.88 49.57, 230.64 37.14, 260.76 1.42 M204.34 66.33 C225.05 42.2, 244.41 18.96, 260.76 1.42 M209.98 65.93 C233.08 38.91, 252.83 15.5, 266.4 1.03 M209.98 65.93 C225.43 47.18, 242.13 28.53, 266.4 1.03 M214.97 66.29 C229.47 53.64, 240.5 39.96, 271.39 1.39 M214.97 66.29 C230.49 49.34, 245.68 32.84, 271.39 1.39 M220.61 65.9 C235.99 44.91, 253.58 28.82, 277.03 0.99 M220.61 65.9 C236.28 47.29, 251.57 29.65, 277.03 0.99 M225.6 66.26 C248.41 40.83, 268.65 15.74, 282.02 1.35 M225.6 66.26 C241.97 47.59, 260.35 28.92, 282.02 1.35 M231.24 65.86 C247.07 48.07, 262.63 30.86, 287.66 0.96 M231.24 65.86 C246.7 48.34, 263.17 28.33, 287.66 0.96 M236.23 66.22 C257.81 38.21, 279.55 15.81, 292.65 1.32 M236.23 66.22 C251.34 48.94, 265.37 33.07, 292.65 1.32 M241.87 65.83 C253.26 53.92, 266.03 38.24, 298.29 0.92 M241.87 65.83 C259.89 45.89, 277.59 23.58, 298.29 0.92 M246.86 66.19 C267.89 39.65, 290.48 15.23, 303.28 1.28 M246.86 66.19 C263.65 48.68, 279.15 29.37, 303.28 1.28 M252.5 65.8 C267.81 47.08, 281.95 31.09, 308.92 0.89 M252.5 65.8 C267.65 46.54, 284.09 28.83, 308.92 0.89 M257.49 66.16 C269.61 53.78, 281.09 40.7, 313.91 1.25 M257.49 66.16 C270.12 51.01, 284.57 34.65, 313.91 1.25 M263.13 65.76 C285.11 44.11, 304.96 19.86, 319.55 0.86 M263.13 65.76 C278.41 47.19, 293.14 29.16, 319.55 0.86 M268.12 66.12 C282.51 50.81, 294.32 37.4, 324.54 1.22 M268.12 66.12 C288.43 44.07, 307.28 20.55, 324.54 1.22 M273.11 66.48 C286.92 52.38, 300.38 35.99, 330.18 0.82 M273.11 66.48 C289.46 48.07, 304.25 32.17, 330.18 0.82 M278.75 66.09 C298.42 44.68, 315.74 28.11, 335.17 1.18 M278.75 66.09 C298.47 42.35, 318.13 19.89, 335.17 1.18 M283.74 66.45 C297.17 48.55, 311.48 35.38, 340.16 1.54 M283.74 66.45 C298.75 49.25, 312.21 32.4, 340.16 1.54 M289.38 66.05 C303.88 51.08, 318.52 33.24, 345.8 1.15 M289.38 66.05 C309.3 42.31, 329.42 20.35, 345.8 1.15 M294.37 66.41 C315.27 41.7, 333.89 20.63, 350.79 1.51 M294.37 66.41 C310.58 48.76, 324.46 29.59, 350.79 1.51 M300.01 66.02 C319.33 43.28, 340.44 19.73, 356.43 1.11 M300.01 66.02 C311.44 52.15, 323.25 39.19, 356.43 1.11 M305 66.38 C325.78 44.8, 343.02 21.3, 361.42 1.47 M305 66.38 C317.94 51.81, 328.46 38.74, 361.42 1.47 M310.64 65.98 C332.19 42.3, 352.61 16.07, 367.06 1.08 M310.64 65.98 C326.59 49.54, 340.42 33.05, 367.06 1.08 M315.63 66.34 C337.81 42.48, 360.08 17.9, 372.05 1.44 M315.63 66.34 C326.58 54.52, 337.18 40.98, 372.05 1.44 M321.27 65.95 C339.76 48.16, 356.19 28.72, 377.69 1.04 M321.27 65.95 C334.66 49.02, 349.7 33.21, 377.69 1.04 M326.26 66.31 C340.84 46.6, 358.61 29.49, 382.68 1.4 M326.26 66.31 C347.59 39.75, 370.22 14.5, 382.68 1.4 M331.9 65.91 C346.2 46.97, 362.62 32.09, 388.32 1.01 M331.9 65.91 C344 52.42, 356.57 37.17, 388.32 1.01 M336.89 66.27 C348.73 52.25, 361.63 37.8, 393.31 1.37 M336.89 66.27 C358.83 41, 381.39 15.5, 393.31 1.37 M342.53 65.88 C358.61 50.21, 371.57 31.25, 397.64 2.48 M342.53 65.88 C363.24 42.82, 385.17 18.12, 397.64 2.48 M347.52 66.24 C368.58 43.81, 391.41 15.83, 402.63 2.84 M347.52 66.24 C361.15 52.67, 373.05 37.77, 402.63 2.84 M353.16 65.84 C370.19 45.61, 384.9 26.48, 404.99 6.22 M353.16 65.84 C367.2 48.71, 380.45 33.55, 404.99 6.22 M358.15 66.2 C372.32 53.41, 384.65 37.42, 407.35 9.6 M358.15 66.2 C367.09 53.82, 377.49 41.54, 407.35 9.6 M363.13 66.57 C378.43 49.32, 397.07 27.06, 407.75 15.24 M363.13 66.57 C379.57 47.2, 395.91 28.23, 407.75 15.24 M368.78 66.17 C379.31 56.21, 384.82 47.49, 407.48 21.64 M368.78 66.17 C378.03 55.73, 388.4 43.82, 407.48 21.64 M373.76 66.53 C382.52 56.48, 392.95 44.74, 407.88 27.29 M373.76 66.53 C383.37 54.75, 392.37 44.8, 407.88 27.29 M379.41 66.14 C389.48 56.91, 397.27 42.84, 407.62 33.68 M379.41 66.14 C388.11 56.76, 398.41 45.99, 407.62 33.68 M384.39 66.5 C388.03 62.67, 396.38 56.9, 408.01 39.33 M384.39 66.5 C393.34 56.09, 402.28 45.3, 408.01 39.33 M390.04 66.1 C393.77 62.13, 398.09 56.08, 407.75 45.72 M390.04 66.1 C394.19 60.78, 397.42 56.75, 407.75 45.72 M394.37 67.22 C396.46 63.26, 400.04 61.17, 408.15 51.37 M394.37 67.22 C399.57 60.53, 403.98 55.47, 408.15 51.37" stroke="#fd7e14" stroke-width="0.5" fill="none"></path><path d="M16.25 0 M16.25 0 C156.95 0.08, 296.79 -0.72, 391.75 0 M16.25 0 C95.2 0.03, 173.24 -0.65, 391.75 0 M391.75 0 C404.2 0.8, 409.13 3.89, 408 16.25 M391.75 0 C401.37 -1.58, 408.51 5.63, 408 16.25 M408 16.25 C407.35 22.97, 409.97 30.31, 408 48.75 M408 16.25 C407.52 26.39, 408.78 38.94, 408 48.75 M408 48.75 C406.14 57.62, 402.16 63.47, 391.75 65 M408 48.75 C407.4 59.11, 402.52 65.17, 391.75 65 M391.75 65 C303.08 65.12, 213.53 67.05, 16.25 65 M391.75 65 C288.14 64.38, 183.7 64.78, 16.25 65 M16.25 65 C4.24 66.16, 0.59 60.41, 0 48.75 M16.25 65 C3.53 62.89, 1.9 58.56, 0 48.75 M0 48.75 C0.88 37.96, 1.93 30.1, 0 16.25 M0 48.75 C0.68 38.6, 0.6 30.41, 0 16.25 M0 16.25 C1.02 6.09, 6.89 0.15, 16.25 0 M0 16.25 C2.04 3.32, 7.08 -0.86, 16.25 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1577.0598366821378 682.8667855446823) rotate(0 131.39988708496094 12)"><text x="131.39988708496094" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">PodSandboxManagerService</text></g><g stroke-linecap="round" transform="translate(1507.221700204594 750.7502644486505) rotate(0 204.00000000000023 32.5)"><path d="M4.33 3.76 C4.33 3.76, 4.33 3.76, 4.33 3.76 M4.33 3.76 C4.33 3.76, 4.33 3.76, 4.33 3.76 M3.41 10.91 C4.56 7.73, 7.42 5.5, 10.62 2.61 M3.41 10.91 C6.67 8.17, 9.03 4.25, 10.62 2.61 M2.49 18.07 C7.99 10.2, 15.63 3.02, 17.58 0.71 M2.49 18.07 C6.53 12.16, 11.23 9.04, 17.58 0.71 M2.23 24.46 C7.43 19.51, 10.29 13.74, 22.57 1.07 M2.23 24.46 C8.13 17, 15.28 9.96, 22.57 1.07 M2.62 30.11 C11.71 19.13, 23.5 7.93, 28.21 0.67 M2.62 30.11 C9.6 22.54, 15.51 16.15, 28.21 0.67 M3.02 35.75 C14 20.5, 25.4 9.94, 33.2 1.03 M3.02 35.75 C14.52 22.9, 24.54 10.78, 33.2 1.03 M2.76 42.15 C16.22 30.7, 24.76 14.98, 38.84 0.64 M2.76 42.15 C14.39 29.27, 23.61 16.86, 38.84 0.64 M3.15 47.79 C17.4 33.87, 28.74 19.12, 43.83 1 M3.15 47.79 C15.11 33.55, 28.01 18.99, 43.83 1 M1.58 55.7 C10.28 44.64, 21.72 33.31, 48.81 1.36 M1.58 55.7 C15.85 37.22, 32.57 21.08, 48.81 1.36 M3.94 59.08 C24.97 35.94, 43.56 14.65, 54.46 0.96 M3.94 59.08 C18.69 41.64, 32.68 27.17, 54.46 0.96 M6.3 62.45 C27.52 38.03, 48.41 15.44, 59.44 1.32 M6.3 62.45 C23.23 42.3, 40.1 23.45, 59.44 1.32 M9.32 65.08 C23.58 50.85, 39.99 34.36, 65.09 0.93 M9.32 65.08 C24.22 46.79, 41.24 28.12, 65.09 0.93 M13.65 66.19 C29.95 46.44, 52.09 26.32, 70.07 1.29 M13.65 66.19 C35.97 40.28, 58.27 14.92, 70.07 1.29 M19.3 65.8 C38.01 41.75, 59.19 20.11, 75.72 0.89 M19.3 65.8 C35.35 46.76, 52.76 29.06, 75.72 0.89 M24.28 66.16 C40.52 47.05, 55.54 31.82, 80.7 1.25 M24.28 66.16 C43.46 42.67, 63.28 20.52, 80.7 1.25 M29.93 65.76 C41.32 50.42, 55.09 34.73, 86.35 0.86 M29.93 65.76 C46.22 46.68, 63.04 28.05, 86.35 0.86 M34.91 66.12 C52.15 44.72, 71.78 26.06, 91.33 1.22 M34.91 66.12 C52.29 46.18, 70.2 26.07, 91.33 1.22 M40.56 65.73 C52.28 50.13, 65.5 36.38, 96.98 0.83 M40.56 65.73 C57.87 46.38, 75.46 27.29, 96.98 0.83 M45.54 66.09 C60.16 48.87, 72.73 34.49, 101.96 1.19 M45.54 66.09 C64.65 43.63, 82.5 23.54, 101.96 1.19 M51.19 65.7 C71 42.17, 92.73 18.21, 107.61 0.79 M51.19 65.7 C69.16 44.64, 88.54 22.54, 107.61 0.79 M56.17 66.06 C69.58 52.83, 82.83 35.91, 112.59 1.15 M56.17 66.06 C75.43 43.03, 94.26 22.99, 112.59 1.15 M61.82 65.66 C75.14 49.96, 88.76 35.34, 118.24 0.76 M61.82 65.66 C84.98 40.48, 106.33 14.2, 118.24 0.76 M66.8 66.02 C82.57 47.18, 100.67 26.7, 123.22 1.12 M66.8 66.02 C88.84 40.33, 110.06 16.3, 123.22 1.12 M71.79 66.38 C94.38 43.44, 112.89 18.12, 128.87 0.72 M71.79 66.38 C89.79 45.28, 108.97 24.15, 128.87 0.72 M77.43 65.99 C95.72 47.12, 112.29 24.19, 133.85 1.08 M77.43 65.99 C91.14 52.23, 105.13 36, 133.85 1.08 M82.42 66.35 C95.36 54.04, 106.78 38.01, 139.5 0.69 M82.42 66.35 C103.32 41.51, 125.29 16.49, 139.5 0.69 M88.06 65.95 C100.75 52.09, 115.12 37.34, 144.48 1.05 M88.06 65.95 C109.76 42.47, 128.87 20.59, 144.48 1.05 M93.05 66.31 C116.68 39.55, 139.3 15.87, 149.47 1.41 M93.05 66.31 C115.07 40.91, 137.2 16.59, 149.47 1.41 M98.69 65.92 C120.34 40.35, 140.47 19.32, 155.11 1.01 M98.69 65.92 C117.7 44.74, 135.04 23.33, 155.11 1.01 M103.68 66.28 C120.56 50.05, 133.77 32.67, 160.1 1.37 M103.68 66.28 C115.14 52.6, 129.36 38.9, 160.1 1.37 M109.32 65.88 C125.78 46.33, 147.36 22.76, 165.74 0.98 M109.32 65.88 C123.07 48.7, 138.2 33.4, 165.74 0.98 M114.31 66.24 C132.77 47.92, 151.07 26.06, 170.73 1.34 M114.31 66.24 C134.05 45.4, 153.4 21.9, 170.73 1.34 M119.95 65.85 C137.31 42.91, 157.93 23.13, 176.38 0.94 M119.95 65.85 C138.77 44.86, 156.93 24.72, 176.38 0.94 M124.94 66.21 C142.79 42.53, 164.73 21.26, 181.36 1.3 M124.94 66.21 C139.45 51.62, 151.51 34.54, 181.36 1.3 M130.58 65.81 C146.35 45.68, 163.21 27.99, 187.01 0.91 M130.58 65.81 C143.08 49.71, 156.45 36.03, 187.01 0.91 M135.57 66.17 C156.77 41.81, 174.92 18.98, 191.99 1.27 M135.57 66.17 C158.12 40.03, 178.92 15.76, 191.99 1.27 M141.21 65.78 C156 46.66, 173.48 27.66, 197.64 0.88 M141.21 65.78 C154.74 52.85, 166.22 39.15, 197.64 0.88 M146.2 66.14 C162.34 49.26, 177.9 33.31, 202.62 1.24 M146.2 66.14 C162.37 45.81, 179.5 28.19, 202.62 1.24 M151.84 65.75 C167.88 47.07, 182.6 30.05, 208.27 0.84 M151.84 65.75 C174.05 40.32, 195.13 15.17, 208.27 0.84 M156.83 66.11 C172.49 47.73, 192.53 29.62, 213.25 1.2 M156.83 66.11 C172.35 48.46, 187.63 31.23, 213.25 1.2 M162.47 65.71 C177.27 48.92, 194.31 27.19, 218.9 0.81 M162.47 65.71 C184.54 38.65, 206.61 14.56, 218.9 0.81 M167.46 66.07 C183.09 48.47, 196.61 33.47, 223.88 1.17 M167.46 66.07 C178.96 53.18, 191.42 38.36, 223.88 1.17 M172.45 66.43 C190.19 46.99, 207.65 23.01, 229.53 0.77 M172.45 66.43 C194.02 40.58, 216.4 15.63, 229.53 0.77 M178.09 66.04 C195.28 49.47, 210.46 29.04, 234.51 1.13 M178.09 66.04 C193.02 48.18, 207.67 31.47, 234.51 1.13 M183.08 66.4 C197.71 46.03, 215.01 28.52, 240.16 0.74 M183.08 66.4 C194.76 53.22, 206.37 39.93, 240.16 0.74 M188.72 66 C200.47 50.99, 216.01 33.65, 245.14 1.1 M188.72 66 C210.07 43.53, 230.18 19.69, 245.14 1.1 M193.71 66.36 C209.18 47.12, 223.2 28.81, 250.13 1.46 M193.71 66.36 C207.22 51.35, 219.35 37.49, 250.13 1.46 M199.35 65.97 C220.64 44.65, 239 19.84, 255.77 1.06 M199.35 65.97 C213.1 51.4, 226.44 35.6, 255.77 1.06 M204.34 66.33 C221.27 47.51, 235.63 33.21, 260.76 1.42 M204.34 66.33 C223.14 45.82, 240.53 27.5, 260.76 1.42 M209.98 65.93 C230.04 41.85, 249.5 19.78, 266.4 1.03 M209.98 65.93 C223.57 48.76, 237.78 34.06, 266.4 1.03 M214.97 66.29 C230.68 48.68, 243.46 31.27, 271.39 1.39 M214.97 66.29 C229.34 50.49, 243.79 33.36, 271.39 1.39 M220.61 65.9 C240.23 41.34, 260.11 20.85, 277.03 0.99 M220.61 65.9 C240.99 41.66, 260.54 19.34, 277.03 0.99 M225.6 66.26 C242.59 49.31, 254.85 28.82, 282.02 1.35 M225.6 66.26 C245.22 43.55, 265.5 20.6, 282.02 1.35 M231.24 65.86 C242.28 51.03, 254.33 38.44, 287.66 0.96 M231.24 65.86 C251.6 44.02, 270.06 20.95, 287.66 0.96 M236.23 66.22 C250.02 51.56, 258.77 39.57, 292.65 1.32 M236.23 66.22 C257.79 41.84, 278.88 16.22, 292.65 1.32 M241.87 65.83 C258.57 49.7, 271.45 33.71, 298.29 0.92 M241.87 65.83 C263.65 41.51, 285.75 16.6, 298.29 0.92 M246.86 66.19 C257.69 55.11, 267.5 41.12, 303.28 1.28 M246.86 66.19 C264.53 48.11, 280.86 29.05, 303.28 1.28 M252.5 65.8 C265.73 48.4, 281.8 32.9, 308.92 0.89 M252.5 65.8 C267.32 46.9, 283.94 29.44, 308.92 0.89 M257.49 66.16 C278.74 38.77, 302.02 13.56, 313.91 1.25 M257.49 66.16 C272.33 48.1, 288.22 31.83, 313.91 1.25 M263.13 65.76 C275 52.89, 287.57 36.85, 319.55 0.86 M263.13 65.76 C274.8 52.12, 287.05 38.22, 319.55 0.86 M268.12 66.12 C290.18 40.46, 313.6 14.4, 324.54 1.22 M268.12 66.12 C284.03 49.44, 298.27 31.03, 324.54 1.22 M273.11 66.48 C294.02 43.5, 317.51 17.53, 330.18 0.82 M273.11 66.48 C295.19 42.18, 318.21 14.84, 330.18 0.82 M278.75 66.09 C293.4 52.98, 304.8 37.43, 335.17 1.18 M278.75 66.09 C297.19 44.82, 314.23 24.12, 335.17 1.18 M283.74 66.45 C298.97 47.04, 312.71 31.39, 340.16 1.54 M283.74 66.45 C298.71 50.92, 313.01 33.56, 340.16 1.54 M289.38 66.05 C298.8 51.82, 310.71 37.49, 345.8 1.15 M289.38 66.05 C309.38 42.76, 331.22 17.23, 345.8 1.15 M294.37 66.41 C315.41 41.55, 336.22 17.28, 350.79 1.51 M294.37 66.41 C308.21 51.76, 319.2 37.92, 350.79 1.51 M300.01 66.02 C313.23 51.17, 328.63 33.49, 356.43 1.11 M300.01 66.02 C314.82 48.91, 330.87 30.57, 356.43 1.11 M305 66.38 C320.85 46.69, 335.54 30.62, 361.42 1.47 M305 66.38 C323.71 45.64, 341.6 22.51, 361.42 1.47 M310.64 65.98 C328.05 47.23, 348.64 25.69, 367.06 1.08 M310.64 65.98 C322.29 53.41, 335.89 39.69, 367.06 1.08 M315.63 66.34 C337.01 41.48, 358.44 15.83, 372.05 1.44 M315.63 66.34 C328.75 51.77, 341.77 36.26, 372.05 1.44 M321.27 65.95 C334.37 49.96, 345.59 36.43, 377.69 1.04 M321.27 65.95 C334.76 49.81, 348.96 34.48, 377.69 1.04 M326.26 66.31 C347.79 40.06, 367.63 17.25, 382.68 1.4 M326.26 66.31 C347.51 42.58, 368.6 17.77, 382.68 1.4 M331.9 65.91 C344.96 52.72, 354.87 36.9, 388.32 1.01 M331.9 65.91 C352.37 40.35, 373.93 16.38, 388.32 1.01 M336.89 66.27 C353.16 45.54, 371.15 28.09, 393.31 1.37 M336.89 66.27 C349.59 50.26, 361.48 36.7, 393.31 1.37 M342.53 65.88 C362.57 42.03, 382.34 20.29, 397.64 2.48 M342.53 65.88 C358.34 47.84, 375.87 27.73, 397.64 2.48 M347.52 66.24 C365.4 43.65, 386.79 21.56, 402.63 2.84 M347.52 66.24 C362.77 49.61, 377.6 31.25, 402.63 2.84 M353.16 65.84 C362.94 53.44, 374.25 42.09, 404.99 6.22 M353.16 65.84 C363.15 53.02, 373.49 41.99, 404.99 6.22 M358.15 66.2 C370.36 53.5, 382.18 39.88, 407.35 9.6 M358.15 66.2 C377.42 43.54, 397.01 22.6, 407.35 9.6 M363.13 66.57 C376.13 50.37, 393.08 35.32, 407.75 15.24 M363.13 66.57 C373.1 55.98, 384.03 44.24, 407.75 15.24 M368.78 66.17 C377.24 55.53, 384.94 45.39, 407.48 21.64 M368.78 66.17 C380.54 53.4, 390.91 40.88, 407.48 21.64 M373.76 66.53 C385.74 55.93, 395.15 43.89, 407.88 27.29 M373.76 66.53 C386.1 52.36, 398.42 38.08, 407.88 27.29 M379.41 66.14 C387.32 56.7, 396.48 47.58, 407.62 33.68 M379.41 66.14 C386.48 58.96, 392.72 50.95, 407.62 33.68 M384.39 66.5 C387.73 60.52, 392.86 55.36, 408.01 39.33 M384.39 66.5 C393.49 56.72, 401.68 46.59, 408.01 39.33 M390.04 66.1 C394.29 60.55, 398.29 57.93, 407.75 45.72 M390.04 66.1 C396.15 58.2, 402.05 50.89, 407.75 45.72 M394.37 67.22 C400.52 60.71, 403.7 56.79, 408.15 51.37 M394.37 67.22 C399.07 62.39, 403.09 57.61, 408.15 51.37" stroke="#82c91e" stroke-width="0.5" fill="none"></path><path d="M16.25 0 M16.25 0 C141.56 0.43, 265.22 1.21, 391.75 0 M16.25 0 C94.91 2.42, 172.4 1.66, 391.75 0 M391.75 0 C401.21 0.23, 408.53 7.16, 408 16.25 M391.75 0 C400.36 -1.6, 408.38 4.47, 408 16.25 M408 16.25 C409.22 26.95, 406.6 42.59, 408 48.75 M408 16.25 C407.06 23.42, 407.09 31.63, 408 48.75 M408 48.75 C409.51 60.38, 404 66.92, 391.75 65 M408 48.75 C409.79 59.6, 400.95 65.39, 391.75 65 M391.75 65 C270.73 66.71, 149.94 64.75, 16.25 65 M391.75 65 C254.65 63.73, 118.42 63.86, 16.25 65 M16.25 65 C5.26 63.76, -1.29 60.77, 0 48.75 M16.25 65 C3.6 66.19, -1.64 58.48, 0 48.75 M0 48.75 C-0.39 39.9, 1.56 27.8, 0 16.25 M0 48.75 C0.23 39.75, -0.4 31.03, 0 16.25 M0 16.25 C2 5.14, 4.01 0.29, 16.25 0 M0 16.25 C2.07 3.54, 7.17 0.95, 16.25 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1600.2417960297894 771.2502644486505) rotate(0 110.97990417480469 12)"><text x="110.97990417480469" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">RuntimeManagerService</text></g><g stroke-linecap="round" transform="translate(1023.5478928169787 1236.6599232674805) rotate(0 150.5 45.5)"><path d="M2.94 9.18 C2.94 9.18, 2.94 9.18, 2.94 9.18 M2.94 9.18 C2.94 9.18, 2.94 9.18, 2.94 9.18 M0.71 17.84 C3.93 13.21, 4.62 12.03, 11.21 5.77 M0.71 17.84 C3.76 14.06, 8.77 8.82, 11.21 5.77 M0.45 24.24 C5.6 18.24, 6.84 16.05, 18.16 3.87 M0.45 24.24 C6.04 15.62, 13.14 8.77, 18.16 3.87 M0.85 29.89 C8.83 19.18, 15.79 12.66, 25.12 1.96 M0.85 29.89 C5.07 25.18, 9.56 18.25, 25.12 1.96 M0.58 36.28 C8.32 26.22, 21.03 14.87, 30.11 2.32 M0.58 36.28 C7.76 27.59, 14.83 20.23, 30.11 2.32 M0.98 41.93 C9.82 33.87, 16.55 23.9, 35.75 1.93 M0.98 41.93 C8.72 32.76, 17.84 23.13, 35.75 1.93 M0.72 48.32 C13.44 32.24, 28.39 14.72, 40.74 2.29 M0.72 48.32 C9.44 39.16, 16.68 29.78, 40.74 2.29 M0.46 54.72 C15.65 35.7, 33.46 18.46, 46.38 1.89 M0.46 54.72 C15.75 36.66, 31.81 18.47, 46.38 1.89 M0.85 60.36 C16.82 43.8, 32.61 26.31, 51.37 2.25 M0.85 60.36 C20.51 38.58, 40.17 15.02, 51.37 2.25 M0.59 66.76 C16.05 50.65, 28.96 35.12, 57.01 1.86 M0.59 66.76 C22.13 41.4, 45.11 14.89, 57.01 1.86 M0.98 72.41 C13.92 57.45, 29.55 39.92, 62 2.22 M0.98 72.41 C20.21 51.98, 39.27 29.55, 62 2.22 M3.35 75.78 C18.97 58.92, 36.73 40.58, 67.64 1.82 M3.35 75.78 C22.89 52.25, 43.26 30.97, 67.64 1.82 M5.05 79.92 C32.66 47.09, 58.16 18.58, 72.63 2.18 M5.05 79.92 C23.08 58.95, 40.54 39.74, 72.63 2.18 M6.76 84.05 C33.74 54.26, 56.9 23.98, 78.27 1.79 M6.76 84.05 C31.47 53.89, 55.68 25.82, 78.27 1.79 M8.47 88.19 C37.47 56.35, 67.41 23.69, 83.26 2.15 M8.47 88.19 C37.26 53.65, 66.21 22.57, 83.26 2.15 M12.14 90.06 C41.2 57.75, 68.64 24.41, 88.9 1.75 M12.14 90.06 C32.84 66.55, 52.03 44.25, 88.9 1.75 M15.82 91.92 C37.71 65.58, 58.84 41.18, 93.89 2.11 M15.82 91.92 C33.23 73.28, 48.46 53.59, 93.89 2.11 M20.15 93.04 C52.1 59.13, 82.98 25.29, 99.53 1.72 M20.15 93.04 C45.9 64.22, 71.44 35.78, 99.53 1.72 M25.13 93.4 C40.25 75.51, 54.5 54.33, 104.52 2.08 M25.13 93.4 C48.14 65.4, 72.44 38.78, 104.52 2.08 M30.78 93 C52.95 69.03, 73.5 41.83, 110.16 1.69 M30.78 93 C52.47 66.26, 76.13 38.68, 110.16 1.69 M35.76 93.37 C61.87 66.9, 83.44 39.61, 115.15 2.05 M35.76 93.37 C51.58 72.99, 68.4 54.38, 115.15 2.05 M41.41 92.97 C57.13 72.85, 74.88 54.71, 120.79 1.65 M41.41 92.97 C65.06 67.87, 86.59 41.79, 120.79 1.65 M46.39 93.33 C75.29 62.02, 103.95 31.22, 125.78 2.01 M46.39 93.33 C70.84 66.2, 92.89 41.58, 125.78 2.01 M52.04 92.94 C82.91 60.73, 110.72 25.05, 130.76 2.37 M52.04 92.94 C72.5 69.11, 94.55 45.08, 130.76 2.37 M57.02 93.3 C74 72.5, 91.91 54.32, 136.41 1.98 M57.02 93.3 C82.84 62.78, 109.94 32.49, 136.41 1.98 M62.67 92.9 C92.81 57.56, 123.18 20.95, 141.39 2.34 M62.67 92.9 C91.26 61.36, 119.57 28.33, 141.39 2.34 M67.65 93.26 C91.78 60.67, 120.39 33.39, 147.04 1.94 M67.65 93.26 C91.7 66.57, 116.4 37.13, 147.04 1.94 M73.3 92.87 C100.35 63.61, 125.71 32.47, 152.02 2.3 M73.3 92.87 C103.23 57.55, 134.17 22.79, 152.02 2.3 M78.28 93.23 C98.85 70.66, 120.67 43.18, 157.67 1.91 M78.28 93.23 C97.25 72.97, 112.55 53.27, 157.67 1.91 M83.27 93.59 C107.25 65.16, 133.52 37.55, 162.65 2.27 M83.27 93.59 C104.05 71.24, 124.57 47.03, 162.65 2.27 M88.91 93.19 C113.92 64.6, 137.33 38.84, 168.3 1.87 M88.91 93.19 C111.22 68.69, 132.23 45.84, 168.3 1.87 M93.9 93.55 C112.69 71.44, 136.12 46.28, 173.28 2.23 M93.9 93.55 C110.12 74.59, 124.64 55.85, 173.28 2.23 M99.54 93.16 C123.93 67.52, 148.71 38.33, 178.93 1.84 M99.54 93.16 C119.3 68.92, 141.2 44.92, 178.93 1.84 M104.53 93.52 C135.36 57.36, 168.21 25.71, 183.92 2.2 M104.53 93.52 C134.01 59.6, 162.92 24.24, 183.92 2.2 M110.17 93.12 C130.15 70.06, 148.62 46.4, 189.56 1.8 M110.17 93.12 C128.39 71.36, 148.06 49.01, 189.56 1.8 M115.16 93.48 C140.42 61.72, 164.35 36.16, 194.55 2.16 M115.16 93.48 C133.23 72.31, 150.85 51.6, 194.55 2.16 M120.81 93.09 C142.03 67.7, 164.26 41.79, 200.19 1.77 M120.81 93.09 C139.43 72.85, 155.19 53.87, 200.19 1.77 M125.79 93.45 C151.45 62.07, 178.39 32.15, 205.18 2.13 M125.79 93.45 C150.22 66.05, 175.88 36.33, 205.18 2.13 M131.44 93.05 C151.98 72.5, 171.16 49.52, 210.82 1.73 M131.44 93.05 C151.4 71.01, 170.77 50.18, 210.82 1.73 M136.42 93.41 C161.99 62.08, 191.57 33.2, 215.81 2.09 M136.42 93.41 C159.93 66.93, 184.25 37.91, 215.81 2.09 M142.07 93.02 C171.76 62.34, 201.24 27.31, 221.45 1.7 M142.07 93.02 C170.62 60.23, 200.06 25.55, 221.45 1.7 M147.05 93.38 C169.03 67.8, 189.89 43.42, 226.44 2.06 M147.05 93.38 C176.61 57.55, 206.93 22.81, 226.44 2.06 M152.7 92.99 C184.55 60.27, 211.4 25.99, 232.08 1.67 M152.7 92.99 C171.55 70.18, 190.41 47.53, 232.08 1.67 M157.68 93.35 C184.29 59.81, 215.07 25.54, 237.07 2.03 M157.68 93.35 C176.25 71.5, 196.7 48.31, 237.07 2.03 M163.33 92.95 C192.98 60.27, 222.78 26.52, 242.05 2.39 M163.33 92.95 C191.4 60.88, 219.03 29.4, 242.05 2.39 M168.31 93.31 C185.51 71.82, 204.53 52.47, 247.7 1.99 M168.31 93.31 C190.71 67.46, 213.5 42.96, 247.7 1.99 M173.96 92.92 C204.73 57.72, 235.54 21.85, 252.68 2.35 M173.96 92.92 C202.81 62.21, 229.38 28.66, 252.68 2.35 M178.94 93.28 C205.09 63.69, 231.05 32.53, 258.33 1.96 M178.94 93.28 C211.57 57.51, 243.07 20.55, 258.33 1.96 M184.59 92.88 C215.39 58.41, 246.4 23.56, 263.31 2.32 M184.59 92.88 C212.83 59.5, 242.23 26.69, 263.31 2.32 M189.57 93.24 C223.43 58.71, 253.12 20.79, 268.96 1.92 M189.57 93.24 C206.42 73.68, 225.07 52.36, 268.96 1.92 M195.22 92.85 C219 64.46, 240.03 38.65, 273.94 2.28 M195.22 92.85 C214.98 70.25, 234.92 49.13, 273.94 2.28 M200.2 93.21 C230.37 58.39, 260.06 22.29, 280.24 1.13 M200.2 93.21 C222.2 67.38, 243.89 42.77, 280.24 1.13 M205.19 93.57 C233.32 62.1, 261.93 29.12, 284.57 2.25 M205.19 93.57 C233.12 64.35, 258.63 31.62, 284.57 2.25 M210.83 93.17 C231.57 66.4, 256.98 39.89, 289.56 2.61 M210.83 93.17 C242.95 56.83, 272.21 22.17, 289.56 2.61 M215.82 93.53 C243.41 63.43, 269.31 32.05, 293.23 4.48 M215.82 93.53 C240.79 64.35, 267.35 32.56, 293.23 4.48 M221.46 93.14 C246.87 61.79, 272.99 31.08, 296.25 7.1 M221.46 93.14 C245.31 64.06, 269.73 37.36, 296.25 7.1 M226.45 93.5 C242.55 73.64, 258.42 56.81, 298.62 10.48 M226.45 93.5 C252.97 62.59, 280.37 31.29, 298.62 10.48 M232.09 93.1 C249.23 75.67, 265.46 52.74, 300.32 14.61 M232.09 93.1 C253.66 67.72, 275.46 42.07, 300.32 14.61 M237.08 93.46 C253.08 73.75, 270.2 57.26, 302.03 18.75 M237.08 93.46 C262.65 63.51, 287.99 34.41, 302.03 18.75 M242.72 93.07 C260.99 72.24, 278.52 54.26, 302.42 24.39 M242.72 93.07 C264.9 68.39, 284.52 43.88, 302.42 24.39 M247.71 93.43 C261.55 77.44, 272.77 62.07, 302.16 30.79 M247.71 93.43 C264.97 72.93, 283.22 53.08, 302.16 30.79 M253.35 93.04 C266.04 80.97, 276.59 65.54, 301.9 37.19 M253.35 93.04 C268.59 77.92, 281.5 61.48, 301.9 37.19 M258.34 93.4 C273.78 75.81, 290.28 57.24, 302.3 42.83 M258.34 93.4 C271.83 76.7, 286.48 59.77, 302.3 42.83 M263.98 93 C277.34 77.46, 286.52 65.07, 302.03 49.23 M263.98 93 C276.59 79.33, 286.74 64.49, 302.03 49.23 M268.97 93.36 C279.56 81.7, 286.21 74.49, 302.43 54.87 M268.97 93.36 C281.06 78.48, 293.02 63.99, 302.43 54.87 M274.61 92.97 C285.41 82.12, 291.43 72.39, 302.17 61.27 M274.61 92.97 C282.36 82.31, 292.14 71.82, 302.17 61.27 M280.26 92.57 C284.03 84.78, 290.13 79.95, 301.25 68.42 M280.26 92.57 C284.3 88.1, 289.69 82.15, 301.25 68.42 M286.56 91.42 C291.17 85, 296.44 79.61, 300.33 75.57 M286.56 91.42 C291.14 86.77, 295.9 81.22, 300.33 75.57" stroke="#82c91e" stroke-width="0.5" fill="none"></path><path d="M22.75 0 M22.75 0 C87.62 0.51, 150.85 0.13, 278.25 0 M22.75 0 C85.26 1.89, 146.63 1.96, 278.25 0 M278.25 0 C291.57 -1.63, 300.82 6.61, 301 22.75 M278.25 0 C294.81 -2.03, 301.15 5.33, 301 22.75 M301 22.75 C302.43 36.99, 299.16 55.71, 301 68.25 M301 22.75 C300.83 37.93, 300.44 53.46, 301 68.25 M301 68.25 C302.06 83.69, 292.47 92.15, 278.25 91 M301 68.25 C303.21 83.16, 294.35 92.52, 278.25 91 M278.25 91 C209.07 90.51, 139.7 91.08, 22.75 91 M278.25 91 C196.21 92.91, 113.49 92.51, 22.75 91 M22.75 91 C8.56 90.17, -1.25 82.26, 0 68.25 M22.75 91 C9.71 89.85, -2.21 81.75, 0 68.25 M0 68.25 C0.51 52.18, 0.21 34.06, 0 22.75 M0 68.25 C-0.17 51.11, -0.9 33.93, 0 22.75 M0 22.75 C0.73 6.9, 6.93 -0.3, 22.75 0 M0 22.75 C0.14 7.25, 6.34 -1.54, 22.75 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1063.067988642174 1270.1599232674805) rotate(0 110.97990417480469 12)"><text x="110.97990417480469" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">RuntimeManagerService</text></g><g stroke-linecap="round" transform="translate(1022.1494531298813 1609.6729397870579) rotate(0 142.5 41)"><path d="M2.98 7.89 C2.98 7.89, 2.98 7.89, 2.98 7.89 M2.98 7.89 C2.98 7.89, 2.98 7.89, 2.98 7.89 M0.75 16.55 C4.66 12.29, 7.88 6.85, 10.59 5.23 M0.75 16.55 C3.82 13.96, 5.54 11.13, 10.59 5.23 M1.15 22.2 C7.29 18.02, 11.93 11.16, 17.55 3.33 M1.15 22.2 C4.75 18.04, 8.77 12.04, 17.55 3.33 M0.89 28.6 C8.25 20.08, 18.13 7.64, 23.19 2.94 M0.89 28.6 C5.95 23.22, 12.04 17.13, 23.19 2.94 M1.28 34.24 C9.47 25.26, 17.26 14.81, 28.84 2.54 M1.28 34.24 C9.7 24.75, 15.68 17.63, 28.84 2.54 M1.02 40.64 C11.44 29.2, 20.26 19.31, 33.82 2.9 M1.02 40.64 C9.4 31.92, 16.62 23.01, 33.82 2.9 M0.76 47.03 C16.18 31.22, 31.27 14.19, 39.47 2.51 M0.76 47.03 C8.97 37.56, 17.58 27.45, 39.47 2.51 M1.15 52.68 C13.03 40.51, 21.64 26.67, 44.45 2.87 M1.15 52.68 C12.84 40.53, 22.71 27.41, 44.45 2.87 M0.89 59.08 C15.75 38.72, 33.07 19.73, 50.1 2.47 M0.89 59.08 C17.16 40.67, 35.07 21.89, 50.1 2.47 M1.29 64.72 C19.4 43.33, 36.14 23.87, 55.08 2.83 M1.29 64.72 C17.36 45.59, 34.47 26.78, 55.08 2.83 M2.99 68.85 C23.67 47.11, 42.8 22.76, 60.73 2.44 M2.99 68.85 C21.17 48.79, 35.96 29.93, 60.73 2.44 M3.39 74.49 C21.6 54.19, 40.03 33.65, 65.71 2.8 M3.39 74.49 C19 54.31, 36.31 36.4, 65.71 2.8 M6.41 77.12 C26.15 54.8, 44.58 34.32, 71.36 2.4 M6.41 77.12 C30.6 50.47, 52.2 24.97, 71.36 2.4 M9.42 79.74 C29.16 55.16, 49.55 31.82, 76.34 2.76 M9.42 79.74 C23.5 62.01, 39.97 44.15, 76.34 2.76 M13.1 81.61 C34.7 55.23, 59.3 30.41, 81.99 2.37 M13.1 81.61 C37.18 54.47, 60.85 27.59, 81.99 2.37 M17.43 82.73 C41.41 57.1, 59.96 31.13, 86.97 2.73 M17.43 82.73 C41.8 54.41, 65.89 25.8, 86.97 2.73 M22.42 83.09 C46.73 54.93, 74.33 26.11, 92.62 2.33 M22.42 83.09 C48.76 52.18, 75.91 20.88, 92.62 2.33 M27.4 83.45 C46.94 60.9, 69.21 36.93, 97.6 2.69 M27.4 83.45 C42.19 64.5, 57.27 46.97, 97.6 2.69 M33.05 83.05 C58.52 51.53, 84.73 21.78, 103.25 2.3 M33.05 83.05 C58.75 54.06, 84.94 23.55, 103.25 2.3 M38.03 83.41 C67.76 50.86, 93.9 18.57, 108.23 2.66 M38.03 83.41 C57.51 61.64, 75.42 41.27, 108.23 2.66 M43.02 83.77 C63.08 61.77, 82.32 37.77, 113.88 2.26 M43.02 83.77 C65.11 58.15, 87.02 33.4, 113.88 2.26 M48.66 83.38 C62.25 63.25, 79.3 47.23, 118.86 2.63 M48.66 83.38 C72.78 54.21, 95.7 27.32, 118.86 2.63 M53.65 83.74 C73.53 58.11, 97.77 36.27, 124.51 2.23 M53.65 83.74 C80.33 51.6, 107.68 20.16, 124.51 2.23 M59.29 83.34 C84.59 51.72, 111.17 24.25, 129.49 2.59 M59.29 83.34 C85.75 53.63, 111.86 22.77, 129.49 2.59 M64.28 83.7 C85.82 56.67, 110.5 28.24, 134.48 2.95 M64.28 83.7 C86.16 60.06, 106.6 35.89, 134.48 2.95 M69.92 83.31 C85.83 64.33, 102.53 47.48, 140.12 2.56 M69.92 83.31 C91.61 58.66, 112.92 32.31, 140.12 2.56 M74.91 83.67 C92.34 64.15, 108.45 44.02, 145.11 2.92 M74.91 83.67 C96.84 58.82, 117.66 34.83, 145.11 2.92 M80.55 83.28 C97.18 64.21, 111.43 46.37, 150.75 2.52 M80.55 83.28 C105.5 55.08, 127.75 29.22, 150.75 2.52 M85.54 83.64 C109.5 54.55, 132.74 26.83, 155.74 2.88 M85.54 83.64 C110.8 56.41, 134.9 27.34, 155.74 2.88 M91.19 83.24 C117.31 52.28, 141.97 21.68, 161.38 2.49 M91.19 83.24 C112.3 60.27, 132.72 36.42, 161.38 2.49 M96.17 83.6 C115.92 58.12, 140.41 33.66, 166.37 2.85 M96.17 83.6 C110.58 66.91, 125.51 50.98, 166.37 2.85 M101.82 83.21 C123.57 59.23, 143.48 33.48, 172.01 2.45 M101.82 83.21 C119.25 63.02, 136.48 43.81, 172.01 2.45 M106.8 83.57 C127.69 59.3, 147.73 36.58, 177 2.81 M106.8 83.57 C129.1 57.33, 150.03 31.45, 177 2.81 M112.45 83.17 C128.57 64.54, 149.36 43.44, 182.64 2.42 M112.45 83.17 C133 59.08, 154.91 33.93, 182.64 2.42 M117.43 83.53 C140.18 54.78, 165.88 25.2, 187.63 2.78 M117.43 83.53 C138.19 60.4, 158.63 36.53, 187.63 2.78 M123.08 83.14 C145.62 57.12, 166.3 36.06, 193.27 2.38 M123.08 83.14 C138.08 63.81, 153.54 47, 193.27 2.38 M128.06 83.5 C149.29 57.43, 168.19 34.79, 198.26 2.74 M128.06 83.5 C148.42 60.89, 169.13 37.43, 198.26 2.74 M133.71 83.1 C151.94 62.3, 173.53 37.62, 203.9 2.35 M133.71 83.1 C148.1 66.03, 162.95 50.09, 203.9 2.35 M138.69 83.46 C164.33 51.91, 195.14 21.29, 208.89 2.71 M138.69 83.46 C166.23 50.31, 194.07 19.86, 208.89 2.71 M144.34 83.07 C163.13 63.72, 181.35 41.82, 214.53 2.31 M144.34 83.07 C162.9 61.44, 181.73 39.02, 214.53 2.31 M149.32 83.43 C169.47 61.97, 183.57 45.49, 219.52 2.67 M149.32 83.43 C164.64 63.8, 181.33 45.26, 219.52 2.67 M154.31 83.79 C179.51 57.78, 203.68 28.12, 225.16 2.28 M154.31 83.79 C174.92 58.58, 197.09 34.1, 225.16 2.28 M159.95 83.39 C184.7 52.05, 212.03 21.45, 230.15 2.64 M159.95 83.39 C186.31 52.22, 213.44 21.84, 230.15 2.64 M164.94 83.75 C193.84 51.36, 218.84 20.57, 235.79 2.25 M164.94 83.75 C181.4 65.38, 196.25 47.51, 235.79 2.25 M170.58 83.36 C195.71 51.45, 224.2 21.96, 240.78 2.61 M170.58 83.36 C194.35 55.74, 218.01 27.68, 240.78 2.61 M175.57 83.72 C196.76 58.7, 218.04 35.44, 245.77 2.97 M175.57 83.72 C201.34 54.61, 228.69 22.95, 245.77 2.97 M181.21 83.33 C207.95 51.62, 237.29 18.89, 251.41 2.57 M181.21 83.33 C208.28 54.04, 234.1 23.52, 251.41 2.57 M186.2 83.69 C214.24 53.54, 239.15 20.76, 256.4 2.93 M186.2 83.69 C213 54.1, 239 23.44, 256.4 2.93 M191.84 83.29 C208.94 63.03, 228.21 40.58, 262.04 2.54 M191.84 83.29 C211.39 62.78, 229.23 40.23, 262.04 2.54 M196.83 83.65 C221.92 52.16, 249.73 21.38, 267.68 2.14 M196.83 83.65 C213.55 63.42, 230.31 45.4, 267.68 2.14 M202.47 83.26 C225.53 55.49, 247.12 31.29, 273.33 1.75 M202.47 83.26 C229.19 55.07, 254.37 25.15, 273.33 1.75 M207.46 83.62 C232.5 54.31, 260.13 23.72, 277 3.62 M207.46 83.62 C225.88 62.85, 242.75 42.3, 277 3.62 M213.1 83.22 C227.49 66.56, 243.22 46.51, 280.68 5.49 M213.1 83.22 C235.1 58.95, 255.3 34.89, 280.68 5.49 M218.09 83.58 C239.05 59.48, 262.17 30.79, 282.38 9.62 M218.09 83.58 C239.8 58.16, 262.71 33.58, 282.38 9.62 M223.73 83.19 C243.46 58.05, 264.58 34.41, 284.09 13.75 M223.73 83.19 C245.83 57.36, 267.28 32.88, 284.09 13.75 M228.72 83.55 C239.48 70.17, 254.21 54.35, 286.45 17.13 M228.72 83.55 C242.9 65.38, 259.39 49.15, 286.45 17.13 M234.36 83.15 C252.65 62.88, 270.12 40.89, 286.19 23.53 M234.36 83.15 C246.54 68.14, 260.56 55.19, 286.19 23.53 M239.35 83.51 C254.75 64.75, 268.4 48.19, 286.59 29.17 M239.35 83.51 C248.71 72.8, 258.73 60.15, 286.59 29.17 M244.99 83.12 C253.23 71.98, 262.85 61.22, 286.33 35.57 M244.99 83.12 C255.11 72.02, 265.71 58.57, 286.33 35.57 M249.98 83.48 C258.77 71.69, 269.82 62.49, 286.72 41.21 M249.98 83.48 C259.11 74.45, 266.81 63.39, 286.72 41.21 M255.62 83.08 C265.21 72.16, 273.41 63.03, 286.46 47.61 M255.62 83.08 C264.12 74.14, 269.59 66.43, 286.46 47.61 M260.61 83.44 C268.52 70.23, 278.86 60.24, 286.85 53.26 M260.61 83.44 C266.21 78.14, 271.7 70.77, 286.85 53.26 M266.25 83.05 C271 76.87, 277.87 68.97, 286.59 59.65 M266.25 83.05 C274.74 74.3, 281.72 66.76, 286.59 59.65 M272.55 81.9 C278.3 75.72, 279.18 70.82, 288.3 63.79 M272.55 81.9 C276.93 76.46, 280.03 71.89, 288.3 63.79" stroke="#12b886" stroke-width="0.5" fill="none"></path><path d="M20.5 0 M20.5 0 C109.8 -0.83, 195.74 -1.53, 264.5 0 M20.5 0 C109.8 -0.06, 198.15 -1.61, 264.5 0 M264.5 0 C278.31 -1.69, 284.75 7.34, 285 20.5 M264.5 0 C276.42 -1.5, 286.59 7.96, 285 20.5 M285 20.5 C284.45 32.04, 285.11 48.4, 285 61.5 M285 20.5 C284.69 29.87, 285.08 39.15, 285 61.5 M285 61.5 C283.47 74.92, 276.64 80.77, 264.5 82 M285 61.5 C284.4 76.56, 278.59 81.85, 264.5 82 M264.5 82 C203.4 80.72, 140.59 80.77, 20.5 82 M264.5 82 C187.14 83.53, 111.85 82.96, 20.5 82 M20.5 82 C6.9 83.38, -0.4 74.44, 0 61.5 M20.5 82 C6.58 82.49, 0 77.11, 0 61.5 M0 61.5 C-1.77 52.17, 0.87 41.35, 0 20.5 M0 61.5 C0.25 51.27, -0.24 41.4, 0 20.5 M0 20.5 C-0.96 7.25, 8.16 1.21, 20.5 0 M0 20.5 C-1.64 5.23, 8.34 1.05, 20.5 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1044.8395547534165 1638.6729397870579) rotate(0 119.80989837646484 12)"><text x="119.80989837646484" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">ContainerManagerService</text></g><g stroke-linecap="round"><g transform="translate(852.535030697732 479.57729013746695) rotate(0 84.13015588609574 382.47763787146175)"><path d="M0.05 0.19 C28.13 127.75, 141.38 638.15, 169.64 765.71 M-1.38 -0.75 C26.49 126.43, 140.59 636.53, 169.02 763.89" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(852.535030697732 479.57729013746695) rotate(0 84.13015588609574 382.47763787146175)"><path d="M153.18 739.65 C160.34 749.69, 166.48 759.64, 169.68 765.71 M151.99 739.59 C158.41 745.26, 161.05 752.52, 169.04 764.35" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(852.535030697732 479.57729013746695) rotate(0 84.13015588609574 382.47763787146175)"><path d="M173.21 735.17 C172.41 747, 170.58 758.73, 169.68 765.71 M172.02 735.11 C172.91 742, 170 750.5, 169.04 764.35" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round"><g transform="translate(830.4390861097081 487.7178540201925) rotate(0 94.98547702519727 580.4567629194348)"><path d="M0.11 0.86 C31.69 194.22, 157.45 966.01, 189.16 1159.35 M-1.3 0.26 C30.67 193.86, 159.45 967.51, 191.27 1160.65" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(830.4390861097081 487.7178540201925) rotate(0 94.98547702519727 580.4567629194348)"><path d="M177.97 1132.61 C179.57 1141.62, 186.11 1146.69, 190.84 1161.48 M176.33 1134.49 C182.86 1144.48, 188.23 1154.69, 190.71 1160.32" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(830.4390861097081 487.7178540201925) rotate(0 94.98547702519727 580.4567629194348)"><path d="M198.21 1129.25 C194.34 1139.18, 195.43 1145.16, 190.84 1161.48 M196.57 1131.14 C195.08 1142.44, 192.41 1153.99, 190.71 1160.32" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(1057.7211342550386 955.1351173936137) rotate(0 134.5 51.5)"><path d="M6.2 6.71 C6.2 6.71, 6.2 6.71, 6.2 6.71 M6.2 6.71 C6.2 6.71, 6.2 6.71, 6.2 6.71 M6.59 12.36 C8.28 11.57, 10.04 8.16, 15.12 2.54 M6.59 12.36 C8.66 9.73, 11.43 6.54, 15.12 2.54 M3.05 22.53 C11.78 14.33, 17.48 8.74, 21.42 1.4 M3.05 22.53 C8.7 16.29, 11.84 10.72, 21.42 1.4 M3.45 28.17 C10.06 19.7, 14.23 12.91, 27.06 1 M3.45 28.17 C12.71 18.4, 19.82 8.15, 27.06 1 M3.18 34.57 C13.81 24.21, 22.32 8.32, 32.05 1.36 M3.18 34.57 C12.59 23.64, 20.56 14.88, 32.05 1.36 M3.58 40.21 C14.41 25.95, 25.3 12.91, 37.69 0.97 M3.58 40.21 C16.31 26.25, 27.22 13.18, 37.69 0.97 M3.32 46.61 C13.28 32.61, 25.41 22.24, 42.68 1.33 M3.32 46.61 C15 32.43, 28.82 17.12, 42.68 1.33 M3.06 53.01 C14.61 39.22, 26.77 25.57, 48.32 0.93 M3.06 53.01 C19.6 34.46, 34.48 16.21, 48.32 0.93 M3.45 58.65 C18.03 40.53, 35.73 19.76, 53.31 1.29 M3.45 58.65 C17.65 42.72, 33.58 25.14, 53.31 1.29 M3.19 65.05 C18.57 46.42, 37.29 28.3, 58.95 0.9 M3.19 65.05 C16.84 49.43, 28.8 34.76, 58.95 0.9 M3.58 70.69 C15.77 57.21, 28.71 42.68, 63.94 1.26 M3.58 70.69 C19.33 50.91, 36.82 32.85, 63.94 1.26 M3.32 77.09 C21.47 56.92, 39.98 33.77, 69.58 0.86 M3.32 77.09 C23.94 52.02, 45.67 27.93, 69.58 0.86 M1.75 85 C21.74 63.28, 41.19 42.15, 74.57 1.22 M1.75 85 C19.71 63.08, 40.38 40.57, 74.57 1.22 M3.46 89.13 C31.07 57.13, 57.91 26.27, 80.21 0.83 M3.46 89.13 C25.51 64.24, 47.74 37.4, 80.21 0.83 M4.51 94.02 C24.04 72.88, 45.02 48.81, 85.2 1.19 M4.51 94.02 C30.03 64.28, 54.36 34.96, 85.2 1.19 M6.87 97.4 C34.11 69.14, 60.08 38.9, 90.84 0.79 M6.87 97.4 C32.56 68.81, 58.12 39.05, 90.84 0.79 M10.54 99.27 C44.8 61.08, 77.13 23.48, 95.83 1.15 M10.54 99.27 C34.51 72.61, 57.1 46.52, 95.83 1.15 M14.22 101.14 C39.53 71.34, 69.57 40.08, 101.47 0.76 M14.22 101.14 C34.84 77.82, 55.87 55.95, 101.47 0.76 M17.24 103.76 C46.92 71.4, 75.58 36.71, 106.46 1.12 M17.24 103.76 C51.41 63.97, 86.18 25.2, 106.46 1.12 M22.22 104.12 C44.56 78.33, 64.38 55.92, 112.1 0.72 M22.22 104.12 C47.92 76.68, 70.02 48.22, 112.1 0.72 M27.21 104.48 C55.41 68.31, 86.6 35.29, 117.09 1.09 M27.21 104.48 C56.61 71.26, 87.32 35.35, 117.09 1.09 M32.85 104.09 C64.68 65.31, 95.58 28.8, 122.08 1.45 M32.85 104.09 C59.01 71.66, 87.62 40.51, 122.08 1.45 M37.84 104.45 C73.13 63.22, 108.3 21.96, 127.72 1.05 M37.84 104.45 C71.18 64.73, 105.76 25.99, 127.72 1.05 M43.48 104.05 C65.13 78.87, 84.08 58.29, 132.71 1.41 M43.48 104.05 C74.99 67.11, 107.41 31.82, 132.71 1.41 M48.47 104.41 C69.81 79.99, 87.44 57.74, 138.35 1.02 M48.47 104.41 C67.81 80.82, 88.05 58.67, 138.35 1.02 M54.11 104.02 C87.26 66.1, 124.48 25.11, 143.34 1.38 M54.11 104.02 C82.87 71.55, 110.02 38.99, 143.34 1.38 M59.1 104.38 C83.31 73.22, 111.28 44.07, 148.98 0.98 M59.1 104.38 C89.39 68.51, 120.43 33.12, 148.98 0.98 M64.09 104.74 C86.82 81.03, 108.7 53.53, 153.97 1.34 M64.09 104.74 C86.29 80.32, 105.8 57.03, 153.97 1.34 M69.73 104.34 C89.32 83.42, 105.63 59.16, 159.61 0.95 M69.73 104.34 C90.62 78.56, 112.91 54.2, 159.61 0.95 M74.72 104.7 C97.95 76.7, 125.38 46.59, 164.6 1.31 M74.72 104.7 C94.98 82.15, 113.5 58.68, 164.6 1.31 M80.36 104.31 C107.22 70.37, 136.59 37.9, 170.24 0.91 M80.36 104.31 C104.21 79.11, 125.25 51.55, 170.24 0.91 M85.35 104.67 C117.62 68.42, 150.6 32.43, 175.23 1.27 M85.35 104.67 C116.39 70.15, 147.03 34.71, 175.23 1.27 M90.99 104.27 C118.22 76.47, 142.84 47.98, 180.87 0.88 M90.99 104.27 C116.46 75.22, 140.72 47.06, 180.87 0.88 M95.98 104.63 C114.84 83.06, 131.88 61.53, 185.86 1.24 M95.98 104.63 C125.03 69.75, 154.06 37.52, 185.86 1.24 M101.62 104.24 C131.07 67.08, 165.17 30.15, 191.5 0.84 M101.62 104.24 C125.79 78.49, 148.5 51.22, 191.5 0.84 M106.61 104.6 C124.36 82.59, 146.08 55.24, 196.49 1.2 M106.61 104.6 C128.4 79.24, 152.03 53.36, 196.49 1.2 M112.25 104.2 C138.64 73.08, 165.19 44.19, 202.13 0.81 M112.25 104.2 C142.15 71.57, 171.17 36.91, 202.13 0.81 M117.24 104.56 C136.72 82.36, 153.58 60.1, 207.12 1.17 M117.24 104.56 C140.76 76.76, 165.96 48.05, 207.12 1.17 M122.88 104.17 C143.17 82.82, 162.5 58.11, 212.76 0.77 M122.88 104.17 C145.75 78.62, 169.44 51.91, 212.76 0.77 M127.87 104.53 C146.22 84.02, 166.11 62.09, 217.75 1.13 M127.87 104.53 C155.35 73.76, 180.77 43.74, 217.75 1.13 M133.51 104.14 C164.81 68.11, 197.28 30.86, 223.39 0.74 M133.51 104.14 C168.19 63.9, 201.17 26.1, 223.39 0.74 M138.5 104.5 C166.8 72.52, 196.72 36.29, 228.38 1.1 M138.5 104.5 C164.03 75.15, 190.8 45.74, 228.38 1.1 M144.14 104.1 C173.58 69.62, 202.28 40.4, 234.02 0.71 M144.14 104.1 C169.22 73.91, 192.33 46.34, 234.02 0.71 M149.13 104.46 C176.4 75.76, 204.16 43.87, 239.01 1.07 M149.13 104.46 C181.7 67.36, 212.26 30.59, 239.01 1.07 M154.77 104.07 C173.07 82.53, 194.43 57.35, 244.65 0.67 M154.77 104.07 C184.37 69, 214.4 34.65, 244.65 0.67 M159.76 104.43 C186.21 75, 209.84 48.04, 248.98 1.79 M159.76 104.43 C185.84 74.1, 214.62 43.36, 248.98 1.79 M165.4 104.03 C196.2 68.27, 222.19 34.49, 253.97 2.15 M165.4 104.03 C192.49 69.98, 221.68 38.09, 253.97 2.15 M170.39 104.39 C195.27 73.83, 222.62 40.88, 257.65 4.02 M170.39 104.39 C192.88 78.58, 215.26 53.5, 257.65 4.02 M176.03 104 C198.07 76.76, 223.23 50.93, 260.66 6.64 M176.03 104 C196.79 80.25, 218.99 55.21, 260.66 6.64 M181.02 104.36 C199.02 83.14, 220.05 59.58, 264.34 8.51 M181.02 104.36 C200.04 81.6, 221.12 58.51, 264.34 8.51 M186.01 104.72 C218.57 68.27, 248.37 29.81, 267.36 11.13 M186.01 104.72 C211.49 73.95, 239.97 42.73, 267.36 11.13 M191.65 104.32 C217.9 72.84, 247.64 39.79, 268.41 16.02 M191.65 104.32 C221.22 70.4, 250.61 38.32, 268.41 16.02 M196.64 104.68 C223.18 73.59, 252.79 42.96, 269.46 20.91 M196.64 104.68 C223.21 73.7, 251.22 42.73, 269.46 20.91 M202.28 104.29 C216.23 90.63, 228.38 74.44, 269.85 26.55 M202.28 104.29 C225.35 78.17, 247.76 52.04, 269.85 26.55 M207.27 104.65 C225.21 83.55, 239.79 64.75, 269.59 32.95 M207.27 104.65 C220.73 90.53, 233.14 76.76, 269.59 32.95 M212.91 104.25 C232.66 82.46, 254.9 55.84, 269.99 38.59 M212.91 104.25 C228.31 87.57, 243.44 69.88, 269.99 38.59 M217.9 104.61 C228.31 91.66, 243.49 74.76, 269.73 44.99 M217.9 104.61 C231.22 89.91, 242.74 74.7, 269.73 44.99 M223.54 104.22 C236.4 92.11, 246.69 77.39, 270.12 50.64 M223.54 104.22 C234.36 93.9, 243.93 81.22, 270.12 50.64 M228.53 104.58 C238.84 93.66, 243.97 86.44, 269.86 57.03 M228.53 104.58 C243.54 86.48, 258.38 70.48, 269.86 57.03 M234.17 104.18 C245.02 89.24, 259.1 77.72, 270.25 62.68 M234.17 104.18 C246.75 88.67, 260.22 73.76, 270.25 62.68 M239.16 104.55 C247.32 98.08, 251.37 90.89, 269.99 69.07 M239.16 104.55 C251.19 91.21, 262.89 75.47, 269.99 69.07 M244.8 104.15 C251.49 95.41, 262.17 85.89, 270.39 74.72 M244.8 104.15 C251.32 98.65, 255.91 92.21, 270.39 74.72 M251.1 103 C257.83 95.92, 259.89 91.73, 271.44 79.61 M251.1 103 C254.95 97.77, 259.69 93.49, 271.44 79.61 M259.37 99.59 C261.67 97.45, 265.73 94.15, 268.55 89.02 M259.37 99.59 C261.74 95.61, 265.57 92.97, 268.55 89.02" stroke="#868e96" stroke-width="0.5" fill="none"></path><path d="M25.75 0 M25.75 0 C86.28 -2.49, 147.04 -2.49, 243.25 0 M25.75 0 C88.89 -1.03, 150.36 -1.69, 243.25 0 M243.25 0 C260.49 0.17, 269.85 7.5, 269 25.75 M243.25 0 C258.71 0.37, 266.75 7.28, 269 25.75 M269 25.75 C270.97 46.59, 267.37 65.89, 269 77.25 M269 25.75 C269.2 38.85, 270.07 52.43, 269 77.25 M269 77.25 C267.91 93.31, 261.03 101.51, 243.25 103 M269 77.25 C267.19 96.48, 261.43 103.14, 243.25 103 M243.25 103 C179.23 104.68, 117.69 104.65, 25.75 103 M243.25 103 C182.86 102.21, 120.89 102.03, 25.75 103 M25.75 103 C10.28 101.6, -1.86 96.34, 0 77.25 M25.75 103 C10.81 103.84, 0.64 96.57, 0 77.25 M0 77.25 C1.71 64.27, 0.1 52.36, 0 25.75 M0 77.25 C-0.34 63.76, -1.17 48.26, 0 25.75 M0 25.75 C-1.62 9.86, 9.81 -1.74, 25.75 0 M0 25.75 C-0.21 6.49, 8.5 -2.24, 25.75 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1079.1512490011323 994.6351173936137) rotate(0 113.06988525390625 12)"><text x="113.06988525390625" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">RuntimeVersionerService</text></g><g stroke-linecap="round"><g transform="translate(857.1867909941507 483.06611035978096) rotate(0 100.60737278544343 262.4776728571893)"><path d="M-0.93 0.53 C32.67 88.24, 168.56 437.96, 202.15 525.19 M0.78 -0.23 C34.23 87.21, 167.85 435.89, 201.56 523.4" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(857.1867909941507 483.06611035978096) rotate(0 100.60737278544343 262.4776728571893)"><path d="M182.77 501.92 C189.56 507.35, 194.18 515.05, 201.52 521.62 M182.72 500.91 C190.55 510.08, 198.21 518.22, 202.47 523.66" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(857.1867909941507 483.06611035978096) rotate(0 100.60737278544343 262.4776728571893)"><path d="M201.92 494.56 C202.02 502.39, 200.02 512.64, 201.52 521.62 M201.87 493.55 C202.2 505.66, 202.33 516.69, 202.47 523.66" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round"><g transform="translate(1335.5531602693736 1291.724826883709) rotate(0 52.58743265948215 0.7275915017951888)"><path d="M-0.52 0 C17.18 0.28, 87.99 2.22, 105.7 2.5 M1.4 -1.04 C18.99 -1.14, 87.42 0.9, 104.66 0.99" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1335.5531602693736 1291.724826883709) rotate(0 52.58743265948215 0.7275915017951888)"><path d="M76.29 10.44 C85.39 8.13, 91.22 7.7, 103.49 -0.55 M75.61 11.46 C85.67 7.04, 96.22 3.51, 105.51 1.67" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1335.5531602693736 1291.724826883709) rotate(0 52.58743265948215 0.7275915017951888)"><path d="M76.68 -10.07 C85.82 -7.18, 91.55 -2.4, 103.49 -0.55 M76 -9.06 C85.8 -6.39, 96.22 -2.84, 105.51 1.67" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(1446.6965859072843 1114.4105290484283) rotate(0 276.23700864765306 124.97508700007086)"><path d="M32 0 M32 0 C212.58 0.82, 393.28 0.24, 520.47 0 M520.47 0 C543.6 -1.17, 550.93 11.57, 552.47 32 M552.47 32 C553.87 75.64, 551.72 121.74, 552.47 217.95 M552.47 217.95 C551.46 240.97, 543.16 251.17, 520.47 249.95 M520.47 249.95 C413.88 250.7, 308.84 251.03, 32 249.95 M32 249.95 C9.64 248.99, 1.32 238.19, 0 217.95 M0 217.95 C-1.06 179.35, -0.74 140.77, 0 32 M0 32 C-1.8 9.44, 9.5 0.97, 32 0" stroke="#000000" stroke-width="1.5" fill="none" stroke-dasharray="8 9"></path></g><g stroke-linecap="round" transform="translate(1521.9572691016738 1147.3991157683051) rotate(0 232 40)"><path d="M20 0 M20 0 C147.21 -1.88, 275.62 -2.81, 444 0 M20 0 C114.45 0.99, 209.34 1.09, 444 0 M444 0 C456.66 0.56, 462.99 8.35, 464 20 M444 0 C458.89 1.4, 462.09 7.4, 464 20 M464 20 C464.36 34.35, 462.66 44.98, 464 60 M464 20 C463.33 35.15, 462.83 49.51, 464 60 M464 60 C462.42 74.21, 455.53 78.77, 444 80 M464 60 C462.66 74.45, 458.04 79, 444 80 M444 80 C351.93 80.63, 259.55 81.62, 20 80 M444 80 C283.51 80.67, 122.19 81.29, 20 80 M20 80 C5.1 79.31, -1.79 74.85, 0 60 M20 80 C4.71 78.64, 2.28 73.81, 0 60 M0 60 C-1.87 51.2, 0.84 38.44, 0 20 M0 60 C0.8 44.59, -0.06 28.46, 0 20 M0 20 C-1.63 7.56, 7.31 -1.58, 20 0 M0 20 C0.95 6.49, 6.5 -0.79, 20 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1587.3373960547988 1175.3991157683051) rotate(0 166.619873046875 12)"><text x="166.619873046875" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">service_executor_t:callback函数</text></g><g stroke-linecap="round" transform="translate(1507.389184922804 1267.1387191473502) rotate(0 238.6888606061775 34.84857190029254)"><path d="M17.42 0 M17.42 0 C194.54 -1.22, 369.11 -0.2, 459.95 0 M17.42 0 C123.9 1.24, 229.47 1.62, 459.95 0 M459.95 0 C470.56 1.69, 478.73 7.03, 477.38 17.42 M459.95 0 C469.66 0.73, 477.93 4.05, 477.38 17.42 M477.38 17.42 C476.46 27.97, 478.82 39.34, 477.38 52.27 M477.38 17.42 C476.27 24.96, 476.43 32.49, 477.38 52.27 M477.38 52.27 C475.58 62.66, 470.41 70.67, 459.95 69.7 M477.38 52.27 C478.09 62.89, 469.67 69.79, 459.95 69.7 M459.95 69.7 C305.82 69.29, 152.62 68.61, 17.42 69.7 M459.95 69.7 C311.23 69.68, 162.89 69.04, 17.42 69.7 M17.42 69.7 C4.02 71.21, -1.7 62.7, 0 52.27 M17.42 69.7 C8.08 70.18, -1.22 64.86, 0 52.27 M0 52.27 C1.31 38.5, 1.95 28.01, 0 17.42 M0 52.27 C-0.39 41.41, -1.07 32.82, 0 17.42 M0 17.42 C0.64 4.23, 6.64 -0.15, 17.42 0 M0 17.42 C-0.16 5.02, 6.85 -1.83, 17.42 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1635.638142269704 1289.9872910476429) rotate(0 110.43990325927734 12)"><text x="110.43990325927734" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">Network::PluginManager</text></g><g stroke-linecap="round"><g transform="translate(1311.983811930343 1652.9412190758428) rotate(0 52.27991990272085 2.1238432744860347)"><path d="M0.32 1.13 C17.94 1.54, 88.21 2.11, 105.53 2.1 M-0.97 0.68 C16.6 1.31, 87.77 3.39, 105.15 3.57" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1311.983811930343 1652.9412190758428) rotate(0 52.27991990272085 2.1238432744860347)"><path d="M78.63 15.04 C84.7 10.02, 91.08 7.08, 105.06 3.32 M75.98 13.08 C86.81 10.28, 98.11 6.33, 104.37 3.53" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1311.983811930343 1652.9412190758428) rotate(0 52.27991990272085 2.1238432744860347)"><path d="M79.07 -5.48 C84.95 -4.63, 91.21 -1.7, 105.06 3.32 M76.42 -7.44 C87.05 -2.8, 98.19 0.68, 104.37 3.53" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(1426.206046020512 1558.7190104169888) rotate(0 280.4444630940757 78.22219848632801)"><path d="M32 0 M32 0 C166.07 -2.36, 299.85 -2.23, 528.89 0 M528.89 0 C550.6 -0.46, 561.84 10.25, 560.89 32 M560.89 32 C562.01 66.38, 563.12 101.59, 560.89 124.44 M560.89 124.44 C560.33 147.57, 548.73 156.6, 528.89 156.44 M528.89 156.44 C411.39 154.7, 292.92 155.58, 32 156.44 M32 156.44 C12.13 158.43, -1.76 144.87, 0 124.44 M0 124.44 C2.39 103.63, -0.84 83.53, 0 32 M0 32 C-1.93 11.48, 11.17 -1.02, 32 0" stroke="#000000" stroke-width="1.5" fill="none" stroke-dasharray="8 9"></path></g><g stroke-linecap="round" transform="translate(1518.2058374837284 1594.274532064124) rotate(0 184.0000406901047 42.66667683919263)"><path d="M21.33 0 M21.33 0 C116.01 1.91, 208.61 0.74, 346.67 0 M21.33 0 C98.39 -1.46, 174.95 -0.65, 346.67 0 M346.67 0 C361.14 -0.31, 367.96 6.6, 368 21.33 M346.67 0 C363.07 1.04, 369.96 5.16, 368 21.33 M368 21.33 C366.11 31.95, 368.61 39.08, 368 64 M368 21.33 C367.44 33.78, 367.4 46.08, 368 64 M368 64 C368.07 76.73, 360.26 85.34, 346.67 85.33 M368 64 C369.49 80.12, 358.71 85.04, 346.67 85.33 M346.67 85.33 C228.02 86.36, 106.88 88.19, 21.33 85.33 M346.67 85.33 C223.68 83.58, 102.41 83.88, 21.33 85.33 M21.33 85.33 C6.72 84.59, 0.8 77.44, 0 64 M21.33 85.33 C8.33 85.32, -0.12 77.28, 0 64 M0 64 C0.67 51.15, -1.77 39.88, 0 21.33 M0 64 C0.4 47.77, 0.24 30.77, 0 21.33 M0 21.33 C-0.31 8.17, 9.04 1.44, 21.33 0 M0 21.33 C0.68 5.47, 8.33 -1.43, 21.33 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1604.1759785766649 1624.9412089033167) rotate(0 98.02989959716797 12)"><text x="98.02989959716797" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">service_executor_t</text></g><g stroke-linecap="round"><g transform="translate(1328.697199738121 1008.8184385328796) rotate(0 46.3504839077973 0.03161763900504866)"><path d="M-0.07 0.04 C15.63 -0.07, 78.55 -0.27, 94.27 -0.16 M-1.57 -0.98 C13.97 -1, 77.53 0.53, 93.37 1.05" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1328.697199738121 1008.8184385328796) rotate(0 46.3504839077973 0.03161763900504866)"><path d="M64.98 10.7 C72.53 7.78, 82.61 6.2, 91.54 0.99 M64.97 10.26 C73.7 8.21, 81.62 4.81, 94.22 0.51" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1328.697199738121 1008.8184385328796) rotate(0 46.3504839077973 0.03161763900504866)"><path d="M65.54 -9.82 C73.01 -6.46, 82.92 -1.77, 91.54 0.99 M65.53 -10.25 C74.21 -6.2, 81.97 -3.48, 94.22 0.51" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(1428.6918110953375 940.8300384524705) rotate(0 287.1111424763999 63.99998982747388)"><path d="M32 0 M32 0 C207.64 -0.49, 383.2 0.07, 542.22 0 M542.22 0 C565 -1.83, 574.16 10.62, 574.22 32 M574.22 32 C574.04 55.69, 575.89 77.55, 574.22 96 M574.22 96 C574.4 119.02, 562.49 127.66, 542.22 128 M542.22 128 C381.82 128.6, 221.38 129.75, 32 128 M32 128 C11.54 127.35, -0.44 118.59, 0 96 M0 96 C-0.11 79.8, -1.4 61.71, 0 32 M0 32 C-1.4 12.2, 10.71 0.77, 32 0" stroke="#000000" stroke-width="1.5" fill="none" stroke-dasharray="8 9"></path></g><g stroke-linecap="round" transform="translate(1512.2474141226812 953.2744455976527) rotate(0 184.00004069010447 42.66667683919263)"><path d="M21.33 0 M21.33 0 C116.22 -0.09, 212.78 0.82, 346.67 0 M21.33 0 C89.53 0.44, 156.95 -0.14, 346.67 0 M346.67 0 C361.62 1.31, 367.63 7.28, 368 21.33 M346.67 0 C362.83 -1.23, 367.61 7.43, 368 21.33 M368 21.33 C366.98 37.08, 369.85 55.57, 368 64 M368 21.33 C367.57 32.1, 368.73 43.91, 368 64 M368 64 C368.84 76.41, 360.7 83.94, 346.67 85.33 M368 64 C369.76 78.28, 361.78 85.07, 346.67 85.33 M346.67 85.33 C253.53 84.52, 161.78 83.85, 21.33 85.33 M346.67 85.33 C268.46 83.68, 191.02 83.38, 21.33 85.33 M21.33 85.33 C8.22 87.09, 1.35 77.74, 0 64 M21.33 85.33 C6.08 85.85, 2.05 77.93, 0 64 M0 64 C0.86 51.04, -1.55 38.04, 0 21.33 M0 64 C-0.42 53.55, 0.96 42.99, 0 21.33 M0 21.33 C-1.84 5.75, 7.2 -1.63, 21.33 0 M0 21.33 C-1.89 8.91, 6.66 0.84, 21.33 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1598.2175552156177 983.9411224368454) rotate(0 98.02989959716797 12)"><text x="98.02989959716797" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">service_executor_t</text></g><g stroke-linecap="round" transform="translate(1517.0948959147177 830.9411987307908) rotate(0 203.55555216471362 28.444417317708343)"><path d="M14.22 0 M14.22 0 C144.71 0.44, 273.84 -1.87, 392.89 0 M14.22 0 C151.5 -0.05, 287.76 -1.08, 392.89 0 M392.89 0 C400.68 1.2, 406.58 6.33, 407.11 14.22 M392.89 0 C402.87 -0.02, 406.31 6.52, 407.11 14.22 M407.11 14.22 C406.1 20.64, 408.07 27.29, 407.11 42.67 M407.11 14.22 C408.1 20.61, 407.05 26.93, 407.11 42.67 M407.11 42.67 C407.83 50.2, 403.63 55.57, 392.89 56.89 M407.11 42.67 C406.53 52.25, 400.81 57.92, 392.89 56.89 M392.89 56.89 C303.35 56.15, 217.2 56.49, 14.22 56.89 M392.89 56.89 C272.19 57.36, 151.15 58.1, 14.22 56.89 M14.22 56.89 C5.5 57.6, -1.18 53.88, 0 42.67 M14.22 56.89 C7.04 55.72, 1.45 52, 0 42.67 M0 42.67 C0 34.39, -1.35 25.98, 0 14.22 M0 42.67 C1.22 33.31, -0.13 24.5, 0 14.22 M0 14.22 C-0.14 4.16, 3.67 1.73, 14.22 0 M0 14.22 C1.09 6.44, 3.89 -2.02, 14.22 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(1610.2105448201537 847.3856160484992) rotate(0 110.43990325927734 12)"><text x="110.43990325927734" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">Network::PluginManager</text></g><g stroke-linecap="round" transform="translate(2735.7072158383617 267.94714751938386) rotate(0 277.10548895775355 143.75842176025026)"><path d="M32 0 M32 0 C157.58 -1.28, 282.5 -0.74, 522.21 0 M522.21 0 C544.65 -0.33, 553.21 12.08, 554.21 32 M554.21 32 C553.44 93.56, 554.21 157.63, 554.21 255.52 M554.21 255.52 C553.15 274.86, 542.82 287.85, 522.21 287.52 M522.21 287.52 C380.38 287.17, 237.8 286.18, 32 287.52 M32 287.52 C11.63 286.22, 1.68 276.83, 0 255.52 M0 255.52 C-1.76 195.82, -2.72 137.98, 0 32 M0 32 C0.69 10.55, 11.87 1.43, 32 0" stroke="#000000" stroke-width="1.5" fill="none" stroke-dasharray="8 9"></path></g><g stroke-linecap="round" transform="translate(2773.772578023987 278.59847954031125) rotate(0 219.5 36)"><path d="M18 0 M18 0 C134.01 1.27, 249.1 1.83, 421 0 M18 0 C99.84 -0.97, 183.04 -0.73, 421 0 M421 0 C431.56 -1.18, 438.15 4.47, 439 18 M421 0 C432.36 -0.64, 438.13 4.07, 439 18 M439 18 C439.93 28.55, 440.28 42.13, 439 54 M439 18 C437.83 29.27, 439.62 39.42, 439 54 M439 54 C440.75 66.73, 432.99 70.27, 421 72 M439 54 C437 67.54, 433.37 73.19, 421 72 M421 72 C333.05 70.74, 247.57 70.04, 18 72 M421 72 C321.63 74.65, 222.76 74.02, 18 72 M18 72 C5.94 70.89, -1.48 67.73, 0 54 M18 72 C7.85 70.81, 0.32 64.66, 0 54 M0 54 C-0.93 43.73, 1.87 30.83, 0 18 M0 54 C-0.84 42.77, 0.49 33.01, 0 18 M0 18 C-0.45 6.65, 6.37 -0.09, 18 0 M0 18 C-0.62 6.01, 4.35 1.03, 18 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2788.7927501431277 290.59847954031125) rotate(0 204.47982788085938 24)"><text x="204.47982788085938" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">1. 创建create容器的request</text><text x="204.47982788085938" y="24" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">GenerateSandboxCreateContainerRequest</text></g><g stroke-linecap="round" transform="translate(2774.251003499352 429.1633565270147) rotate(0 226.13333129882812 50.666656494140625)"><path d="M25.33 0 M25.33 0 C144.01 1.75, 261.57 0.8, 426.93 0 M25.33 0 C110.73 -0.5, 195.6 -1.33, 426.93 0 M426.93 0 C444.43 -1.45, 453.76 7.11, 452.27 25.33 M426.93 0 C444.28 1.39, 450.42 8.3, 452.27 25.33 M452.27 25.33 C451.54 36.08, 450.66 48.32, 452.27 76 M452.27 25.33 C452.44 37.05, 452.68 49.69, 452.27 76 M452.27 76 C454.15 94.1, 442.67 99.74, 426.93 101.33 M452.27 76 C451.55 92.49, 444.58 102.6, 426.93 101.33 M426.93 101.33 C272.96 100.42, 120.59 99.16, 25.33 101.33 M426.93 101.33 C332.29 100.73, 237.42 100.61, 25.33 101.33 M25.33 101.33 C9 101.74, 0.96 94.03, 0 76 M25.33 101.33 C9.61 100.41, 0.74 93.51, 0 76 M0 76 C1.16 55.92, 0.38 37.58, 0 25.33 M0 76 C0.66 62.78, -0.69 48.73, 0 25.33 M0 25.33 C-1.51 8.94, 6.47 0.92, 25.33 0 M0 25.33 C0.81 9.29, 8.26 -1.08, 25.33 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2797.2145044759145 443.8300130211553) rotate(0 203.16983032226562 36)"><text x="203.16983032226562" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">2. 发送请求</text><text x="203.16983032226562" y="24" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">m_cb->container.create(create_request, </text><text x="203.16983032226562" y="48" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">&create_response)</text></g><g stroke-linecap="round" transform="translate(2231.317409749352 599.0300862633428) rotate(0 189 37.5)"><path d="M18.75 0 M18.75 0 C143.2 2.11, 265.78 2.42, 359.25 0 M18.75 0 C133.25 -0.01, 247.08 -0.1, 359.25 0 M359.25 0 C372.07 -1.29, 378.14 7.19, 378 18.75 M359.25 0 C369.97 -1.12, 378.84 7.78, 378 18.75 M378 18.75 C377.21 31.41, 379.4 47.31, 378 56.25 M378 18.75 C378.12 25.98, 378.04 32.9, 378 56.25 M378 56.25 C377.86 69.41, 373.55 76.32, 359.25 75 M378 56.25 C376.16 69.34, 370.48 74.34, 359.25 75 M359.25 75 C236.85 73.62, 114.95 73.21, 18.75 75 M359.25 75 C256.86 75.75, 154.22 76.06, 18.75 75 M18.75 75 C4.58 74.67, -1.88 68.14, 0 56.25 M18.75 75 C5.29 72.82, 1.23 70.1, 0 56.25 M0 56.25 C2.15 46.46, 1.97 33.27, 0 18.75 M0 56.25 C0.62 47.96, 0.86 37.46, 0 18.75 M0 18.75 C-1.13 7.22, 5.47 1.49, 18.75 0 M0 18.75 C-1.5 6.65, 8.46 -1.76, 18.75 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2315.4574854329458 612.5300862633428) rotate(0 104.85992431640625 24)"><text x="104.85992431640625" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">3: Enable network</text><text x="104.85992431640625" y="24" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge"> SetNetworkReady</text></g><g stroke-linecap="round"><g transform="translate(2413.7176783040395 530.7633015953741) rotate(0 1.6016726373413803 32.85641445443035)"><path d="M0.31 -1.08 C0.79 9.87, 1.46 55.47, 2.07 66.79 M-0.99 0.97 C-0.09 12.09, 3.43 54.37, 4.19 65.01" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2413.7176783040395 530.7633015953741) rotate(0 1.6016726373413803 32.85641445443035)"><path d="M-10.03 35.83 C-4.01 45.32, -1.08 51.75, 5.32 63.84 M-8.67 38.39 C-5.02 43.71, -3.67 48.85, 3.85 65.87" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2413.7176783040395 530.7633015953741) rotate(0 1.6016726373413803 32.85641445443035)"><path d="M10.42 34.23 C11.71 44.14, 9.89 50.94, 5.32 63.84 M11.79 36.79 C10.9 42.39, 7.74 47.89, 3.85 65.87" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round"><g transform="translate(2412.544587229366 675.8299825035772) rotate(0 -0.6410961522424259 25.11273169694934)"><path d="M-0.76 -0.2 C-1.31 8.42, -2.12 42.88, -2.33 51.57 M1.05 -1.35 C0.81 6.92, 0.73 41.09, 0.08 49.65" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2412.544587229366 675.8299825035772) rotate(0 -0.6410961522424259 25.11273169694934)"><path d="M-8.18 24.32 C-4.25 34.18, -0.63 45.28, 0 50.72 M-8.28 25.22 C-4.77 33.51, -1.29 42.96, -0.4 48.7" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2412.544587229366 675.8299825035772) rotate(0 -0.6410961522424259 25.11273169694934)"><path d="M9.33 24.92 C6.61 34.45, 3.57 45.32, 0 50.72 M9.23 25.82 C6.33 33.77, 3.39 43, -0.4 48.7" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(2211.051052327477 736.6299702965459) rotate(0 212 53)"><path d="M26.5 0 M26.5 0 C107.1 -1.06, 185.21 -2.83, 397.5 0 M26.5 0 C151.21 2.52, 276.18 2.41, 397.5 0 M397.5 0 C414.02 0.7, 425.06 9.48, 424 26.5 M397.5 0 C417.26 0.85, 424.03 9.54, 424 26.5 M424 26.5 C422 37.73, 421.81 51.29, 424 79.5 M424 26.5 C423.7 44.66, 423.92 64.87, 424 79.5 M424 79.5 C425.06 96.98, 417.12 106, 397.5 106 M424 79.5 C423.74 99.34, 415.83 106.43, 397.5 106 M397.5 106 C273.92 106.96, 153.49 107.27, 26.5 106 M397.5 106 C283.99 108.55, 169.41 108.54, 26.5 106 M26.5 106 C10.14 105.09, 0.6 96.2, 0 79.5 M26.5 106 C7.76 105.04, 1.53 97.46, 0 79.5 M0 79.5 C-0.4 62.66, 0.45 47.25, 0 26.5 M0 79.5 C-1.05 64.93, 0.35 52.71, 0 26.5 M0 26.5 C-0.64 9.76, 10.05 -1.37, 26.5 0 M0 26.5 C1.74 9.74, 7.47 0.79, 26.5 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2220.7811396077504 741.6299702965459) rotate(0 202.26991271972656 48)"><text x="202.26991271972656" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">4: Inspect </text><text x="202.26991271972656" y="24" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">container:直接从container_t中获得容器信</text><text x="202.26991271972656" y="48" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">息:inspect_data </text><text x="202.26991271972656" y="72" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">=CRIHelpers::InspectContainer</text></g><g stroke-linecap="round"><g transform="translate(2414.7843286946645 846.4966695152959) rotate(0 0.6599427431219738 26.067010908694925)"><path d="M0.34 -0.51 C0.55 8.2, 2.24 43.85, 2.27 52.73 M-0.95 -1.82 C-0.87 6.99, 1.89 44.68, 1.95 53.95" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2414.7843286946645 846.4966695152959) rotate(0 0.6599427431219738 26.067010908694925)"><path d="M-9.12 31.32 C-5.27 37.72, -1.27 49.6, 1.08 53.67 M-8.35 28.84 C-5.83 34.74, -4.76 38.26, 2.13 54.66" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2414.7843286946645 846.4966695152959) rotate(0 0.6599427431219738 26.067010908694925)"><path d="M9.11 30.51 C6.01 37.27, 3.07 49.47, 1.08 53.67 M9.87 28.03 C8.72 34.14, 6.1 37.82, 2.13 54.66" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(2260.117702718102 910.4966695152959) rotate(0 173.86663818359375 33.600006103515625)"><path d="M16.8 0 M16.8 0 C108.05 -0.52, 196.9 -1.92, 330.93 0 M16.8 0 C108.23 -1.99, 199 -2.1, 330.93 0 M330.93 0 C340.14 1.17, 348.15 4.88, 347.73 16.8 M330.93 0 C343.42 -0.52, 346.46 7.22, 347.73 16.8 M347.73 16.8 C347.57 26.87, 349.37 36.12, 347.73 50.4 M347.73 16.8 C348.24 26.28, 347.63 34.65, 347.73 50.4 M347.73 50.4 C349.67 63.29, 342.27 66.3, 330.93 67.2 M347.73 50.4 C349.02 62.37, 340.67 66.3, 330.93 67.2 M330.93 67.2 C228.87 64.62, 125.35 66.02, 16.8 67.2 M330.93 67.2 C228.49 67.29, 125.29 67.31, 16.8 67.2 M16.8 67.2 C6.05 65.84, -1.32 62.13, 0 50.4 M16.8 67.2 C4.12 68.2, 0.92 62.14, 0 50.4 M0 50.4 C-0.93 37.58, 1.41 24.66, 0 16.8 M0 50.4 C0.69 39.32, 1.1 27.48, 0 16.8 M0 16.8 C-0.34 3.73, 5.1 1.27, 16.8 0 M0 16.8 C-0.23 3.65, 4.01 0.8, 16.8 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2296.744442220055 920.0966756188116) rotate(0 137.23989868164062 24)"><text x="137.23989868164062" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">5: Get networking info</text><text x="137.23989868164062" y="24" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge"> GetSandboxNetworkInfo</text></g><g stroke-linecap="round"><g transform="translate(2611.051052327477 945.6966817223272) rotate(0 75.23476771557216 -134.5340545804938)"><path d="M1.11 0.08 C26.08 -44.66, 125.36 -222.84, 150.23 -267.64 M0.24 -0.92 C25.09 -46.04, 124.78 -224.74, 149.76 -269.15" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2611.051052327477 945.6966817223272) rotate(0 75.23476771557216 -134.5340545804938)"><path d="M145.07 -238.04 C148.3 -247.85, 148.11 -260.39, 150.88 -268.48 M145.47 -240.3 C147.04 -247.17, 147.99 -253.17, 150.04 -269.66" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2611.051052327477 945.6966817223272) rotate(0 75.23476771557216 -134.5340545804938)"><path d="M127.17 -248.06 C136.48 -254.67, 142.32 -263.84, 150.88 -268.48 M127.57 -250.32 C133.32 -254.87, 138.51 -258.51, 150.04 -269.66" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(2775.317653889977 589.4299580895147) rotate(0 240.00006103515625 41)"><path d="M20.5 0 M20.5 0 C176.81 1.56, 334.63 1.09, 459.5 0 M20.5 0 C167.5 -0.35, 314.44 -0.66, 459.5 0 M459.5 0 C472.83 -1.87, 479.5 8.1, 480 20.5 M459.5 0 C472.94 -1.95, 478.41 7.64, 480 20.5 M480 20.5 C481.52 32.89, 479.43 43.6, 480 61.5 M480 20.5 C478.92 35.39, 479.88 49.65, 480 61.5 M480 61.5 C478.28 77.15, 472.2 82.36, 459.5 82 M480 61.5 C479.61 73.02, 472.5 82.02, 459.5 82 M459.5 82 C289.7 82.11, 119.76 82.08, 20.5 82 M459.5 82 C357.21 80.43, 254.8 80.09, 20.5 82 M20.5 82 C6.59 83.58, -1.19 75.19, 0 61.5 M20.5 82 C5.75 80.04, 0.35 76.28, 0 61.5 M0 61.5 C-1.79 48.79, -2.15 32.13, 0 20.5 M0 61.5 C-0.46 50.62, -0.69 40.31, 0 20.5 M0 20.5 C0.44 6.59, 8.46 -0.41, 20.5 0 M0 20.5 C0.41 6.21, 7.26 -1.79, 20.5 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2780.9879114583364 594.4299580895147) rotate(0 234.32980346679688 36)"><text x="234.32980346679688" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge"> 6: Mount network namespace when network </text><text x="234.32980346679688" y="24" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">mode is cni</text><text x="234.32980346679688" y="48" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">namespace_is_cn;prepare_network_namespace</text></g><g stroke-linecap="round"><g transform="translate(2999.317653889977 672.6299702965459) rotate(0 0.003132433262635459 24.92200941368938)"><path d="M-0.9 -0.55 C-0.96 7.89, -1.25 41.78, -1.13 50.39 M0.82 1.77 C1.19 9.86, 1.29 40.49, 0.99 48.41" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2999.317653889977 672.6299702965459) rotate(0 0.003132433262635459 24.92200941368938)"><path d="M-8.18 22.86 C-4.94 30.11, -1.81 35.51, 1.35 47.87 M-7 24.59 C-5.72 31.28, -2.16 40.18, 0.25 49.15" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2999.317653889977 672.6299702965459) rotate(0 0.003132433262635459 24.92200941368938)"><path d="M8.97 23.09 C8.18 30.17, 7.29 35.51, 1.35 47.87 M10.15 24.83 C5.96 31.6, 4.06 40.42, 0.25 49.15" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(2790.251003499352 724.8966328942022) rotate(0 230.933349609375 37.866668701171875)"><path d="M18.93 0 M18.93 0 C106.29 -0.2, 195.93 0.02, 442.93 0 M18.93 0 C132.11 -1.91, 244.34 -1.74, 442.93 0 M442.93 0 C455.25 -1.15, 462.68 7.83, 461.87 18.93 M442.93 0 C456.26 -1.97, 463.72 4.63, 461.87 18.93 M461.87 18.93 C460.39 34.3, 461.5 49.52, 461.87 56.8 M461.87 18.93 C462.96 26.57, 461.35 34.64, 461.87 56.8 M461.87 56.8 C460.89 69.57, 456.64 75.08, 442.93 75.73 M461.87 56.8 C460.78 67.63, 453.72 73.81, 442.93 75.73 M442.93 75.73 C357 75.9, 271.84 75.72, 18.93 75.73 M442.93 75.73 C331.87 74.95, 220.97 74.16, 18.93 75.73 M18.93 75.73 C4.64 75.1, -1.02 68.44, 0 56.8 M18.93 75.73 C8.53 75.8, -1.2 71.13, 0 56.8 M0 56.8 C0.27 42.57, 0.58 30.28, 0 18.93 M0 56.8 C-0.83 48.14, 0.53 39.52, 0 18.93 M0 18.93 C-0.31 5.01, 5.11 -0.52, 18.93 0 M0 18.93 C1 5.92, 5.58 -1.15, 18.93 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2839.9145166829458 738.7633015953741) rotate(0 181.26983642578125 24)"><text x="181.26983642578125" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">7: Setup networking for the sandbox.</text><text x="181.26983642578125" y="24" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge"> SetupSandboxNetwork</text></g><g stroke-linecap="round"><g transform="translate(3003.5843775227895 801.6966206871709) rotate(0 -0.5284792393740645 23.662108942130573)"><path d="M-1.18 -0.15 C-1.14 7.91, -0.62 39.42, -0.7 47.39 M0.4 -1.28 C0.4 7.03, -0.93 40.26, -1.45 48.6" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(3003.5843775227895 801.6966206871709) rotate(0 -0.5284792393740645 23.662108942130573)"><path d="M-8.81 26.28 C-8.21 29.81, -6.2 34.52, -2.21 48.52 M-7.85 24.78 C-7.57 31.04, -5.47 36.61, -0.49 48.63" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(3003.5843775227895 801.6966206871709) rotate(0 -0.5284792393740645 23.662108942130573)"><path d="M7.59 27.08 C4.59 30.28, 2.99 34.8, -2.21 48.52 M8.55 25.59 C4.73 31.75, 2.72 37.11, -0.49 48.63" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(2795.5843775227895 861.4299580895147) rotate(0 230.933349609375 29.333343505859375)"><path d="M14.67 0 M14.67 0 C120.84 -2.32, 228.97 -1.66, 447.2 0 M14.67 0 C149.25 1.78, 283.04 1.56, 447.2 0 M447.2 0 C458.37 1.57, 461.55 3.59, 461.87 14.67 M447.2 0 C455.59 -0.6, 462.86 4.5, 461.87 14.67 M461.87 14.67 C462.59 20.73, 461.66 29.72, 461.87 44 M461.87 14.67 C461.45 25.97, 461.29 36.57, 461.87 44 M461.87 44 C463.17 52.87, 456.38 59.1, 447.2 58.67 M461.87 44 C461.23 55.33, 455.25 59.93, 447.2 58.67 M447.2 58.67 C337.76 58.29, 228.09 57.35, 14.67 58.67 M447.2 58.67 C284.91 57.4, 123.02 57.83, 14.67 58.67 M14.67 58.67 C6.86 58.59, 0.17 54.56, 0 44 M14.67 58.67 C3.24 59.61, 0.64 55.41, 0 44 M0 44 C0.27 35.16, -1.47 21.73, 0 14.67 M0 44 C0.26 34.43, 0.14 24.49, 0 14.67 M0 14.67 C-1.64 4.28, 4.04 -0.19, 14.67 0 M0 14.67 C0.78 4.38, 4.9 0.86, 14.67 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2866.4178430989614 866.7633015953741) rotate(0 160.09988403320312 24)"><text x="160.09988403320312" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">8: Start the sandbox container.</text><text x="160.09988403320312" y="24" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge"> StartSandboxContainer</text></g><g stroke-linecap="round"><g transform="translate(3001.4510157063833 921.4299580895147) rotate(0 2.6135365419090704 22.424533641417497)"><path d="M-0.29 0.69 C0.72 8.28, 4.73 37.52, 5.51 44.85 M1.77 0 C2.68 7.35, 3.82 36.18, 4.66 43.31" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(3001.4510157063833 921.4299580895147) rotate(0 2.6135365419090704 22.424533641417497)"><path d="M-3.24 23.52 C-0.34 26.81, 1.6 35.42, 3.67 43.38 M-3.4 23.19 C-1.71 28.92, -0.11 34.29, 4.94 44.02" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(3001.4510157063833 921.4299580895147) rotate(0 2.6135365419090704 22.424533641417497)"><path d="M11.79 22.41 C10.85 25.94, 8.98 34.84, 3.67 43.38 M11.63 22.07 C9.32 28.18, 6.93 33.84, 4.94 44.02" stroke="#000000" stroke-width="1" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(2798.7843897298208 970.4966084801397) rotate(0 233.066650390625 41)"><path d="M20.5 0 M20.5 0 C178.82 0.44, 336.88 -0.47, 445.63 0 M20.5 0 C116 -0.69, 212.03 -0.76, 445.63 0 M445.63 0 C459.48 -1.64, 465.53 5.99, 466.13 20.5 M445.63 0 C459.08 0.78, 465.62 6.84, 466.13 20.5 M466.13 20.5 C465.28 36.16, 465.15 46.81, 466.13 61.5 M466.13 20.5 C466.02 36.41, 466.03 51.01, 466.13 61.5 M466.13 61.5 C467.94 74.83, 460.2 82.14, 445.63 82 M466.13 61.5 C465.29 76.63, 457.59 83.38, 445.63 82 M445.63 82 C314.55 84.28, 183.29 83.81, 20.5 82 M445.63 82 C349.6 83.12, 252.59 82.57, 20.5 82 M20.5 82 C6.96 83.42, 1.81 76.8, 0 61.5 M20.5 82 C7.38 80.8, -0.14 75.11, 0 61.5 M0 61.5 C-0.97 48.34, 1.62 38.46, 0 20.5 M0 61.5 C-0.09 46.84, -0.96 31.54, 0 20.5 M0 20.5 C-0.4 8.53, 6.11 1.04, 20.5 0 M0 20.5 C-2.06 6.35, 7.85 -0.35, 20.5 0" stroke="#000000" stroke-width="1" fill="none"></path></g><g transform="translate(2837.941189046227 975.4966084801397) rotate(0 193.90985107421875 36)"><text x="193.90985107421875" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge"> 9: Save network settings json to disk</text><text x="193.90985107421875" y="24" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge"> ips_request = </text><text x="193.90985107421875" y="48" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">GenerateUpdateNetworkSettingsReqest</text></g><g stroke-linecap="round" transform="translate(2735.8511927083364 54.229945882483435) rotate(0 285.86669921875 93.33332824707031)"><path d="M7.86 8.16 C7.86 8.16, 7.86 8.16, 7.86 8.16 M7.86 8.16 C7.86 8.16, 7.86 8.16, 7.86 8.16 M2.75 26.24 C6.98 20.84, 10.7 16.49, 21.12 5.11 M2.75 26.24 C8.58 21.2, 11.69 15.27, 21.12 5.11 M2.88 38.28 C13.06 25.8, 21.16 14.74, 34.37 2.05 M2.88 38.28 C12.1 29.8, 18.38 19.07, 34.37 2.05 M2.36 51.07 C16.8 32.96, 32.29 15.43, 45 2.02 M2.36 51.07 C16.31 34.22, 30.11 19.21, 45 2.02 M2.49 63.11 C23.14 40.12, 44.63 15.17, 55.63 1.98 M2.49 63.11 C13.98 49.4, 26.79 35.39, 55.63 1.98 M2.62 75.16 C19.52 54.85, 39.98 29.54, 66.26 1.95 M2.62 75.16 C22.87 53.09, 42.52 29.3, 66.26 1.95 M2.76 87.2 C20.87 66, 39.63 46.31, 76.89 1.91 M2.76 87.2 C24.42 61.55, 46.56 37.83, 76.89 1.91 M2.89 99.24 C24.36 74.65, 46.96 52.82, 87.52 1.88 M2.89 99.24 C27.79 70.67, 52.84 42.23, 87.52 1.88 M2.37 112.03 C36.14 73.63, 70.42 33.4, 98.15 1.85 M2.37 112.03 C29.37 81.32, 56.14 49.23, 98.15 1.85 M2.5 124.07 C35.32 86.4, 65.62 47.81, 108.78 1.81 M2.5 124.07 C43.86 76.87, 84.07 28.58, 108.78 1.81 M2.63 136.11 C36.97 96.55, 75.27 55.7, 119.41 1.78 M2.63 136.11 C41.68 92.95, 80.83 46.47, 119.41 1.78 M2.77 148.16 C42.01 105.01, 78.83 60.84, 130.04 1.74 M2.77 148.16 C42.24 99.65, 83.59 53.19, 130.04 1.74 M0.93 162.46 C35.78 126.07, 66.06 86.07, 140.67 1.71 M0.93 162.46 C42.61 113.97, 84.25 65.88, 140.67 1.71 M3.69 171.48 C34.26 133.61, 67.05 95.94, 150.65 2.43 M3.69 171.48 C35.75 132.38, 68.71 96.11, 150.65 2.43 M9.07 177.49 C67.99 110.03, 127.6 39.1, 161.28 2.39 M9.07 177.49 C58.02 117.81, 109.2 60.1, 161.28 2.39 M14.45 183.49 C73.61 114.75, 133.33 45.84, 171.91 2.36 M14.45 183.49 C49.92 142.84, 85.4 103.09, 171.91 2.36 M21.15 187.98 C61.94 143.98, 100.72 97.16, 182.54 2.32 M21.15 187.98 C70.33 130.44, 118.43 74.73, 182.54 2.32 M32.43 187.19 C67.92 145.27, 105.44 102.09, 193.17 2.29 M32.43 187.19 C75.77 135.78, 119.96 84.88, 193.17 2.29 M43.06 187.16 C89.52 133.68, 135.34 78.73, 203.8 2.26 M43.06 187.16 C105.79 115.29, 169.93 41.62, 203.8 2.26 M53.69 187.12 C116.27 114.27, 181.55 42.32, 214.43 2.22 M53.69 187.12 C107.88 123.74, 161.64 60.89, 214.43 2.22 M64.32 187.09 C110.91 133.98, 158.73 79.27, 225.06 2.19 M64.32 187.09 C110.02 134.08, 154.49 83.35, 225.06 2.19 M74.95 187.06 C139.02 112.15, 204.47 40.83, 235.69 2.15 M74.95 187.06 C136.05 119.92, 195.47 52.04, 235.69 2.15 M85.58 187.02 C118.23 147.91, 153.18 110.71, 246.32 2.12 M85.58 187.02 C145.59 117.76, 205.86 49.08, 246.32 2.12 M96.21 186.99 C149.86 124.83, 204.22 63.94, 256.95 2.08 M96.21 186.99 C150.95 121.87, 207.28 56.97, 256.95 2.08 M106.84 186.95 C155.69 131.98, 203.41 78.96, 267.58 2.05 M106.84 186.95 C164.61 120.68, 222.52 54.56, 267.58 2.05 M117.47 186.92 C151.66 144.39, 185.57 103.34, 278.21 2.01 M117.47 186.92 C159.82 137.95, 203.48 89.14, 278.21 2.01 M128.1 186.88 C163.45 148.97, 198.61 107.89, 288.84 1.98 M128.1 186.88 C170.39 139.76, 212.93 90.14, 288.84 1.98 M138.73 186.85 C174.22 145.3, 211.04 102.17, 299.47 1.94 M138.73 186.85 C187.91 128.8, 236.35 71.51, 299.47 1.94 M149.36 186.81 C192.68 139.37, 236.93 88.83, 310.1 1.91 M149.36 186.81 C181.52 146.52, 215.99 107.99, 310.1 1.91 M159.99 186.78 C198.3 141.2, 240.86 95.64, 320.73 1.88 M159.99 186.78 C206.16 134.56, 251.83 81.17, 320.73 1.88 M170.62 186.75 C202.99 148.54, 236.74 109.92, 331.36 1.84 M170.62 186.75 C207.06 142.93, 243.98 100.07, 331.36 1.84 M180.6 187.47 C234.45 123.97, 289.58 59.41, 341.99 1.81 M180.6 187.47 C214.3 148.88, 247.67 110.75, 341.99 1.81 M191.23 187.43 C229.05 145.05, 267.37 102.47, 352.62 1.77 M191.23 187.43 C233.53 137.13, 278.03 86.82, 352.62 1.77 M201.86 187.4 C248.01 133.84, 295.61 81.08, 363.25 1.74 M201.86 187.4 C248.31 133.12, 295.33 78.07, 363.25 1.74 M212.49 187.36 C273.48 121.27, 331.9 53.15, 373.88 1.7 M212.49 187.36 C275.09 115.08, 336.63 44.34, 373.88 1.7 M223.12 187.33 C256.15 146.76, 289.76 108.78, 383.85 2.42 M223.12 187.33 C283.76 118.37, 343.93 48.72, 383.85 2.42 M233.75 187.29 C280.8 133.84, 325.71 81.05, 394.48 2.39 M233.75 187.29 C285.15 128.42, 334.96 70.54, 394.48 2.39 M244.38 187.26 C307.42 116.1, 371.86 43.44, 405.11 2.35 M244.38 187.26 C293.89 130.82, 343.2 73.45, 405.11 2.35 M255.01 187.22 C301.68 128.64, 352.48 73.45, 415.74 2.32 M255.01 187.22 C313.13 120.44, 371.04 54.11, 415.74 2.32 M265.64 187.19 C300.82 146.6, 338.69 105.11, 426.37 2.29 M265.64 187.19 C317.16 126.96, 369.17 67.49, 426.37 2.29 M276.27 187.15 C335.92 118.38, 394.96 50.17, 437 2.25 M276.27 187.15 C320.45 136.83, 366.11 83.6, 437 2.25 M286.9 187.12 C329.78 139.62, 369.94 91.1, 447.63 2.22 M286.9 187.12 C333.54 135.24, 379.18 82.54, 447.63 2.22 M297.53 187.09 C343.92 135.19, 389.19 84.6, 458.26 2.18 M297.53 187.09 C356.77 117.98, 417.55 48.6, 458.26 2.18 M308.16 187.05 C370.62 116.95, 430.46 46, 468.89 2.15 M308.16 187.05 C350.08 136.4, 393.86 86.41, 468.89 2.15 M318.79 187.02 C361.31 138.47, 405.77 87.68, 479.52 2.11 M318.79 187.02 C356.86 144.38, 394.35 99.06, 479.52 2.11 M329.42 186.98 C369.12 144.13, 407.9 101.19, 490.15 2.08 M329.42 186.98 C386.01 121.99, 443.36 56.78, 490.15 2.08 M340.05 186.95 C374.25 150.22, 409.7 110.96, 500.78 2.04 M340.05 186.95 C401.91 116.89, 463.92 46.23, 500.78 2.04 M350.68 186.91 C414.42 117.22, 476.24 46.27, 511.41 2.01 M350.68 186.91 C386.58 145.8, 423.55 103.34, 511.41 2.01 M361.31 186.88 C396.53 144.12, 433.93 104.9, 522.04 1.98 M361.31 186.88 C396.99 144.09, 431.46 103.35, 522.04 1.98 M371.94 186.84 C426.89 126.01, 478.63 62.1, 532.67 1.94 M371.94 186.84 C433.03 116.55, 493.9 45.16, 532.67 1.94 M382.57 186.81 C423.66 136.86, 465.7 87.71, 542.65 2.66 M382.57 186.81 C428.62 133.1, 474.15 81.37, 542.65 2.66 M393.2 186.78 C450.54 122.46, 505.89 59.75, 553.93 1.87 M393.2 186.78 C445.35 125.13, 497.55 64.18, 553.93 1.87 M403.83 186.74 C455.72 126.56, 504.37 70.38, 560.63 6.37 M403.83 186.74 C435.92 149.63, 467.82 112.15, 560.63 6.37 M413.8 187.46 C445.83 150.31, 475.06 117.17, 567.32 10.86 M413.8 187.46 C471.74 119.25, 529.01 51.85, 567.32 10.86 M424.43 187.43 C462.79 143.94, 499.74 99.9, 572.7 16.86 M424.43 187.43 C474.53 129.74, 524.24 73.21, 572.7 16.86 M435.06 187.39 C470.17 146.45, 502.51 109.74, 574.15 27.39 M435.06 187.39 C473.42 144.69, 510.4 103.41, 574.15 27.39 M445.69 187.36 C488.01 140.33, 525.37 96.6, 574.28 39.43 M445.69 187.36 C489.32 140.06, 531.4 92.05, 574.28 39.43 M456.32 187.32 C495.74 139.91, 535.04 93.96, 574.41 51.48 M456.32 187.32 C494.71 145.47, 531.78 101.54, 574.41 51.48 M466.95 187.29 C491.5 157.23, 516.05 131.92, 574.55 63.52 M466.95 187.29 C504.75 144.88, 541.12 101.69, 574.55 63.52 M477.58 187.25 C508.46 151.42, 538.35 116.18, 574.68 75.56 M477.58 187.25 C505.73 156.43, 533.1 124.22, 574.68 75.56 M488.21 187.22 C520.38 150.81, 551.95 113.18, 574.81 87.6 M488.21 187.22 C509.09 161.5, 531.97 135.78, 574.81 87.6 M498.84 187.19 C520.67 159.46, 545.91 133.35, 574.29 100.39 M498.84 187.19 C521.57 159.38, 544.43 132.59, 574.29 100.39 M509.47 187.15 C531.46 163.19, 552.33 136.57, 574.42 112.43 M509.47 187.15 C527.18 168.23, 542.4 149.05, 574.42 112.43 M520.1 187.12 C530.93 175.2, 545.02 159.47, 574.56 124.48 M520.1 187.12 C540.49 163.25, 559.19 140.06, 574.56 124.48 M530.73 187.08 C546.26 168.94, 561.99 152.48, 574.69 136.52 M530.73 187.08 C539.96 174.28, 550.11 164.17, 574.69 136.52 M540.71 187.8 C550.34 176.35, 561.32 166.31, 574.82 148.56 M540.71 187.8 C549.3 178.36, 556.67 170.63, 574.82 148.56 M553.96 184.75 C560.39 175.11, 568.31 167.75, 576.27 159.09 M553.96 184.75 C562.63 175.68, 571.46 165.88, 576.27 159.09" stroke="#ced4da" stroke-width="1" fill="none"></path><path d="M32 0 M32 0 C186.58 -0.24, 340.19 0.32, 539.73 0 M539.73 0 C560.34 0.4, 570.28 8.72, 571.73 32 M571.73 32 C572.79 70.84, 572.49 108.26, 571.73 154.67 M571.73 154.67 C572.43 175.4, 559.77 185.13, 539.73 186.67 M539.73 186.67 C349.87 188.14, 158.98 188.32, 32 186.67 M32 186.67 C9.69 185.86, 0.7 177.6, 0 154.67 M0 154.67 C1.43 110.92, 1.79 68.83, 0 32 M0 32 C1.15 9.32, 12.14 -1.39, 32 0" stroke="#000000" stroke-width="2.5" fill="none" stroke-dasharray="8 10"></path></g><g stroke-linecap="round"><g transform="translate(2614.2511560872426 135.29662679068656) rotate(0 58.622661489993334 -0.220267592696473)"><path d="M-0.59 -1.11 C19.24 -1, 98.04 0.25, 117.84 0.13 M1.3 0.93 C21.11 0.78, 97.72 -1.07, 117.16 -1.37" stroke="#000000" stroke-width="2" fill="none"></path></g><g transform="translate(2614.2511560872426 135.29662679068656) rotate(0 58.622661489993334 -0.220267592696473)"><path d="M87.33 10.3 C96.94 8.85, 99.44 3.33, 118.05 -2.15 M89.34 8.5 C97.2 6.19, 105.1 3.25, 116.82 -2.16" stroke="#000000" stroke-width="2" fill="none"></path></g><g transform="translate(2614.2511560872426 135.29662679068656) rotate(0 58.622661489993334 -0.220267592696473)"><path d="M86.92 -10.22 C96.52 -7.47, 99.1 -8.79, 118.05 -2.15 M88.93 -12.02 C96.86 -8.69, 104.86 -6, 116.82 -2.16" stroke="#000000" stroke-width="2" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(2810.517879720055 83.02993367545218) rotate(0 212.26669311523438 33.600006103515625)"><path d="M16.8 0 M16.8 0 C171.87 2.65, 327.48 2.27, 407.73 0 M16.8 0 C104.37 -0.2, 192.58 -0.38, 407.73 0 M407.73 0 C418.89 1.49, 425.95 4.1, 424.53 16.8 M407.73 0 C421.16 0.01, 422.72 5.76, 424.53 16.8 M424.53 16.8 C426.34 25.74, 423.71 31.98, 424.53 50.4 M424.53 16.8 C424.02 23.74, 424.67 30.98, 424.53 50.4 M424.53 50.4 C423.11 63.04, 417.88 66.63, 407.73 67.2 M424.53 50.4 C424.61 63.48, 417.84 67.7, 407.73 67.2 M407.73 67.2 C327.4 67.45, 245.13 65.41, 16.8 67.2 M407.73 67.2 C258.27 67.71, 108.81 68.08, 16.8 67.2 M16.8 67.2 C4.04 66.54, 1.19 60.15, 0 50.4 M16.8 67.2 C7.09 66.77, -0.68 61.37, 0 50.4 M0 50.4 C-1.95 41.32, 1.22 30.43, 0 16.8 M0 50.4 C-0.76 39.71, 0.77 27.34, 0 16.8 M0 16.8 C1.9 6.09, 5.57 -1.35, 16.8 0 M0 16.8 C0.02 4.56, 7.48 -1.64, 16.8 0" stroke="#000000" stroke-width="2" fill="none"></path></g><g transform="translate(2816.5646631673208 104.62993977896781) rotate(0 206.21990966796875 12)"><text x="206.21990966796875" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">input:run pod的config文件解析出来的结构体</text></g><g stroke-linecap="round" transform="translate(2808.117702718102 162.89657185904593) rotate(0 212.26669311523438 33.600006103515625)"><path d="M16.8 0 M16.8 0 C142.19 -1.83, 268.57 -0.98, 407.73 0 M16.8 0 C109.39 0.54, 200.67 -0.16, 407.73 0 M407.73 0 C418.91 -0.39, 426.41 4.87, 424.53 16.8 M407.73 0 C420.65 -1.08, 426.02 3.71, 424.53 16.8 M424.53 16.8 C424.62 27.34, 423.66 37.08, 424.53 50.4 M424.53 16.8 C424.23 26.22, 425.69 37.36, 424.53 50.4 M424.53 50.4 C425.03 61.35, 417.86 69.12, 407.73 67.2 M424.53 50.4 C422.51 61.51, 419.7 69.4, 407.73 67.2 M407.73 67.2 C269.89 65.59, 132.73 65.69, 16.8 67.2 M407.73 67.2 C258.18 66.68, 107.1 67.26, 16.8 67.2 M16.8 67.2 C5.02 68.17, -0.89 62.9, 0 50.4 M16.8 67.2 C7.07 65.96, -1.71 59.49, 0 50.4 M0 50.4 C-0.92 40.54, 0.92 30.71, 0 16.8 M0 50.4 C0.93 37.31, 0.59 26.1, 0 16.8 M0 16.8 C-1.11 7.12, 5.14 0.74, 16.8 0 M0 16.8 C-0.52 7.44, 6.19 -0.01, 16.8 0" stroke="#000000" stroke-width="2" fill="none"></path></g><g transform="translate(2911.19447731527 184.49657796256156) rotate(0 109.1899185180664 12)"><text x="109.1899185180664" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">output:pause容器的id</text></g><g stroke-linecap="round"><g transform="translate(1991.8112459929644 1636.479491913558) rotate(0 61.700568190979084 -1.0385327604118402)"><path d="M-0.74 0.87 C19.9 0.32, 103.36 -2.57, 124.14 -2.94 M1.07 0.28 C21.57 -0.18, 103.04 -1.27, 123.41 -1.87" stroke="#000000" stroke-width="2" fill="none"></path></g><g transform="translate(1991.8112459929644 1636.479491913558) rotate(0 61.700568190979084 -1.0385327604118402)"><path d="M96.88 9.63 C102.91 5.99, 114.52 0.2, 123.29 -3.47 M94.57 9.9 C100.61 6.22, 107.28 4.01, 123.93 -1.54" stroke="#000000" stroke-width="2" fill="none"></path></g><g transform="translate(1991.8112459929644 1636.479491913558) rotate(0 61.700568190979084 -1.0385327604118402)"><path d="M96.46 -10.89 C102.69 -7.92, 114.44 -7.12, 123.29 -3.47 M94.15 -10.62 C100.38 -9.95, 107.13 -7.81, 123.93 -1.54" stroke="#000000" stroke-width="2" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(2132.050533528649 1426.39731953971) rotate(0 530.0000381469727 407.33333587646484)"><path d="M32 0 M32 0 C242.58 0.35, 453.98 -0.73, 1028 0 M1028 0 C1049.39 -0.66, 1061.06 11.34, 1060 32 M1060 32 C1060.47 208.59, 1060.09 385.42, 1060 782.67 M1060 782.67 C1059.8 804.43, 1049.74 816.38, 1028 814.67 M1028 814.67 C822.35 818.51, 616.43 817.69, 32 814.67 M32 814.67 C11.98 814.17, 0.73 802.45, 0 782.67 M0 782.67 C-0.33 530.12, -0.48 275.64, 0 32 M0 32 C1.77 12.17, 11.23 0.05, 32 0" stroke="#000000" stroke-width="2.5" fill="none" stroke-dasharray="8 10"></path></g><g stroke-linecap="round" transform="translate(2174.717312093102 1491.7306477867803) rotate(0 152.66666412353516 43.33332061767578)"><path d="M5.11 5.77 C5.11 5.77, 5.11 5.77, 5.11 5.77 M5.11 5.77 C5.11 5.77, 5.11 5.77, 5.11 5.77 M3.93 19.32 C7.72 12.68, 14.59 9.06, 16.4 4.98 M3.93 19.32 C7.44 17.12, 10.03 13.49, 16.4 4.98 M3.41 32.11 C12.03 23.15, 22.6 10.39, 27.03 4.94 M3.41 32.11 C11.43 21.3, 20.87 12.36, 27.03 4.94 M3.54 44.15 C15.45 30.35, 27.03 17.21, 37.66 4.91 M3.54 44.15 C11.24 33.11, 19.92 22.91, 37.66 4.91 M3.68 56.2 C18.06 43.22, 29.78 29.45, 48.29 4.88 M3.68 56.2 C13.18 46.49, 22.27 34.36, 48.29 4.88 M1.18 71.26 C15.84 56.97, 29.96 37.63, 58.92 4.84 M1.18 71.26 C19.15 52.07, 35.95 32.22, 58.92 4.84 M5.25 78.77 C26.38 55.57, 46.97 32.38, 69.55 4.81 M5.25 78.77 C17.38 63.59, 30.11 49.35, 69.55 4.81 M9.98 85.53 C36.9 56.83, 62.8 28.3, 80.18 4.77 M9.98 85.53 C29.4 64.75, 47.74 42.8, 80.18 4.77 M17.33 89.26 C43.39 62.31, 68.23 33.62, 90.81 4.74 M17.33 89.26 C44.71 57.89, 72.15 27.23, 90.81 4.74 M26.65 90.74 C51.31 63.85, 71.73 35.33, 101.44 4.7 M26.65 90.74 C43.2 71.54, 57.82 54.2, 101.44 4.7 M37.28 90.71 C59.04 65.71, 78.01 44.21, 112.07 4.67 M37.28 90.71 C62.36 61.49, 89.32 33.18, 112.07 4.67 M47.91 90.67 C74.65 59.05, 101.55 25.94, 122.7 4.63 M47.91 90.67 C69.69 67.93, 88.25 43.85, 122.7 4.63 M58.54 90.64 C86.45 56.54, 114.05 25.37, 133.33 4.6 M58.54 90.64 C83.35 62.93, 108.33 34.17, 133.33 4.6 M69.17 90.6 C89.53 68.01, 110.65 43.26, 143.96 4.56 M69.17 90.6 C92.08 64.59, 116.31 38.53, 143.96 4.56 M79.8 90.57 C101.5 67.09, 123.95 40.34, 154.59 4.53 M79.8 90.57 C107.95 58.02, 135.68 25.98, 154.59 4.53 M90.43 90.53 C117.79 56.07, 148.87 23.01, 165.22 4.5 M90.43 90.53 C111.67 65.36, 135.3 39.18, 165.22 4.5 M100.4 91.25 C131.72 59.87, 160.18 25.14, 175.85 4.46 M100.4 91.25 C125.5 61.79, 149.46 33.37, 175.85 4.46 M111.03 91.22 C135.79 62.69, 158.04 38.09, 186.48 4.43 M111.03 91.22 C132.79 65.24, 156.92 37.27, 186.48 4.43 M121.66 91.18 C146.97 61.05, 173.17 31.29, 196.45 5.15 M121.66 91.18 C139.57 70.78, 159.73 48.55, 196.45 5.15 M132.29 91.15 C155.67 66.13, 176.77 40.07, 207.08 5.11 M132.29 91.15 C156.94 63.71, 179.52 36.45, 207.08 5.11 M142.92 91.12 C159.68 70.51, 176.94 55, 217.71 5.08 M142.92 91.12 C156.39 73.25, 172.27 56.19, 217.71 5.08 M153.55 91.08 C171.47 71.69, 190.02 51.04, 228.34 5.04 M153.55 91.08 C171.42 71.42, 188.6 48.99, 228.34 5.04 M164.18 91.05 C191.78 57.75, 221.48 23.94, 238.97 5.01 M164.18 91.05 C183.62 68.68, 203.07 45.33, 238.97 5.01 M174.81 91.01 C194.9 69.92, 211.53 48.58, 249.6 4.97 M174.81 91.01 C195.02 68.13, 214.52 43.54, 249.6 4.97 M185.44 90.98 C214.72 57.9, 240.28 27.09, 260.23 4.94 M185.44 90.98 C208.62 64.47, 230.81 36.1, 260.23 4.94 M196.07 90.94 C217.18 64.59, 240.88 38.98, 270.86 4.91 M196.07 90.94 C220.32 62.9, 244.91 33.83, 270.86 4.91 M206.7 90.91 C224.44 66.17, 246.56 43.97, 284.77 1.1 M206.7 90.91 C226.09 69.04, 246.04 46.92, 284.77 1.1 M217.33 90.87 C236.79 69.42, 251.39 49.64, 294.09 2.57 M217.33 90.87 C236.66 69.53, 254.84 48.6, 294.09 2.57 M227.96 90.84 C242.54 74.28, 260.15 56.73, 301.44 6.31 M227.96 90.84 C255.81 59.17, 282.21 28.65, 301.44 6.31 M238.59 90.8 C256.38 70.63, 271.52 49.25, 304.86 14.58 M238.59 90.8 C259.91 65.49, 282.33 40.79, 304.86 14.58 M249.22 90.77 C259.94 75.75, 275.99 61.91, 310.24 20.58 M249.22 90.77 C262.4 75.85, 273.58 62.7, 310.24 20.58 M259.85 90.74 C275.59 74.21, 286.55 57.67, 310.37 32.62 M259.85 90.74 C270.83 77.62, 283.76 61.79, 310.37 32.62 M270.48 90.7 C283.8 76.84, 296.79 60.42, 309.85 45.42 M270.48 90.7 C281.42 78.63, 291.71 67.94, 309.85 45.42 M284.39 86.89 C292.02 79.47, 297.57 73.73, 309.98 57.46 M284.39 86.89 C293.71 77.23, 301.57 66.7, 309.98 57.46" stroke="#228be6" stroke-width="1" fill="none"></path><path d="M21.67 0 M21.67 0 C122.78 -1.61, 225.07 -1.76, 283.67 0 M21.67 0 C91.21 0.79, 161.08 -0.48, 283.67 0 M283.67 0 C297.86 0.54, 304.72 5.47, 305.33 21.67 M283.67 0 C296.86 1.63, 304.18 6.32, 305.33 21.67 M305.33 21.67 C306.57 32.03, 306.11 43.62, 305.33 65 M305.33 21.67 C304.22 37.21, 305.29 51.6, 305.33 65 M305.33 65 C304.86 80.77, 297.48 85.73, 283.67 86.67 M305.33 65 C305.21 79.25, 297.05 86.56, 283.67 86.67 M283.67 86.67 C212.4 84.39, 141.85 84.71, 21.67 86.67 M283.67 86.67 C207.95 86.13, 134.15 86.07, 21.67 86.67 M21.67 86.67 C5.66 86.22, 1.83 77.45, 0 65 M21.67 86.67 C9.48 87.35, 1.73 81.01, 0 65 M0 65 C-1.84 49.82, -1.85 29.97, 0 21.67 M0 65 C0.24 52.83, -0.59 42.98, 0 21.67 M0 21.67 C1.95 6.05, 9.05 -1.96, 21.67 0 M0 21.67 C-1.27 5.98, 7.1 -0.14, 21.67 0" stroke="#000000" stroke-width="2" fill="none"></path></g><g transform="translate(2247.784046407067 1523.063968404456) rotate(0 79.59992980957031 12)"><text x="79.59992980957031" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">CreateContainer</text></g><g stroke-linecap="round" transform="translate(2579.68381905111 1454.3971516930303) rotate(0 285.86669921875 93.33332824707031)"><path d="M7.86 8.16 C7.86 8.16, 7.86 8.16, 7.86 8.16 M7.86 8.16 C7.86 8.16, 7.86 8.16, 7.86 8.16 M2.75 26.24 C9.07 21.57, 13.27 15.02, 21.12 5.11 M2.75 26.24 C9.47 19.84, 15.65 12.67, 21.12 5.11 M2.88 38.28 C15.9 24.51, 27.79 12.63, 34.37 2.05 M2.88 38.28 C13.36 26.97, 21.75 14.81, 34.37 2.05 M2.36 51.07 C13.18 39.3, 20.17 30.2, 45 2.02 M2.36 51.07 C14.68 36.96, 25.56 24.61, 45 2.02 M2.49 63.11 C19.23 42.1, 40.62 22.93, 55.63 1.98 M2.49 63.11 C21.59 40.74, 40.87 17.48, 55.63 1.98 M2.62 75.16 C22.28 56.36, 36.25 34.83, 66.26 1.95 M2.62 75.16 C26.42 46.69, 50.16 19.67, 66.26 1.95 M2.76 87.2 C27.24 60.48, 52.65 31.19, 76.89 1.91 M2.76 87.2 C22.72 64.03, 43.36 40.06, 76.89 1.91 M2.89 99.24 C28.47 70.1, 56.88 41.04, 87.52 1.88 M2.89 99.24 C27.96 71.56, 53.33 41.92, 87.52 1.88 M2.37 112.03 C38.65 70.45, 74.01 29.51, 98.15 1.85 M2.37 112.03 C38.09 69.19, 76.19 26.71, 98.15 1.85 M2.5 124.07 C32.17 88.8, 66.7 51.07, 108.78 1.81 M2.5 124.07 C44.96 78.16, 85.84 30.13, 108.78 1.81 M2.63 136.11 C41.56 90.34, 78.19 46.47, 119.41 1.78 M2.63 136.11 C40.17 93.34, 76.02 52.59, 119.41 1.78 M2.77 148.16 C39.21 104.62, 80.35 56.88, 130.04 1.74 M2.77 148.16 C47.51 97.45, 91.94 46.53, 130.04 1.74 M0.93 162.46 C34.55 124.16, 72.07 82.7, 140.67 1.71 M0.93 162.46 C43.5 115.27, 84.95 66.8, 140.67 1.71 M3.69 171.48 C51.93 117.58, 96.61 63.98, 150.65 2.43 M3.69 171.48 C35.87 133.66, 68.71 97.75, 150.65 2.43 M9.07 177.49 C37.08 141.23, 69.18 106.33, 161.28 2.39 M9.07 177.49 C44.54 135.91, 81.16 94.07, 161.28 2.39 M14.45 183.49 C52.07 141.63, 88.61 95.15, 171.91 2.36 M14.45 183.49 C74.65 112.85, 136.32 42.06, 171.91 2.36 M21.15 187.98 C63.26 139.56, 105.35 89.52, 182.54 2.32 M21.15 187.98 C63.39 140.28, 103.41 93.32, 182.54 2.32 M32.43 187.19 C75.9 137.74, 118.3 85.41, 193.17 2.29 M32.43 187.19 C93.8 117.65, 153.55 48.33, 193.17 2.29 M43.06 187.16 C92.94 129.94, 141.27 69.66, 203.8 2.26 M43.06 187.16 C92.09 129.67, 142.93 71.86, 203.8 2.26 M53.69 187.12 C106.04 126.71, 158.99 64.51, 214.43 2.22 M53.69 187.12 C93.18 139.25, 134.62 92.92, 214.43 2.22 M64.32 187.09 C104.31 141.85, 145.22 96.16, 225.06 2.19 M64.32 187.09 C102.64 143.46, 138.97 100.44, 225.06 2.19 M74.95 187.06 C114.96 142.49, 153.07 98.64, 235.69 2.15 M74.95 187.06 C109.15 149.21, 144.19 110.02, 235.69 2.15 M85.58 187.02 C146.31 117.76, 204.72 50.32, 246.32 2.12 M85.58 187.02 C126.68 138.92, 167.61 90.2, 246.32 2.12 M96.21 186.99 C148.02 125.72, 201.79 65.66, 256.95 2.08 M96.21 186.99 C128.78 147.16, 163.72 108.86, 256.95 2.08 M106.84 186.95 C140.88 148.24, 171.57 112.45, 267.58 2.05 M106.84 186.95 C152.44 134.51, 196.51 82.19, 267.58 2.05 M117.47 186.92 C153.95 143.75, 193.88 96.23, 278.21 2.01 M117.47 186.92 C169.53 126.56, 221.94 65.66, 278.21 2.01 M128.1 186.88 C171.22 138.57, 212.82 92.62, 288.84 1.98 M128.1 186.88 C166.02 142.3, 203.52 99.43, 288.84 1.98 M138.73 186.85 C194.18 125.18, 246.4 62.65, 299.47 1.94 M138.73 186.85 C200.67 113.9, 263.4 41.75, 299.47 1.94 M149.36 186.81 C191.75 138.94, 234.67 87.02, 310.1 1.91 M149.36 186.81 C194.4 133.65, 241.89 80.57, 310.1 1.91 M159.99 186.78 C201.25 139.9, 244.79 91.34, 320.73 1.88 M159.99 186.78 C219.04 118.9, 278.78 51.58, 320.73 1.88 M170.62 186.75 C230.58 115.93, 291.13 47.41, 331.36 1.84 M170.62 186.75 C210.83 139.53, 252.56 92.79, 331.36 1.84 M180.6 187.47 C245.66 113.59, 308.54 37.99, 341.99 1.81 M180.6 187.47 C241.63 120.4, 300.85 51.59, 341.99 1.81 M191.23 187.43 C244.8 126.09, 297.61 62.7, 352.62 1.77 M191.23 187.43 C254.79 116.33, 317.93 44.43, 352.62 1.77 M201.86 187.4 C241.02 142.96, 277.68 100.02, 363.25 1.74 M201.86 187.4 C242.11 142.21, 282.39 96.24, 363.25 1.74 M212.49 187.36 C259.68 134.27, 307.78 79.99, 373.88 1.7 M212.49 187.36 C274.75 114.98, 336.23 43.55, 373.88 1.7 M223.12 187.33 C276.89 127.78, 329.34 63.34, 383.85 2.42 M223.12 187.33 C284.16 117.72, 344.82 47.81, 383.85 2.42 M233.75 187.29 C270.69 144.87, 310.5 102.39, 394.48 2.39 M233.75 187.29 C293.81 118.02, 354.08 48.54, 394.48 2.39 M244.38 187.26 C279.16 144.07, 316.34 103.57, 405.11 2.35 M244.38 187.26 C292.67 131.5, 341.68 74.56, 405.11 2.35 M255.01 187.22 C309.02 126.52, 362.59 63.14, 415.74 2.32 M255.01 187.22 C309.9 125.33, 363.78 63.22, 415.74 2.32 M265.64 187.19 C314.89 130.92, 360.39 76.74, 426.37 2.29 M265.64 187.19 C313.75 133.9, 360.63 79.83, 426.37 2.29 M276.27 187.15 C334.92 118.74, 394.86 47.5, 437 2.25 M276.27 187.15 C322.59 133.57, 369.08 80.71, 437 2.25 M286.9 187.12 C332.08 136.08, 374.7 86.66, 447.63 2.22 M286.9 187.12 C344.55 121.37, 401.28 56.69, 447.63 2.22 M297.53 187.09 C353.22 120.85, 411.61 53.32, 458.26 2.18 M297.53 187.09 C340.86 134.73, 385.86 82.49, 458.26 2.18 M308.16 187.05 C368.92 116.84, 429.72 47.5, 468.89 2.15 M308.16 187.05 C345.03 143.25, 381.94 99.91, 468.89 2.15 M318.79 187.02 C365.73 132.42, 410.65 78.64, 479.52 2.11 M318.79 187.02 C351.39 147.73, 386.17 108.92, 479.52 2.11 M329.42 186.98 C388.97 120.85, 444.05 56.18, 490.15 2.08 M329.42 186.98 C367.99 142.57, 406.35 98.79, 490.15 2.08 M340.05 186.95 C399.85 115.67, 463.89 43.77, 500.78 2.04 M340.05 186.95 C377.49 144.82, 413.27 103.41, 500.78 2.04 M350.68 186.91 C391.74 141.62, 430.19 96.63, 511.41 2.01 M350.68 186.91 C412.35 114.62, 475.33 43.66, 511.41 2.01 M361.31 186.88 C405.61 136.79, 449.5 83.87, 522.04 1.98 M361.31 186.88 C407.98 132.16, 454.41 78.53, 522.04 1.98 M371.94 186.84 C402.1 149.45, 435.39 110.98, 532.67 1.94 M371.94 186.84 C415.23 137.56, 458.17 88.59, 532.67 1.94 M382.57 186.81 C445.67 115.24, 506.25 42.94, 542.65 2.66 M382.57 186.81 C419.35 143.34, 457.93 101.47, 542.65 2.66 M393.2 186.78 C441.22 131.12, 490.23 76.27, 553.93 1.87 M393.2 186.78 C452.81 118.29, 511.9 49.84, 553.93 1.87 M403.83 186.74 C459.27 120.57, 517.07 53.49, 560.63 6.37 M403.83 186.74 C454.11 130.37, 502.62 72.88, 560.63 6.37 M413.8 187.46 C460.64 133.98, 505.18 81.32, 567.32 10.86 M413.8 187.46 C470.1 120.83, 528.99 53.76, 567.32 10.86 M424.43 187.43 C466.43 142.33, 507.34 96.36, 572.7 16.86 M424.43 187.43 C458.45 147.83, 491.26 108.92, 572.7 16.86 M435.06 187.39 C476.51 143.02, 515.64 97.7, 574.15 27.39 M435.06 187.39 C474.06 144.3, 511.14 100.5, 574.15 27.39 M445.69 187.36 C493.35 131.87, 541.74 76.76, 574.28 39.43 M445.69 187.36 C487.67 141.2, 527.1 93.84, 574.28 39.43 M456.32 187.32 C487.82 151.71, 517.88 116.08, 574.41 51.48 M456.32 187.32 C502.77 134.41, 549.19 80.4, 574.41 51.48 M466.95 187.29 C501.85 149.84, 534.77 110, 574.55 63.52 M466.95 187.29 C506.77 141.96, 547.63 96.77, 574.55 63.52 M477.58 187.25 C509.15 150.52, 538.3 114.48, 574.68 75.56 M477.58 187.25 C497.24 164.55, 518 140.14, 574.68 75.56 M488.21 187.22 C504.91 166.94, 521.67 147.71, 574.81 87.6 M488.21 187.22 C523.25 147.53, 557.4 108.42, 574.81 87.6 M498.84 187.19 C517.86 162.99, 539.1 140.11, 574.29 100.39 M498.84 187.19 C526.04 155.24, 552.42 125.23, 574.29 100.39 M509.47 187.15 C528.74 162.44, 551.16 140.79, 574.42 112.43 M509.47 187.15 C527.46 166.76, 544.12 146.72, 574.42 112.43 M520.1 187.12 C534.49 169.52, 545.16 156.44, 574.56 124.48 M520.1 187.12 C541.4 163.54, 560.52 140.77, 574.56 124.48 M530.73 187.08 C547.35 169.12, 563.54 149.32, 574.69 136.52 M530.73 187.08 C543.69 171.8, 556.24 157.37, 574.69 136.52 M540.71 187.8 C551.72 178.81, 557.89 165.01, 574.82 148.56 M540.71 187.8 C553.46 173.74, 564.41 160.63, 574.82 148.56 M553.96 184.75 C559.89 176.12, 567.71 169.93, 576.27 159.09 M553.96 184.75 C563.33 175.16, 571.42 165.45, 576.27 159.09" stroke="#ced4da" stroke-width="1" fill="none"></path><path d="M32 0 M32 0 C176.75 -0.48, 321.21 -1, 539.73 0 M539.73 0 C562.29 -0.99, 572.13 11.71, 571.73 32 M571.73 32 C571.09 76.49, 569.46 117.89, 571.73 154.67 M571.73 154.67 C571.87 177.07, 559.98 188.21, 539.73 186.67 M539.73 186.67 C355.91 186.55, 171.98 187.09, 32 186.67 M32 186.67 C10.31 186.69, -1.46 174.71, 0 154.67 M0 154.67 C1.23 106.58, -0.59 56.99, 0 32 M0 32 C1.39 10.64, 10.07 -1.59, 32 0" stroke="#000000" stroke-width="2.5" fill="none" stroke-dasharray="8 10"></path></g><g stroke-linecap="round" transform="translate(2654.3505060628286 1483.197139485999) rotate(0 212.26669311523438 33.600006103515625)"><path d="M16.8 0 M16.8 0 C171.75 -0.16, 328.26 0.88, 407.73 0 M16.8 0 C107.79 -0.6, 198.88 -1.15, 407.73 0 M407.73 0 C418.85 -0.78, 425.96 6.48, 424.53 16.8 M407.73 0 C419.78 -0.05, 425.44 4.85, 424.53 16.8 M424.53 16.8 C425.39 26.48, 426.12 35.28, 424.53 50.4 M424.53 16.8 C425.71 26.14, 424.89 33.92, 424.53 50.4 M424.53 50.4 C426.1 62.86, 419.18 65.79, 407.73 67.2 M424.53 50.4 C422.56 59.48, 418.84 69.48, 407.73 67.2 M407.73 67.2 C252.52 66.87, 96.48 67.49, 16.8 67.2 M407.73 67.2 C266.21 67.8, 124.6 67.57, 16.8 67.2 M16.8 67.2 C3.93 66.81, 1.78 62.66, 0 50.4 M16.8 67.2 C7.22 68.47, -2.15 61.04, 0 50.4 M0 50.4 C-0.4 39.57, 0.32 33.74, 0 16.8 M0 50.4 C0.99 39.05, 0.13 26.97, 0 16.8 M0 16.8 C-1.56 4.74, 5.45 0.51, 16.8 0 M0 16.8 C-1.57 7.02, 5.36 1.7, 16.8 0" stroke="#000000" stroke-width="2" fill="none"></path></g><g transform="translate(2661.3773767903676 1492.7971455895147) rotate(0 205.2398223876953 24)"><text x="205.2398223876953" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">input:podSandboxID、containerConfig、po</text><text x="205.2398223876953" y="24" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">dSandboxConfig</text></g><g stroke-linecap="round" transform="translate(2651.9503290608754 1563.0637776695928) rotate(0 212.26669311523438 33.600006103515625)"><path d="M16.8 0 M16.8 0 C131.61 1.31, 245.89 0.26, 407.73 0 M16.8 0 C150.79 -1.55, 283.98 -1.58, 407.73 0 M407.73 0 C420.36 0.88, 425.27 5.56, 424.53 16.8 M407.73 0 C419.84 -0.75, 423.32 6.56, 424.53 16.8 M424.53 16.8 C426.1 31.02, 423.86 43.17, 424.53 50.4 M424.53 16.8 C424.92 25.82, 425.62 36.21, 424.53 50.4 M424.53 50.4 C424.78 60.19, 417.22 65.36, 407.73 67.2 M424.53 50.4 C424.44 63.88, 421.16 65.88, 407.73 67.2 M407.73 67.2 C288.29 66.64, 171.36 65.26, 16.8 67.2 M407.73 67.2 C283.48 68.04, 159.66 67.94, 16.8 67.2 M16.8 67.2 C7.38 68.26, 1.41 62.7, 0 50.4 M16.8 67.2 C3.45 66.64, -0.53 60.12, 0 50.4 M0 50.4 C0.21 45.14, 1.06 34.75, 0 16.8 M0 50.4 C-0.34 41.69, -1.05 32.69, 0 16.8 M0 16.8 C-0.15 6.11, 4.24 1.23, 16.8 0 M0 16.8 C-0.24 7.3, 6.85 -0.08, 16.8 0" stroke="#000000" stroke-width="2" fill="none"></path></g><g transform="translate(2757.0470926717153 1584.6637837731084) rotate(0 107.16992950439453 12)"><text x="107.16992950439453" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">output:pod中容器的id</text></g><g stroke-linecap="round"><g transform="translate(2481.050548787438 1539.0638692223272) rotate(0 47.524189323932205 1.5068456864078144)"><path d="M-0.62 0.68 C15.18 0.91, 79.37 1.64, 95.67 1.9 M1.26 -0.01 C16.79 0.32, 78.59 2.48, 94.54 3.03" stroke="#000000" stroke-width="2" fill="none"></path></g><g transform="translate(2481.050548787438 1539.0638692223272) rotate(0 47.524189323932205 1.5068456864078144)"><path d="M67.14 11.76 C75.41 10.89, 86.62 7.01, 94.32 4.17 M66.28 12.91 C74.76 10.14, 82.04 8.32, 94.82 2.12" stroke="#000000" stroke-width="2" fill="none"></path></g><g transform="translate(2481.050548787438 1539.0638692223272) rotate(0 47.524189323932205 1.5068456864078144)"><path d="M67.85 -8.75 C75.94 -1.92, 86.87 1.91, 94.32 4.17 M66.99 -7.6 C75.18 -4.66, 82.26 -0.77, 94.82 2.12" stroke="#000000" stroke-width="2" fill="none"></path></g></g><mask></mask><g stroke-linecap="round"><g transform="translate(2316.883818542484 1578.0637700401983) rotate(0 0.05112979133582485 36.59865191488734)"><path d="M-1.15 -0.15 C-1.08 12.15, 0.98 62.29, 1.26 74.48 M0.44 -1.28 C0.29 10.63, -0.11 60.23, 0.32 72.76" stroke="#000000" stroke-width="2" fill="none"></path></g><g transform="translate(2316.883818542484 1578.0637700401983) rotate(0 0.05112979133582485 36.59865191488734)"><path d="M-10.48 44.73 C-4.89 53.95, -3.52 66.64, -0.02 73.46 M-11.05 44.6 C-7.3 53.85, -3.15 61.26, 1.31 72.63" stroke="#000000" stroke-width="2" fill="none"></path></g><g transform="translate(2316.883818542484 1578.0637700401983) rotate(0 0.05112979133582485 36.59865191488734)"><path d="M10.04 44.52 C8.06 53.62, 1.86 66.39, -0.02 73.46 M9.47 44.39 C6.83 53.7, 4.58 61.18, 1.31 72.63" stroke="#000000" stroke-width="2" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(2152.8839558715854 1662.06382344596) rotate(0 209 51.5)"><path d="M25.75 0 M25.75 0 C146.08 1.36, 264.4 0.42, 392.25 0 M25.75 0 C106.84 1.93, 188.29 1.76, 392.25 0 M392.25 0 C411 -0.65, 416.4 8.33, 418 25.75 M392.25 0 C410.03 -0.71, 415.98 7.33, 418 25.75 M418 25.75 C416.49 44.52, 415.82 65.28, 418 77.25 M418 25.75 C418.59 40.67, 418.62 56.27, 418 77.25 M418 77.25 C419.89 94.96, 410.9 102.53, 392.25 103 M418 77.25 C419.53 93.7, 408.34 102.88, 392.25 103 M392.25 103 C285.11 101.86, 177.73 103.23, 25.75 103 M392.25 103 C299.54 102.45, 207.04 101.85, 25.75 103 M25.75 103 C6.94 104.42, -1.78 92.86, 0 77.25 M25.75 103 C8.07 105.11, -2.29 96.67, 0 77.25 M0 77.25 C1.43 61.24, 0.9 41.65, 0 25.75 M0 77.25 C0.06 57.45, 0.72 37.61, 0 25.75 M0 25.75 C-1.44 7.56, 10.34 1.95, 25.75 0 M0 25.75 C-1.35 10.68, 6.33 -1.27, 25.75 0" stroke="#000000" stroke-width="2" fill="none"></path></g><g transform="translate(2170.814024841312 1689.56382344596) rotate(0 191.06993103027344 24)"><text x="191.06993103027344" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">1. 根据podsandbox的id或者名字获得容器id</text><text x="191.06993103027344" y="24" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">GetRealContainerOrSandboxID</text></g><g stroke-linecap="round"><g transform="translate(2318.217131530765 1764.7304494225225) rotate(0 0.22807704887600266 24.859794658201167)"><path d="M0.84 -0.29 C0.8 8.16, 0.93 41.34, 0.67 49.91 M-0.17 -1.49 C-0.44 7.19, -0.47 42.76, -0.18 51.21" stroke="#000000" stroke-width="2" fill="none"></path></g><g transform="translate(2318.217131530765 1764.7304494225225) rotate(0 0.22807704887600266 24.859794658201167)"><path d="M-9.59 27.95 C-6.67 34.48, -1.25 44.64, -0.25 52.66 M-8.42 28.33 C-6.46 32.79, -5.02 37.67, -0.15 50.94" stroke="#000000" stroke-width="2" fill="none"></path></g><g transform="translate(2318.217131530765 1764.7304494225225) rotate(0 0.22807704887600266 24.859794658201167)"><path d="M7.64 27.72 C4.08 34.45, 3.03 44.7, -0.25 52.66 M8.8 28.1 C6.86 32.62, 4.4 37.55, -0.15 50.94" stroke="#000000" stroke-width="2" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(2163.5504597778354 1818.0637700401983) rotate(0 184.5 41.5)"><path d="M20.75 0 M20.75 0 C113.18 -1.14, 205.17 0.69, 348.25 0 M20.75 0 C139.96 1.25, 258.8 1.9, 348.25 0 M348.25 0 C360.33 1.48, 370.02 5.41, 369 20.75 M348.25 0 C362.81 -0.87, 366.83 8.04, 369 20.75 M369 20.75 C368.05 37.16, 368 51.81, 369 62.25 M369 20.75 C368.84 34.31, 368.12 46.12, 369 62.25 M369 62.25 C368.9 75.63, 363.07 84.44, 348.25 83 M369 62.25 C367.97 76.74, 360.26 84.82, 348.25 83 M348.25 83 C220.62 81.65, 92.2 81.13, 20.75 83 M348.25 83 C280.15 83.41, 212.06 82.7, 20.75 83 M20.75 83 C7.49 83.54, 1.55 75.15, 0 62.25 M20.75 83 C6.1 81.79, -0.33 75.68, 0 62.25 M0 62.25 C0.7 52.58, -0.89 38.81, 0 20.75 M0 62.25 C-0.88 51.9, 0.56 41.49, 0 20.75 M0 20.75 C1.53 6.83, 6.22 1.52, 20.75 0 M0 20.75 C-2.3 7.35, 6.81 -2.15, 20.75 0" stroke="#000000" stroke-width="2" fill="none"></path></g><g transform="translate(2195.760573303226 1835.5637700401983) rotate(0 152.28988647460938 24)"><text x="152.28988647460938" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">2. 获得pod的runtime</text><text x="152.28988647460938" y="24" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">GetContainerOrSandboxRuntime</text></g><g stroke-linecap="round"><g transform="translate(2316.883818542484 1902.0637852989873) rotate(0 -1.8344186557551438 27.498918659971082)"><path d="M-0.96 1.03 C-1.6 10.12, -2.88 45.75, -3.19 54.47 M0.74 0.53 C-0.16 9.32, -3.68 43.92, -4.41 52.94" stroke="#000000" stroke-width="2" fill="none"></path></g><g transform="translate(2316.883818542484 1902.0637852989873) rotate(0 -1.8344186557551438 27.498918659971082)"><path d="M-9.46 26.25 C-6.9 36.12, -6.39 46.59, -3.36 52.62 M-10.96 26.79 C-8.57 36.25, -6.54 48.03, -4.9 53.14" stroke="#000000" stroke-width="2" fill="none"></path></g><g transform="translate(2316.883818542484 1902.0637852989873) rotate(0 -1.8344186557551438 27.498918659971082)"><path d="M8.72 27.93 C4.82 37.06, -1.12 46.94, -3.36 52.62 M7.23 28.48 C2.46 37.41, -2.69 48.52, -4.9 53.14" stroke="#000000" stroke-width="2" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(2164.883849060062 1968.730426534339) rotate(0 186.00006103515625 43.33335876464844)"><path d="M21.67 0 M21.67 0 C120.69 0.81, 218.31 2.21, 350.33 0 M21.67 0 C137.66 -0.8, 252.93 -1.43, 350.33 0 M350.33 0 C364.91 1.07, 370.92 8.77, 372 21.67 M350.33 0 C366.21 0.6, 372.18 7.64, 372 21.67 M372 21.67 C371.74 33.22, 370.64 43.43, 372 65 M372 21.67 C372.2 39.22, 371.29 56.02, 372 65 M372 65 C373.39 79.41, 364.18 85.08, 350.33 86.67 M372 65 C371.41 78.15, 363.7 88.86, 350.33 86.67 M350.33 86.67 C283.83 85.17, 218.37 85.35, 21.67 86.67 M350.33 86.67 C262.53 87.94, 176.49 88.59, 21.67 86.67 M21.67 86.67 C5.65 86.21, -0.31 80.82, 0 65 M21.67 86.67 C5.88 88.55, 0.34 77.65, 0 65 M0 65 C-0.95 54.72, -1.13 45.15, 0 21.67 M0 65 C-0.55 50.96, 0.73 37.39, 0 21.67 M0 21.67 C-0.35 8.69, 5.98 -0.43, 21.67 0 M0 21.67 C2.01 5.53, 5.69 1.6, 21.67 0" stroke="#000000" stroke-width="2" fill="none"></path></g><g transform="translate(2185.7540578002963 1988.0637852989873) rotate(0 165.12985229492188 24)"><text x="165.12985229492188" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">3. 创建pod中的容器的create请求</text><text x="165.12985229492188" y="24" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">GenerateCreateContainerRequest</text></g><g stroke-linecap="round"><g transform="translate(2316.883818542484 2048.730426534339) rotate(0 -1.660878687352124 33.212335953847514)"><path d="M0.24 -0.82 C-0.45 10.32, -2.88 54.93, -3.56 66.03 M-1.1 1.37 C-1.45 12.72, -1.24 56.36, -1.32 67.24" stroke="#000000" stroke-width="2" fill="none"></path></g><g transform="translate(2316.883818542484 2048.730426534339) rotate(0 -1.660878687352124 33.212335953847514)"><path d="M-12.91 37.55 C-9.26 46.78, -6.13 59.21, -2.31 66.26 M-12.02 38.9 C-9.32 46.25, -6.49 53.7, -0.99 68.04" stroke="#000000" stroke-width="2" fill="none"></path></g><g transform="translate(2316.883818542484 2048.730426534339) rotate(0 -1.660878687352124 33.212335953847514)"><path d="M7.61 37.57 C4.18 46.83, 0.22 59.26, -2.31 66.26 M8.5 38.92 C6.06 46.32, 3.75 53.77, -0.99 68.04" stroke="#000000" stroke-width="2" fill="none"></path></g></g><mask></mask><g stroke-linecap="round" transform="translate(2180.883833801273 2120.7305486046516) rotate(0 182.66666412353516 42.6666259765625)"><path d="M21.33 0 M21.33 0 C126.81 1.58, 230.46 -0.7, 344 0 M21.33 0 C136.79 1.15, 252.28 1.21, 344 0 M344 0 C359.76 1.81, 366.6 8.82, 365.33 21.33 M344 0 C356.77 -1.14, 363.5 5.36, 365.33 21.33 M365.33 21.33 C364.8 35.88, 363.46 53.35, 365.33 64 M365.33 21.33 C365.68 39.24, 365.33 56.01, 365.33 64 M365.33 64 C363.37 79.91, 359.19 83.78, 344 85.33 M365.33 64 C367.21 77.37, 360.1 83.44, 344 85.33 M344 85.33 C257.71 86.07, 172.21 88.07, 21.33 85.33 M344 85.33 C241.61 85.67, 137.53 85.14, 21.33 85.33 M21.33 85.33 C6.05 84.74, -1.43 78.04, 0 64 M21.33 85.33 C8.82 84.87, 1.19 80.27, 0 64 M0 64 C-0.98 53.16, 1 40.72, 0 21.33 M0 64 C0.25 50.93, 0.22 38.14, 0 21.33 M0 21.33 C-1.2 5.2, 9.1 1.67, 21.33 0 M0 21.33 C0.49 9.07, 9.24 1.61, 21.33 0" stroke="#000000" stroke-width="2" fill="none"></path></g><g transform="translate(2250.400595581058 2139.397174581214) rotate(0 113.14990234375 24)"><text x="113.14990234375" y="0" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">4. 调用创建cb</text><text x="113.14990234375" y="24" font-family="Virgil, Segoe UI Emoji" font-size="20px" fill="#000000" text-anchor="middle" style="white-space: pre;" direction="ltr" dominant-baseline="text-before-edge">m_cb->container.create</text></g></svg> +\ No newline at end of file +diff --git a/docs/manual/README_zh.md b/docs/manual/README_zh.md +index 8de5604b..1f3f37f8 100644 +--- a/docs/manual/README_zh.md ++++ b/docs/manual/README_zh.md +@@ -22,4 +22,8 @@ Device Mapper 是一个基于内核的框架,它支持 Linux 上的许多高 + + ## isula search使用指南 + +-关于如何使用isula search请参考[isula_search](isula_search_zh.md) +\ No newline at end of file ++关于如何使用isula search请参考[isula_search](isula_search_zh.md) ++ ++## runc使用指南 ++ ++关于如何在isulad中使用runc请参考[runc_config](runc_config_zh.md) +\ No newline at end of file +diff --git a/docs/manual/runc_config_zh.md b/docs/manual/runc_config_zh.md +new file mode 100644 +index 00000000..003cd2c5 +--- /dev/null ++++ b/docs/manual/runc_config_zh.md +@@ -0,0 +1,134 @@ ++# runc使用指南 ++本文主要是指导iSulad社区开发者和使用者,如何配置isulad使用runc作为runtime创建容器。 ++ ++## 一、runc的安装 ++ ++`tips`: 在安装runc之前需要安装好go环境。 ++ ++isulad当前推荐的runc验证版本为v1.0.0-rc5。 ++ ++runc可以使用以下两种安装方式: ++ ++1. 直接使用包管理器安装runc: ++ ++```sh ++# centOS ++sudo yum install runc ++# Ubuntu ++sudo apt-get install runc ++``` ++ ++2. 源码编译安装runc(注意建议切换成isulad推荐的runc版本:`git checkout v1.0.0-rc5`) ++ ++```sh ++# 在GOPATH/src下创建 'github.com/opencontainers' 文件夹 ++cd github.com/opencontainers ++git clone https://github.com/opencontainers/runc ++cd runc ++ ++make ++sudo make install ++``` ++ ++还可以使用go get安装到`GOPATH`路径下(需要在GOPATH/src下创建github.com父文件夹): ++ ++```sh ++go get github.com/opencontainers/runc ++cd $GOPATH/src/github.com/opencontainers/runc ++make ++sudo make install ++``` ++ ++最终安装好的runc会在`/usr/local/sbin/runc`目录下。 ++ ++## 二、配置iSulad使用runc ++ ++### 配置文件配置 ++ ++1. 修改isulad的daemon.json,配置isulad默认使用的runtime。 ++ ++```sh ++$ vim /etc/isulad/daemon.json ++ ... ++ "default-runtime": "runc" ++ ... ++``` ++ ++2. 也可以在配置文件中配置runtimes,在其中指定使用的`path`(用于修改isulad使用的runc路径)以及`runtime-args`(对runtime所有命令配置的参数)。 ++ ++```sh ++"runtimes": { ++ "runc": { ++ "path": "/usr/local/sbin/runc", ++ "runtime-args": [ ++ ] ++ } ++ }, ++``` ++ ++之后使用root权限启动isulad服务,使修改后的配置生效即可: ++ ++```sh ++$ sudo isulad ++``` ++ ++### 单个容器配置 ++ ++使用`--runtime=runc`启动一个runtime为runc的容器。 ++ ++```sh ++isula run -tid -n test --runtime=runc busybox sh ++``` ++ ++## 三、K8s中配置pod的runtime为runc ++ ++如何与kubernetes集成请参考[k8s_integration](https://gitee.com/openeuler/iSulad/blob/master/docs/manual/k8s_integration_zh.md)。 ++ ++### 全局配置 ++ ++直接参照第二节中配置文件配置的方式修改isulad默认使用的runtime为runc,则后续使用k8s启动容器时会默认使用的runtime即为runc。 ++ ++### 使用RuntimeClass配置 ++ ++RuntimeClass 是K8s的一种内置集群资源,是一种容器运行时配置,用于运行pod中的容器。 ++ ++1. 在`/etc/isulad/daemon.json`中配置`isulad`: ++ ++ ```json ++ "runtimes": { ++ "runc-runtime": { ++ "path": "/usr/local/sbin/runc", ++ "runtime-args": [ ++ ] ++ } ++ }, ++ ``` ++ ++2. 定义 `runc-runtime.yaml`,例如创建一个`runc-runtime.yaml`内容如下:(注意handler需要与daemon.json中的名称一致) ++ ++ ```yamlapiVersion: v1 ++ apiVersion: node.k8s.io/v1beta1 ++ kind: RuntimeClass ++ metadata: ++ name: runc-runtime ++ handler: runc-runtime ++ ``` ++ ++ 之后运行`kubectl apply -f runc-runtime.yaml`命令在kubectl中让这个配置生效。 ++ ++3. 之后在创建pod时,可以在其定义的yaml文件中的`spec.runtimeClassName`中设置pod使用的runtime: ++ ++```yaml ++apiVersion: v1 ++kind: Pod ++metadata: ++ name: runc-pod-example ++spec: ++ runtimeClassName: runc-runtime ++ containers: ++ - name: runc-pod ++ image: busybox:latest ++ command: ["/bin/sh"] ++ args: ["-c", "sleep 1000"] ++``` ++ +-- +2.25.1 + diff --git a/0050-fix-isula_rt_ops_ut-bugs.patch b/0050-fix-isula_rt_ops_ut-bugs.patch new file mode 100644 index 0000000..6f96568 --- /dev/null +++ b/0050-fix-isula_rt_ops_ut-bugs.patch @@ -0,0 +1,28 @@ +From e2382f841c7bc3215793fdd8ce29132871281810 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Wed, 22 Mar 2023 08:02:51 +0800 +Subject: [PATCH 50/53] fix isula_rt_ops_ut bugs + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + test/runtime/isula/isula_rt_ops_ut.cc | 3 +++ + 1 file changed, 3 insertions(+) + +diff --git a/test/runtime/isula/isula_rt_ops_ut.cc b/test/runtime/isula/isula_rt_ops_ut.cc +index f37e62a0..9e014ac3 100644 +--- a/test/runtime/isula/isula_rt_ops_ut.cc ++++ b/test/runtime/isula/isula_rt_ops_ut.cc +@@ -211,7 +211,10 @@ TEST_F(IsulaRtOpsUnitTest, test_rt_isula_resources_stats) + TEST_F(IsulaRtOpsUnitTest, test_rt_isula_kill) + { + rt_kill_params_t kill_params = { ++ .signal = SIGTERM, ++ .stop_signal = SIGKILL, + .pid = -1, ++ .start_time = 12345, + }; + ASSERT_EQ(rt_isula_kill(nullptr, nullptr, nullptr), -1); + +-- +2.25.1 + diff --git a/0051-refactor-remote-ro-code.patch b/0051-refactor-remote-ro-code.patch new file mode 100644 index 0000000..c9269cb --- /dev/null +++ b/0051-refactor-remote-ro-code.patch @@ -0,0 +1,2089 @@ +From d4816441dc5ad67cc8cbab565e5ddc0eaf838611 Mon Sep 17 00:00:00 2001 +From: "Neil.wrz" <wangrunze13@huawei.com> +Date: Mon, 6 Mar 2023 17:43:17 -0800 +Subject: [PATCH 51/53] refactor remote ro code + +Signed-off-by: Neil.wrz <wangrunze13@huawei.com> +--- + CI/make-and-install.sh | 2 +- + .../oci/storage/image_store/CMakeLists.txt | 3 - + .../oci/storage/image_store/image_store.c | 37 +-- + .../oci/storage/image_store/image_store.h | 13 +- + .../oci/storage/layer_store/CMakeLists.txt | 3 - + .../graphdriver/overlay2/CMakeLists.txt | 3 - + .../graphdriver/overlay2/driver_overlay2.h | 8 - + .../oci/storage/layer_store/layer_store.c | 230 +++++++++--------- + .../oci/storage/layer_store/layer_store.h | 9 +- + .../image_remote_impl.c | 61 +++-- + .../layer_remote_impl.c | 71 +++--- + .../overlay_remote_impl.c | 103 +++++--- + .../remote_layer_support/remote_support.c | 141 ++++++----- + .../remote_layer_support/remote_support.h | 62 +++-- + .../ro_symlink_maintain.c | 124 +--------- + .../ro_symlink_maintain.h | 19 +- + .../modules/image/oci/storage/storage.c | 4 +- + src/utils/cutils/map/rb_tree.c | 1 + + test/image/oci/registry/CMakeLists.txt | 2 + + test/image/oci/storage/CMakeLists.txt | 3 + + test/image/oci/storage/images/CMakeLists.txt | 2 + + test/image/oci/storage/layers/CMakeLists.txt | 4 + + .../remote_layer_support/CMakeLists.txt | 44 ++++ + .../remote_layer_support/remote_layer_ut.cc | 93 +++++++ + test/image/oci/storage/rootfs/CMakeLists.txt | 2 + + test/mocks/remote_store_mock.cc | 68 ++++++ + test/mocks/remote_store_mock.h | 40 +++ + 27 files changed, 683 insertions(+), 469 deletions(-) + rename src/daemon/modules/image/oci/storage/{image_store => remote_layer_support}/image_remote_impl.c (71%) + rename src/daemon/modules/image/oci/storage/{layer_store => remote_layer_support}/layer_remote_impl.c (76%) + rename src/daemon/modules/image/oci/storage/{layer_store/graphdriver/overlay2 => remote_layer_support}/overlay_remote_impl.c (72%) + create mode 100644 test/image/oci/storage/remote_layer_support/CMakeLists.txt + create mode 100644 test/image/oci/storage/remote_layer_support/remote_layer_ut.cc + create mode 100644 test/mocks/remote_store_mock.cc + create mode 100644 test/mocks/remote_store_mock.h + +diff --git a/CI/make-and-install.sh b/CI/make-and-install.sh +index faeaf005..fa9c2250 100755 +--- a/CI/make-and-install.sh ++++ b/CI/make-and-install.sh +@@ -103,7 +103,7 @@ rm -rf build + mkdir build + cd build + if [[ ${enable_gcov} -ne 0 ]]; then +- cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_UT=ON -DENABLE_METRICS=ON .. ++ cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_UT=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON .. + else + cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DENABLE_EMBEDDED=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON .. + fi +diff --git a/src/daemon/modules/image/oci/storage/image_store/CMakeLists.txt b/src/daemon/modules/image/oci/storage/image_store/CMakeLists.txt +index 7d4fb77c..ecf21caa 100644 +--- a/src/daemon/modules/image/oci/storage/image_store/CMakeLists.txt ++++ b/src/daemon/modules/image/oci/storage/image_store/CMakeLists.txt +@@ -1,8 +1,5 @@ + # get current directory sources files + aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_image_store_srcs) +-IF (NOT ENABLE_REMOTE_LAYER_STORE) +-list(REMOVE_ITEM local_image_store_srcs "${CMAKE_CURRENT_SOURCE_DIR}/image_remote_impl.c") +-ENDIF() + + set(IMAGE_STORE_SRCS + ${local_image_store_srcs} +diff --git a/src/daemon/modules/image/oci/storage/image_store/image_store.c b/src/daemon/modules/image/oci/storage/image_store/image_store.c +index 84187ded..9a32b247 100644 +--- a/src/daemon/modules/image/oci/storage/image_store/image_store.c ++++ b/src/daemon/modules/image/oci/storage/image_store/image_store.c +@@ -132,7 +132,7 @@ static void free_image_store(image_store_t *store) + (void)map_free(store->bydigest); + store->bydigest = NULL; + +- linked_list_for_each_safe(item, &(store->images_list), next) { ++ linked_list_for_each_safe (item, &(store->images_list), next) { + linked_list_del(item); + image_ref_dec((image_t *)item->elem); + free(item); +@@ -164,7 +164,7 @@ static void image_store_digest_field_kvfree(void *key, void *value) + + free(key); + if (val != NULL) { +- linked_list_for_each_safe(item, &(val->images_list), next) { ++ linked_list_for_each_safe (item, &(val->images_list), next) { + linked_list_del(item); + free(item); + item = NULL; +@@ -500,7 +500,7 @@ static void digest_image_slice_without_value(digest_image_t *digest_filter_image + return; + } + +- linked_list_for_each_safe(item, &(digest_filter_images->images_list), next) { ++ linked_list_for_each_safe (item, &(digest_filter_images->images_list), next) { + tmp = (image_t *)item->elem; + if (strcmp(tmp->simage->id, img->simage->id) == 0) { + linked_list_del(item); +@@ -581,7 +581,7 @@ static int remove_image_from_memory(const char *id) + goto out; + } + +- linked_list_for_each_safe(item, &(g_image_store->images_list), next) { ++ linked_list_for_each_safe (item, &(g_image_store->images_list), next) { + image_t *tmp = (image_t *)item->elem; + if (strcmp(tmp->simage->id, id) != 0) { + continue; +@@ -680,7 +680,7 @@ static void free_digest_image(digest_image_t *ptr) + return; + } + +- linked_list_for_each_safe(item, &(ptr->images_list), next) { ++ linked_list_for_each_safe (item, &(ptr->images_list), next) { + linked_list_del(item); + free(item); + item = NULL; +@@ -2678,7 +2678,7 @@ int image_store_get_all_images(imagetool_images_list *images_list) + goto unlock; + } + +- linked_list_for_each_safe(item, &(g_image_store->images_list), next) { ++ linked_list_for_each_safe (item, &(g_image_store->images_list), next) { + imagetool_image_summary *imginfo = NULL; + image_t *img = (image_t *)item->elem; + imginfo = get_image_summary(img); +@@ -3099,7 +3099,7 @@ out: + return ret; + } + +-int validate_manifest_schema_version_1(const char *path, bool *valid) ++int image_store_validate_manifest_schema_version_1(const char *path, bool *valid) + { + int ret = 0; + int nret; +@@ -3506,7 +3506,7 @@ static int get_images_from_json() + continue; + } + +- if (validate_manifest_schema_version_1(image_path, &valid_v1_image) != 0) { ++ if (image_store_validate_manifest_schema_version_1(image_path, &valid_v1_image) != 0) { + ERROR("Failed to validate manifest schema version 1 format"); + continue; + } +@@ -3543,7 +3543,7 @@ static void image_store_check_all_images() + return; + } + +- linked_list_for_each_safe(item, &(g_image_store->images_list), next) { ++ linked_list_for_each_safe (item, &(g_image_store->images_list), next) { + image_t *img = (image_t *)item->elem; + if (img->spec == NULL) { + ERROR("Failed to check spec info of image: %s, try to delete", img->simage->id); +@@ -3657,18 +3657,23 @@ out: + } + + #ifdef ENABLE_REMOTE_LAYER_STORE +-int append_image_by_directory_with_lock(const char *id) ++int remote_append_image_by_directory_with_lock(const char *id) + { + int ret = 0; + int nret = 0; + char image_path[PATH_MAX] = { 0x00 }; + ++ if (id == NULL) { ++ ERROR("can't add NULL remote image"); ++ return -1; ++ } ++ + if (!image_store_lock(EXCLUSIVE)) { + ERROR("Failed to lock remote image store when handle: %s", id); + return -1; + } + +- if (map_search(g_image_store->byid, (void *)id) != NULL ) { ++ if (map_search(g_image_store->byid, (void *)id) != NULL) { + DEBUG("remote image already exist, not added: %s", id); + goto out; + } +@@ -3687,10 +3692,15 @@ out: + return ret; + } + +-int remove_image_from_memory_with_lock(const char *id) ++int remote_remove_image_from_memory_with_lock(const char *id) + { + int ret = 0; + ++ if (id == NULL) { ++ ERROR("can't remove NULL remote image"); ++ return -1; ++ } ++ + if (!image_store_lock(EXCLUSIVE)) { + ERROR("Failed to lock remote image store when handle: %s", id); + return -1; +@@ -3709,9 +3719,8 @@ out: + return ret; + } + +-char *get_top_layer_from_json(const char *img_id) ++char *remote_image_get_top_layer_from_json(const char *img_id) + { +- + char *ret = NULL; + int nret = 0; + char image_path[PATH_MAX] = { 0x00 }; +diff --git a/src/daemon/modules/image/oci/storage/image_store/image_store.h b/src/daemon/modules/image/oci/storage/image_store/image_store.h +index c3cb50e3..5164cc73 100644 +--- a/src/daemon/modules/image/oci/storage/image_store/image_store.h ++++ b/src/daemon/modules/image/oci/storage/image_store/image_store.h +@@ -28,9 +28,6 @@ + #include "isula_libutils/imagetool_image.h" + #include "isula_libutils/imagetool_images_list.h" + #include "isula_libutils/imagetool_image_summary.h" +-#ifdef ENABLE_REMOTE_LAYER_STORE +-#include "remote_support.h" +-#endif + + struct storage_module_init_options; + +@@ -112,11 +109,11 @@ void image_store_free(); + imagetool_image_summary *image_store_get_image_summary(const char *id); + + #ifdef ENABLE_REMOTE_LAYER_STORE +-remote_support *image_store_impl_remote_support(); +-int validate_manifest_schema_version_1(const char *path, bool *valid); +-int append_image_by_directory_with_lock(const char *image_dir); +-int remove_image_from_memory_with_lock(const char *id); +-char *get_top_layer_from_json(const char *img_id); /* return top layer id */ ++int image_store_validate_manifest_schema_version_1(const char *path, bool *valid); ++int remote_append_image_by_directory_with_lock(const char *image_dir); ++int remote_remove_image_from_memory_with_lock(const char *id); ++// return top layer id ++char *remote_image_get_top_layer_from_json(const char *img_id); + #endif + + #ifdef __cplusplus +diff --git a/src/daemon/modules/image/oci/storage/layer_store/CMakeLists.txt b/src/daemon/modules/image/oci/storage/layer_store/CMakeLists.txt +index e04b4ad7..c218a7c0 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/CMakeLists.txt ++++ b/src/daemon/modules/image/oci/storage/layer_store/CMakeLists.txt +@@ -1,8 +1,5 @@ + # get current directory sources files + aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_layer_store_srcs) +-IF (NOT ENABLE_REMOTE_LAYER_STORE) +-list(REMOVE_ITEM local_layer_store_srcs "${CMAKE_CURRENT_SOURCE_DIR}/layer_remote_impl.c") +-ENDIF() + add_subdirectory(graphdriver) + + set(LAYER_STORE_SRCS +diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/CMakeLists.txt b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/CMakeLists.txt +index dd4e82aa..ceed16b7 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/CMakeLists.txt ++++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/CMakeLists.txt +@@ -1,8 +1,5 @@ + # get current directory sources files + aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_overlay2_srcs) +-IF (NOT ENABLE_REMOTE_LAYER_STORE) +-list(REMOVE_ITEM local_overlay2_srcs "${CMAKE_CURRENT_SOURCE_DIR}/overlay_remote_impl.c") +-ENDIF() + + set(OVERLAY2_SRCS + ${local_overlay2_srcs} +diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.h b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.h +index 5c1d93fb..e14271b1 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.h ++++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.h +@@ -22,9 +22,6 @@ + #include <stdint.h> + + #include "driver.h" +-#ifdef ENABLE_REMOTE_LAYER_STORE +-#include "remote_support.h" +-#endif + + struct driver_create_opts; + struct driver_mount_opts; +@@ -71,11 +68,6 @@ int overlay2_repair_lowers(const char *id, const char *parent, const struct grap + + int overlay2_get_layer_fs_info(const char *id, const struct graphdriver *driver, imagetool_fs_info *fs_info); + +-#ifdef ENABLE_REMOTE_LAYER_STORE +-remote_support *overlay_driver_impl_remote_support(void); +-bool overlay_remote_layer_valid(const char *layer_id); +-#endif +- + #ifdef __cplusplus + } + #endif +diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c +index 4edd0cad..79339757 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c ++++ b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c +@@ -1789,119 +1789,6 @@ out: + return ret; + } + +-static layer_t *load_one_layer_from_json(const char *id) +-{ +- int nret = 0; +- char *mount_point_path = NULL; +- char tmpdir[PATH_MAX] = { 0 }; +- char *rpath = NULL; +- layer_t *l = NULL; +- bool layer_valid = false; +- +- nret = snprintf(tmpdir, PATH_MAX, "%s/%s", g_root_dir, id); +- if (nret < 0 || nret >= PATH_MAX) { +- ERROR("Sprintf: %s failed", id); +- goto free_out; +- } +- +- mount_point_path = mountpoint_json_path(id); +- if (mount_point_path == NULL) { +- ERROR("Out of Memory"); +- goto free_out; +- } +- +- rpath = layer_json_path(id); +- if (rpath == NULL) { +- ERROR("%s is invalid layer", id); +- goto free_out; +- } +- +- l = load_layer(rpath, mount_point_path); +- if (l == NULL) { +- ERROR("load layer: %s failed, remove it", id); +- goto free_out; +- } +- +- if (do_validate_image_layer(tmpdir, l) != 0) { +- ERROR("%s is invalid image layer", id); +- goto free_out; +- } +- +- if (do_validate_rootfs_layer(l) != 0) { +- ERROR("%s is invalid rootfs layer", id); +- goto free_out; +- } +- +- layer_valid = true; +- +-free_out: +- free(rpath); +- free(mount_point_path); +- if (!layer_valid) { +- free_layer_t(l); +- l = NULL; +- } +- // always return true; +- // if load layer failed, just remove it +- return l; +-} +- +-int load_one_layer(const char *id) +-{ +- int ret = 0; +- layer_t *tl = NULL; +- int i = 0; +- +- if (!layer_store_lock(true)) { +- return -1; +- } +- +- if (map_search(g_metadata.by_id, (void *)id) != NULL) { +- DEBUG("remote layer already exist, not added: %s", id); +- goto unlock_out; +- } +- +- tl = load_one_layer_from_json(id); +- if (tl == NULL) { +- ret = -1; +- goto unlock_out; +- } +- +- if (!map_insert(g_metadata.by_id, (void *)tl->slayer->id, (void *)tl)) { +- ERROR("Insert id: %s for layer failed", tl->slayer->id); +- ret = -1; +- goto unlock_out; +- } +- +- for (; i < tl->slayer->names_len; i++) { +- // this should be done by master isulad +- // if (remove_name(tl->slayer->names[i])) { +- // should_save = true; +- // } +- if (!map_insert(g_metadata.by_name, (void *)tl->slayer->names[i], (void *)tl)) { +- ret = -1; +- ERROR("Insert name: %s for layer failed", tl->slayer->names[i]); +- goto unlock_out; +- } +- } +- ret = insert_digest_into_map(g_metadata.by_compress_digest, tl->slayer->compressed_diff_digest, tl->slayer->id); +- if (ret != 0) { +- ERROR("update layer: %s compress failed", tl->slayer->id); +- goto unlock_out; +- } +- +- ret = insert_digest_into_map(g_metadata.by_uncompress_digest, tl->slayer->diff_digest, tl->slayer->id); +- if (ret != 0) { +- ERROR("update layer: %s uncompress failed", tl->slayer->id); +- goto unlock_out; +- } +- +- ret = 0; +-unlock_out: +- layer_store_unlock(); +- return ret; +-} +- + static bool load_layer_json_cb(const char *path_name, const struct dirent *sub_dir, void *context) + { + #define LAYER_NAME_LEN 64 +@@ -2483,7 +2370,7 @@ container_inspect_graph_driver *layer_store_get_metadata_by_layer_id(const char + } + + #ifdef ENABLE_REMOTE_LAYER_STORE +-int remove_memory_stores_with_lock(const char *id) ++int remote_layer_remove_memory_stores_with_lock(const char *id) + { + int ret = 0; + +@@ -2496,6 +2383,11 @@ int remove_memory_stores_with_lock(const char *id) + goto unlock_out; + } + ++ if (map_search(g_metadata.by_id, (void *)id) == NULL) { ++ DEBUG("remote layer already removed, don't delete: %s", id); ++ goto unlock_out; ++ } ++ + ret = remove_memory_stores(id); + + unlock_out: +@@ -2503,4 +2395,114 @@ unlock_out: + + return ret; + } ++ ++static layer_t *load_one_layer_from_json(const char *id) ++{ ++ int nret = 0; ++ char *mount_point_path = NULL; ++ char tmpdir[PATH_MAX] = { 0 }; ++ char *rpath = NULL; ++ layer_t *l = NULL; ++ bool layer_valid = false; ++ ++ nret = snprintf(tmpdir, PATH_MAX, "%s/%s", g_root_dir, id); ++ if (nret < 0 || nret >= PATH_MAX) { ++ ERROR("Sprintf: %s failed", id); ++ goto free_out; ++ } ++ ++ mount_point_path = mountpoint_json_path(id); ++ if (mount_point_path == NULL) { ++ ERROR("Out of Memory"); ++ goto free_out; ++ } ++ ++ rpath = layer_json_path(id); ++ if (rpath == NULL) { ++ ERROR("%s is invalid layer", id); ++ goto free_out; ++ } ++ ++ l = load_layer(rpath, mount_point_path); ++ if (l == NULL) { ++ ERROR("load layer: %s failed, remove it", id); ++ goto free_out; ++ } ++ ++ if (do_validate_image_layer(tmpdir, l) != 0) { ++ ERROR("%s is invalid image layer", id); ++ goto free_out; ++ } ++ ++ if (do_validate_rootfs_layer(l) != 0) { ++ ERROR("%s is invalid rootfs layer", id); ++ goto free_out; ++ } ++ ++ layer_valid = true; ++ ++free_out: ++ free(rpath); ++ free(mount_point_path); ++ if (!layer_valid) { ++ free_layer_t(l); ++ l = NULL; ++ } ++ // always return true; ++ // if load layer failed, just remove it ++ return l; ++} ++ ++int remote_load_one_layer(const char *id) ++{ ++ int ret = 0; ++ layer_t *tl = NULL; ++ int i = 0; ++ ++ if (!layer_store_lock(true)) { ++ return -1; ++ } ++ ++ if (map_search(g_metadata.by_id, (void *)id) != NULL) { ++ DEBUG("remote layer already exist, not added: %s", id); ++ goto unlock_out; ++ } ++ ++ tl = load_one_layer_from_json(id); ++ if (tl == NULL) { ++ ret = -1; ++ goto unlock_out; ++ } ++ ++ if (!map_insert(g_metadata.by_id, (void *)tl->slayer->id, (void *)tl)) { ++ ERROR("Insert id: %s for layer failed", tl->slayer->id); ++ ret = -1; ++ goto unlock_out; ++ } ++ ++ for (; i < tl->slayer->names_len; i++) { ++ // this should be done by master isulad ++ if (!map_insert(g_metadata.by_name, (void *)tl->slayer->names[i], (void *)tl)) { ++ ret = -1; ++ ERROR("Insert name: %s for layer failed", tl->slayer->names[i]); ++ goto unlock_out; ++ } ++ } ++ ret = insert_digest_into_map(g_metadata.by_compress_digest, tl->slayer->compressed_diff_digest, tl->slayer->id); ++ if (ret != 0) { ++ ERROR("update layer: %s compress failed", tl->slayer->id); ++ goto unlock_out; ++ } ++ ++ ret = insert_digest_into_map(g_metadata.by_uncompress_digest, tl->slayer->diff_digest, tl->slayer->id); ++ if (ret != 0) { ++ ERROR("update layer: %s uncompress failed", tl->slayer->id); ++ goto unlock_out; ++ } ++ ++ ret = 0; ++unlock_out: ++ layer_store_unlock(); ++ return ret; ++} + #endif +diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_store.h b/src/daemon/modules/image/oci/storage/layer_store/layer_store.h +index 44bd297e..4677e5ee 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/layer_store.h ++++ b/src/daemon/modules/image/oci/storage/layer_store/layer_store.h +@@ -24,9 +24,6 @@ + #include "storage.h" + #include "io_wrapper.h" + #include "map.h" +-#ifdef ENABLE_REMOTE_LAYER_STORE +-#include "remote_support.h" +-#endif + + struct io_read_wrapper; + struct layer_list; +@@ -84,10 +81,8 @@ int layer_store_check(const char *id); + container_inspect_graph_driver *layer_store_get_metadata_by_layer_id(const char *id); + + #ifdef ENABLE_REMOTE_LAYER_STORE +-remote_support *layer_store_impl_remote_support(); +-bool layer_remote_layer_valid(const char *layer_id); +-int load_one_layer(const char *id); +-int remove_memory_stores_with_lock(const char *id); ++int remote_load_one_layer(const char *id); ++int remote_layer_remove_memory_stores_with_lock(const char *id); + #endif + + #ifdef __cplusplus +diff --git a/src/daemon/modules/image/oci/storage/image_store/image_remote_impl.c b/src/daemon/modules/image/oci/storage/remote_layer_support/image_remote_impl.c +similarity index 71% +rename from src/daemon/modules/image/oci/storage/image_store/image_remote_impl.c +rename to src/daemon/modules/image/oci/storage/remote_layer_support/image_remote_impl.c +index 20da8116..a822ea81 100644 +--- a/src/daemon/modules/image/oci/storage/image_store/image_remote_impl.c ++++ b/src/daemon/modules/image/oci/storage/remote_layer_support/image_remote_impl.c +@@ -13,27 +13,23 @@ + * Description: provide remote image store functions + ******************************************************************************/ + #define _GNU_SOURCE +-#include "image_store.h" ++#include "remote_support.h" + + #include <isula_libutils/log.h> + #include <stdio.h> + +-#include "remote_support.h" + #include "ro_symlink_maintain.h" + #include "map.h" + #include "utils_file.h" + #include "utils.h" + #include "layer_store.h" ++#include "image_store.h" + #include "utils_array.h" + +-struct remote_image_data { +- const char *image_home; +-}; +- + static map_t *image_byid_old = NULL; + static map_t *image_byid_new = NULL; + +-static void *remote_support_create(const char *remote_home, const char *remote_ro) ++struct remote_image_data *remote_image_create(const char *remote_home, const char *remote_ro) + { + struct remote_image_data *data = util_common_calloc_s(sizeof(struct remote_image_data)); + if (data == NULL) { +@@ -46,7 +42,7 @@ static void *remote_support_create(const char *remote_home, const char *remote_r + return data; + } + +-static void remote_support_destroy(void *data) ++void remote_image_destroy(struct remote_image_data *data) + { + if (data == NULL) { + return; +@@ -59,7 +55,7 @@ static void remote_support_destroy(void *data) + return; + } + +-static int remote_support_scan(void *data) ++static int remote_dir_scan(void *data) + { + int ret = 0; + int nret; +@@ -79,7 +75,7 @@ static int remote_support_scan(void *data) + image_dirs_num = util_array_len((const char **)image_dirs); + + for (i = 0; i < image_dirs_num; i++) { +- bool valid_v1_image = false; ++ bool is_v1_image = false; + + if (util_reg_match(id_patten, image_dirs[i]) != 0) { + DEBUG("Image's json is placed inside image's data directory, so skip any other file or directory: %s", +@@ -93,12 +89,14 @@ static int remote_support_scan(void *data) + continue; + } + +- if (validate_manifest_schema_version_1(image_path, &valid_v1_image) != 0) { ++ if (image_store_validate_manifest_schema_version_1(image_path, &is_v1_image) != 0) { + ERROR("Failed to validate manifest schema version 1 format"); + continue; + } + +- if (!valid_v1_image) { ++ // for refresh, we don't care v1 image, cause image should be handled by master isulad ++ // when master isulad pull images ++ if (!is_v1_image) { + map_insert(image_byid_new, util_strdup_s(image_dirs[i]), (void *)&exist); + } + } +@@ -108,12 +106,13 @@ out: + return ret; + } + +-static int remote_support_add(void *data) ++static int remote_image_add(void *data) + { + char **array_added = NULL; + char **array_deleted = NULL; + char *top_layer = NULL; + map_t *tmp_map = NULL; ++ bool exist = true; + int i = 0; + int ret = 0; + +@@ -121,26 +120,28 @@ static int remote_support_add(void *data) + return -1; + } + +- array_added = added_layers(image_byid_old, image_byid_new); +- array_deleted = deleted_layers(image_byid_old, image_byid_new); ++ array_added = remote_added_layers(image_byid_old, image_byid_new); ++ array_deleted = remote_deleted_layers(image_byid_old, image_byid_new); + + for (i = 0; i < util_array_len((const char **)array_added); i++) { +- top_layer = get_top_layer_from_json(array_added[i]); +- if (top_layer != NULL && !layer_remote_layer_valid(top_layer)) { +- ERROR("ERROR not find valid under layer, remoet image:%s not added", array_added[i]); ++ top_layer = remote_image_get_top_layer_from_json(array_added[i]); ++ if (top_layer != NULL && !remote_layer_layer_valid(top_layer)) { ++ WARN("Current not find valid under layer, remoet image:%s not added", array_added[i]); + map_remove(image_byid_new, (void *)array_added[i]); + continue; + } + +- if (append_image_by_directory_with_lock(array_added[i]) != 0) { ++ if (remote_append_image_by_directory_with_lock(array_added[i]) != 0) { + ERROR("Failed to load image into memrory: %s", array_added[i]); ++ map_remove(image_byid_new, (void *)array_added[i]); + ret = -1; + } + } + + for (i = 0; i < util_array_len((const char **)array_deleted); i++) { +- if (remove_image_from_memory_with_lock(array_deleted[i]) != 0) { ++ if (remote_remove_image_from_memory_with_lock(array_deleted[i]) != 0) { + ERROR("Failed to remove remote memory store"); ++ map_insert(image_byid_new, array_deleted[i], (void *)&exist); + ret = -1; + } + } +@@ -148,7 +149,7 @@ static int remote_support_add(void *data) + tmp_map = image_byid_old; + image_byid_old = image_byid_new; + image_byid_new = tmp_map; +- empty_map(image_byid_new); ++ map_clear(image_byid_new); + + util_free_array(array_added); + util_free_array(array_deleted); +@@ -157,17 +158,13 @@ static int remote_support_add(void *data) + return ret; + } + +-remote_support *image_store_impl_remote_support(void) +-{ +- remote_support *rs = util_common_calloc_s(sizeof(remote_support)); +- if (rs == NULL) { +- return NULL; ++void remote_image_refresh(struct remote_image_data *data) { ++ if (remote_dir_scan(data) != 0) { ++ ERROR("remote overlay failed to scan dir, skip refresh"); ++ return; + } + +- rs->create = remote_support_create; +- rs->destroy = remote_support_destroy; +- rs->scan_remote_dir = remote_support_scan; +- rs->load_item = remote_support_add; +- +- return rs; ++ if (remote_image_add(data) != 0) { ++ ERROR("refresh overlay failed"); ++ } + } +diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c b/src/daemon/modules/image/oci/storage/remote_layer_support/layer_remote_impl.c +similarity index 76% +rename from src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c +rename to src/daemon/modules/image/oci/storage/remote_layer_support/layer_remote_impl.c +index d676458c..3e3afff6 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c ++++ b/src/daemon/modules/image/oci/storage/remote_layer_support/layer_remote_impl.c +@@ -13,7 +13,7 @@ + * Description: remote layer store implementation + ******************************************************************************/ + #define _GNU_SOURCE +-#include "layer_store.h" ++#include "remote_support.h" + + #include <pthread.h> + #include <isula_libutils/log.h> +@@ -21,20 +21,15 @@ + + #include "map.h" + #include "utils.h" +-#include "remote_support.h" + #include "ro_symlink_maintain.h" ++#include "layer_store.h" + #include "path.h" + #include "driver_overlay2.h" + +-struct remote_layer_data { +- const char *layer_home; +- const char *layer_ro; +-}; +- + static map_t *layer_byid_old = NULL; + static map_t *layer_byid_new = NULL; + +-static void *remote_support_create(const char *layer_home, const char *layer_ro) ++struct remote_layer_data *remote_layer_create(const char *layer_home, const char *layer_ro) + { + struct remote_layer_data *data = util_common_calloc_s(sizeof(struct remote_layer_data)); + if (data == NULL) { +@@ -49,7 +44,7 @@ static void *remote_support_create(const char *layer_home, const char *layer_ro) + return data; + }; + +-static void remote_support_destroy(void *data) ++void remote_layer_destroy(struct remote_layer_data *data) + { + if (data == NULL) { + return; +@@ -72,10 +67,9 @@ static bool layer_walk_dir_cb(const char *path_name, const struct dirent *sub_di + return true; + } + +-static int remote_support_scan(void *data) ++static int remote_dir_scan(struct remote_layer_data *data) + { +- struct remote_layer_data *remote_data = data; +- return util_scan_subdirs(remote_data->layer_ro, layer_walk_dir_cb, data); ++ return util_scan_subdirs(data->layer_ro, layer_walk_dir_cb, data); + } + + static int remove_one_remote_layer(struct remote_layer_data *data, char *layer_id) +@@ -85,6 +79,11 @@ static int remove_one_remote_layer(struct remote_layer_data *data, char *layer_i + int nret = 0; + int ret = 0; + ++ if (layer_id == NULL) { ++ ERROR("can't delete NULL remote layer"); ++ return -1; ++ } ++ + nret = asprintf(&ro_symlink, "%s/%s", data->layer_home, layer_id); + if (nret < 0 || nret > PATH_MAX) { + SYSERROR("Create layer symbol link path failed"); +@@ -98,11 +97,14 @@ static int remove_one_remote_layer(struct remote_layer_data *data, char *layer_i + goto out; + } + ++ // return 0 if path already removed + if (util_path_remove(clean_path) != 0) { + SYSERROR("Failed to remove link path %s", clean_path); ++ ret = -1; ++ goto out; + } + +- if (remove_memory_stores_with_lock(layer_id) != 0) { ++ if (remote_layer_remove_memory_stores_with_lock(layer_id) != 0) { + ERROR("Failed to remove remote layer store memory"); + ret = -1; + } +@@ -110,7 +112,6 @@ static int remove_one_remote_layer(struct remote_layer_data *data, char *layer_i + out: + free(ro_symlink); + return ret; +- + } + + static int add_one_remote_layer(struct remote_layer_data *data, char *layer_id) +@@ -119,6 +120,11 @@ static int add_one_remote_layer(struct remote_layer_data *data, char *layer_id) + char *layer_dir = NULL; + int ret = 0; + ++ if (layer_id == NULL) { ++ ERROR("can't add NULL remote layer"); ++ return -1; ++ } ++ + ro_symlink = util_path_join(data->layer_home, layer_id); + layer_dir = util_path_join(data->layer_ro, layer_id); + +@@ -140,7 +146,7 @@ static int add_one_remote_layer(struct remote_layer_data *data, char *layer_id) + goto free_out; + } + // insert layer into memory +- if (load_one_layer(layer_id) != 0) { ++ if (remote_load_one_layer(layer_id) != 0) { + ERROR("Failed to load new layer: %s into memory", layer_id); + ret = -1; + } +@@ -152,30 +158,32 @@ free_out: + return ret; + } + +-static int remote_support_add(void *data) ++static int remote_layer_add(struct remote_layer_data *data) + { + int ret = 0; + char **array_added = NULL; + char **array_deleted = NULL; + map_t *tmp_map = NULL; ++ bool exist = true; + int i = 0; + + if (data == NULL) { + return -1; + } + +- array_added = added_layers(layer_byid_old, layer_byid_new); +- array_deleted = deleted_layers(layer_byid_old, layer_byid_new); ++ array_added = remote_added_layers(layer_byid_old, layer_byid_new); ++ array_deleted = remote_deleted_layers(layer_byid_old, layer_byid_new); + + for (i = 0; i < util_array_len((const char **)array_added); i++) { +- if (!overlay_remote_layer_valid(array_added[i]) != 0) { ++ if (!remote_overlay_layer_valid(array_added[i]) != 0) { ++ WARN("remote overlay layer current not valid: %s", array_added[i]); + map_remove(layer_byid_new, (void *)array_added[i]); +- ERROR("remote overlay layer current not valid: %s", array_added[i]); + continue; + } + + if (add_one_remote_layer(data, array_added[i]) != 0) { + ERROR("Failed to add remote layer: %s", array_added[i]); ++ map_remove(layer_byid_new, (void *)array_added[i]); + ret = -1; + } + } +@@ -183,6 +191,7 @@ static int remote_support_add(void *data) + for (i = 0; i < util_array_len((const char **)array_deleted); i++) { + if (remove_one_remote_layer(data, array_deleted[i]) != 0) { + ERROR("Failed to delete remote overlay layer: %s", array_deleted[i]); ++ map_insert(layer_byid_new, array_deleted[i], (void *)&exist); + ret = -1; + } + } +@@ -190,7 +199,7 @@ static int remote_support_add(void *data) + tmp_map = layer_byid_old; + layer_byid_old = layer_byid_new; + layer_byid_new = tmp_map; +- empty_map(layer_byid_new); ++ map_clear(layer_byid_new); + + util_free_array(array_added); + util_free_array(array_deleted); +@@ -198,22 +207,20 @@ static int remote_support_add(void *data) + return ret; + } + +-remote_support *layer_store_impl_remote_support() ++void remote_layer_refresh(struct remote_layer_data *data) + { +- remote_support *rs = util_common_calloc_s(sizeof(remote_support)); +- if (rs == NULL) { +- return NULL; ++ if (remote_dir_scan(data) != 0) { ++ ERROR("remote layer failed to scan dir, skip refresh"); ++ return; + } + +- rs->create = remote_support_create; +- rs->destroy = remote_support_destroy; +- rs->scan_remote_dir = remote_support_scan; +- rs->load_item = remote_support_add; +- +- return rs; ++ if (remote_layer_add(data) != 0) { ++ ERROR("refresh overlay failed"); ++ } + } + +-bool layer_remote_layer_valid(const char *layer_id) ++ ++bool remote_layer_layer_valid(const char *layer_id) + { + return map_search(layer_byid_old, (void *)layer_id) != NULL; + } +diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/overlay_remote_impl.c b/src/daemon/modules/image/oci/storage/remote_layer_support/overlay_remote_impl.c +similarity index 72% +rename from src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/overlay_remote_impl.c +rename to src/daemon/modules/image/oci/storage/remote_layer_support/overlay_remote_impl.c +index a674a00f..de2e583c 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/overlay_remote_impl.c ++++ b/src/daemon/modules/image/oci/storage/remote_layer_support/overlay_remote_impl.c +@@ -13,13 +13,13 @@ + * Description: provide remote implementation for driver overlay + ******************************************************************************/ + #define _GNU_SOURCE +-#include "driver_overlay2.h" ++#include "remote_support.h" + + #include <stdio.h> + + #include "map.h" +-#include "remote_support.h" + #include "ro_symlink_maintain.h" ++#include "driver_overlay2.h" + #include "isula_libutils/log.h" + #include "utils.h" + #include "utils_array.h" +@@ -29,15 +29,13 @@ + #define OVERLAY_LINK_DIR "l" + #define OVERLAY_LAYER_LINK "link" + +-struct remote_overlay_data { +- const char *overlay_home; +- const char *overlay_ro; +-}; +- ++// key: id, value: short id in 'l' dir ++// store short id to delete symbol link in 'l' dir + static map_t *overlay_byid_old = NULL; + static map_t *overlay_byid_new = NULL; ++static map_t *overlay_id_link = NULL; + +-static void *remote_support_create(const char *remote_home, const char *remote_ro) ++struct remote_overlay_data *remote_overlay_create(const char *remote_home, const char *remote_ro) + { + struct remote_overlay_data *data = util_common_calloc_s(sizeof(struct remote_overlay_data)); + if (data == NULL) { +@@ -48,11 +46,12 @@ static void *remote_support_create(const char *remote_home, const char *remote_r + data->overlay_ro = remote_ro; + overlay_byid_old = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); + overlay_byid_new = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); ++ overlay_id_link = map_new(MAP_STR_STR, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); + + return data; + } + +-static void remote_support_destroy(void *data) ++void remote_overlay_destroy(struct remote_overlay_data *data) + { + if (data == NULL) { + return; +@@ -60,6 +59,7 @@ static void remote_support_destroy(void *data) + + map_free(overlay_byid_old); + map_free(overlay_byid_new); ++ map_free(overlay_id_link); + free(data); + } + +@@ -74,10 +74,9 @@ static bool overlay_walk_dir_cb(const char *path_name, const struct dirent *sub_ + return true; + } + +-static int remote_support_scan(void *data) ++static int remote_dir_scan(struct remote_overlay_data *data) + { +- struct remote_overlay_data *remote_data = data; +- return util_scan_subdirs(remote_data->overlay_ro, overlay_walk_dir_cb, data); ++ return util_scan_subdirs(data->overlay_ro, overlay_walk_dir_cb, data); + } + + static int do_diff_symlink(const char *id, char *link_id, const char *driver_home) +@@ -128,10 +127,17 @@ out: + static int remove_one_remote_overlay_layer(struct remote_overlay_data *data, const char *overlay_id) + { + char *ro_symlink = NULL; ++ char *link_path = NULL; ++ char *link_id = NULL; + char clean_path[PATH_MAX] = { 0 }; + int nret = 0; + int ret = 0; + ++ if (overlay_id == NULL) { ++ ERROR("can't remove NULL remote layer"); ++ return -1; ++ } ++ + nret = asprintf(&ro_symlink, "%s/%s", data->overlay_home, overlay_id); + if (nret < 0 || nret > PATH_MAX) { + SYSERROR("Create layer symbol link path failed"); +@@ -149,8 +155,40 @@ static int remove_one_remote_overlay_layer(struct remote_overlay_data *data, con + SYSERROR("Failed to remove link path %s", clean_path); + } + ++ link_id = (char *)map_search(overlay_id_link, (void *)overlay_id); ++ ++ if (link_id == NULL) { ++ ERROR("Failed to find link id for overlay layer: %s", overlay_id); ++ ret = -1; ++ goto out; ++ } ++ ++ nret = asprintf(&link_path, "%s/%s/%s", data->overlay_home, OVERLAY_LINK_DIR, link_id); ++ if (nret < 0 || nret > PATH_MAX) { ++ SYSERROR("Create link path failed"); ++ ret = -1; ++ goto out; ++ } ++ ++ if (util_clean_path(link_path, clean_path, sizeof(clean_path)) == NULL) { ++ ERROR("Failed to clean path: %s", ro_symlink); ++ ret = -1; ++ goto out; ++ } ++ ++ if (util_path_remove(clean_path) != 0) { ++ SYSERROR("Failed to remove link path %s", clean_path); ++ } ++ ++ if (!map_remove(overlay_id_link, (void *)overlay_id)) { ++ ERROR("Failed to remove link id for overlay layers: %s", overlay_id); ++ ret = -1; ++ goto out; ++ } ++ + out: + free(ro_symlink); ++ free(link_path); + return ret; + } + +@@ -162,6 +200,11 @@ static int add_one_remote_overlay_layer(struct remote_overlay_data *data, const + char *diff_symlink = NULL; + int ret = 0; + ++ if (overlay_id == NULL) { ++ ERROR("can't add NULL remote layer"); ++ return -1; ++ } ++ + ro_symlink = util_path_join(data->overlay_home, overlay_id); + if (ro_symlink == NULL) { + ERROR("Failed to join ro symlink path: %s", overlay_id); +@@ -211,6 +254,11 @@ static int add_one_remote_overlay_layer(struct remote_overlay_data *data, const + ret = -1; + } + ++ if (!map_insert(overlay_id_link, util_strdup_s(overlay_id), (void *)diff_symlink)) { ++ ERROR("can't insert remote layer into map"); ++ ret = -1; ++ } ++ + free_out: + free(ro_symlink); + free(layer_dir); +@@ -220,24 +268,26 @@ free_out: + return ret; + } + +-static int remote_support_add(void *data) ++static int remote_image_add(struct remote_overlay_data *data) + { + int ret = 0; + char **array_added = NULL; + char **array_deleted = NULL; + map_t *tmp_map = NULL; ++ bool exist = true; + int i = 0; + + if (data == NULL) { + return -1; + } + +- array_added = added_layers(overlay_byid_old, overlay_byid_new); +- array_deleted = deleted_layers(overlay_byid_old, overlay_byid_new); ++ array_added = remote_added_layers(overlay_byid_old, overlay_byid_new); ++ array_deleted = remote_deleted_layers(overlay_byid_old, overlay_byid_new); + + for (i = 0; i < util_array_len((const char **)array_added); i++) { + if (add_one_remote_overlay_layer(data, array_added[i]) != 0) { + ERROR("Failed to add remote overlay layer: %s", array_added[i]); ++ map_remove(overlay_byid_new, (void *)array_added[i]); + ret = -1; + } + } +@@ -245,6 +295,7 @@ static int remote_support_add(void *data) + for (i = 0; i < util_array_len((const char **)array_deleted); i++) { + if (remove_one_remote_overlay_layer(data, array_deleted[i]) != 0) { + ERROR("Failed to delete remote overlay layer: %s", array_deleted[i]); ++ map_insert(overlay_byid_new, array_deleted[i], (void *)&exist); + ret = -1; + } + } +@@ -252,7 +303,7 @@ static int remote_support_add(void *data) + tmp_map = overlay_byid_old; + overlay_byid_old = overlay_byid_new; + overlay_byid_new = tmp_map; +- empty_map(overlay_byid_new); ++ map_clear(overlay_byid_new); + + util_free_array(array_added); + util_free_array(array_deleted); +@@ -260,23 +311,19 @@ static int remote_support_add(void *data) + return ret; + } + +-remote_support *overlay_driver_impl_remote_support(void) ++void remote_overlay_refresh(struct remote_overlay_data *data) + { +- remote_support *rs = util_common_calloc_s(sizeof(remote_support)); +- if (rs == NULL) { +- ERROR("Failed to calloc overlay supporter"); +- return NULL; ++ if (remote_dir_scan(data) != 0) { ++ ERROR("remote overlay failed to scan dir, skip refresh"); ++ return; + } + +- rs->create = remote_support_create; +- rs->destroy = remote_support_destroy; +- rs->scan_remote_dir = remote_support_scan; +- rs->load_item = remote_support_add; +- +- return rs; ++ if (remote_image_add(data) != 0) { ++ ERROR("refresh overlay failed"); ++ } + } + +-bool overlay_remote_layer_valid(const char *layer_id) ++bool remote_overlay_layer_valid(const char *layer_id) + { + return map_search(overlay_byid_old, (void *)layer_id) != NULL; + } +diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c +index 9dc096f7..3c7d0f54 100644 +--- a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c ++++ b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c +@@ -15,108 +15,107 @@ + + #include "remote_support.h" + +-#include "layer_store.h" +-#include "image_store.h" ++#include <pthread.h> ++ + #include "isula_libutils/log.h" +-#include "driver_overlay2.h" + #include "utils.h" + +-remote_supporter *create_layer_supporter(const char *remote_home, const char *remote_ro) +-{ +- remote_support *handlers = layer_store_impl_remote_support(); +- if (handlers == NULL || handlers->create == NULL) { +- return NULL; +- } ++struct supporters { ++ struct remote_image_data *image_data; ++ struct remote_layer_data *layer_data; ++ struct remote_overlay_data *overlay_data; ++}; + +- remote_supporter *supporter = (remote_supporter *)util_common_calloc_s(sizeof(remote_supporter)); +- if (supporter == NULL) { +- goto err_out; +- } ++static struct supporters supporters; ++ ++static void *remote_refresh_ro_symbol_link(void *arg) ++{ ++ struct supporters *refresh_supporters = (struct supporters *)arg; ++ prctl(PR_SET_NAME, "RoLayerRefresh"); + +- supporter->handlers = handlers; +- supporter->data = handlers->create(remote_home, remote_ro); ++ while (true) { ++ util_usleep_nointerupt(5 * 1000 * 1000); ++ DEBUG("remote refresh start\n"); + +- return supporter; ++ remote_overlay_refresh(refresh_supporters->overlay_data); ++ remote_layer_refresh(refresh_supporters->layer_data); ++ remote_image_refresh(refresh_supporters->image_data); + +-err_out: +- free(handlers); +- free(supporter); ++ DEBUG("remote refresh end\n"); ++ } + return NULL; + } + +-remote_supporter *create_image_supporter(const char *remote_home, const char *remote_ro) ++int remote_start_refresh_thread(void) + { +- remote_support *handlers = image_store_impl_remote_support(); +- if (handlers == NULL || handlers->create == NULL) { +- return NULL; +- } ++ int res = 0; ++ pthread_t a_thread; ++ maintain_context ctx = get_maintain_context(); + +- remote_supporter *supporter = (remote_supporter *)util_common_calloc_s(sizeof(remote_supporter)); +- if (supporter == NULL) { +- goto err_out; ++ supporters.image_data = remote_image_create(ctx.image_home, NULL); ++ if (supporters.image_data == NULL) { ++ goto free_out; + } + +- supporter->handlers = handlers; +- supporter->data = handlers->create(remote_home, remote_ro); +- +- return supporter; +- +-err_out: +- free(handlers); +- free(supporter); +- return NULL; +-} ++ supporters.layer_data = remote_layer_create(ctx.layer_home, ctx.layer_ro_dir); ++ if (supporters.layer_data == NULL) { ++ goto free_out; ++ } + +-remote_supporter *create_overlay_supporter(const char *remote_home, const char *remote_ro) +-{ +- remote_support *handlers = overlay_driver_impl_remote_support(); +- if (handlers == NULL || handlers->create == NULL) { +- return NULL; ++ supporters.overlay_data = remote_overlay_create(ctx.overlay_home, ctx.overlay_ro_dir); ++ if (supporters.overlay_data == NULL) { ++ goto free_out; + } + +- remote_supporter *supporter = (remote_supporter *)util_common_calloc_s(sizeof(remote_supporter)); +- if (supporter == NULL) { +- goto err_out; ++ res = pthread_create(&a_thread, NULL, remote_refresh_ro_symbol_link, (void *)&supporters); ++ if (res != 0) { ++ CRIT("Thread creation failed"); ++ return -1; + } + +- supporter->handlers = handlers; +- supporter->data = handlers->create(remote_home, remote_ro); ++ if (pthread_detach(a_thread) != 0) { ++ SYSERROR("Failed to detach 0x%lx", a_thread); ++ return -1; ++ } + +- return supporter; ++ return 0; + +-err_out: +- free(handlers); +- free(supporter); +- return NULL; ++free_out: ++ remote_image_destroy(supporters.image_data); ++ remote_layer_destroy(supporters.layer_data); ++ remote_overlay_destroy(supporters.overlay_data); + ++ return -1; + } + +-void destroy_suppoter(remote_supporter *supporter) ++// this function calculate map_a - map_b => diff_list ++// diff_list contains keys inside map_a but not inside map_b ++static char **map_diff(const map_t *map_a, const map_t *map_b) + { +- if (supporter->handlers->destroy == NULL) { +- ERROR("destroy_supporter operation not supported"); +- return; ++ char **array = NULL; ++ map_itor *itor = map_itor_new(map_a); ++ bool *found = NULL; ++ ++ // iter new_map, every item not in old, append them to new_layers ++ for (; map_itor_valid(itor); map_itor_next(itor)) { ++ char *id = map_itor_key(itor); ++ found = map_search(map_b, id); ++ if (found == NULL) { ++ util_array_append(&array, util_strdup_s(id)); ++ } + } + +- supporter->handlers->destroy(supporter->data); +- free(supporter->handlers); +- free(supporter); ++ map_itor_free(itor); ++ ++ return array; + } + +-int scan_remote_dir(remote_supporter *supporter) ++char **remote_deleted_layers(const map_t *old, const map_t *new) + { +- if (supporter->handlers->scan_remote_dir == NULL) { +- ERROR("scan_remote_dir operation not supported"); +- return -1; +- } +- return supporter->handlers->scan_remote_dir(supporter->data); ++ return map_diff(old, new); + } + +-int load_item(remote_supporter *supporter) ++char **remote_added_layers(const map_t *old, const map_t *new) + { +- if (supporter->handlers->scan_remote_dir == NULL) { +- ERROR("load_item operation not supported"); +- return -1; +- } +- return supporter->handlers->load_item(supporter->data); ++ return map_diff(new, old); + } +diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h +index d1f7af35..892a9155 100644 +--- a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h ++++ b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h +@@ -17,39 +17,59 @@ + #define DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_REMOTE_SUPPORT_H + + #include "linked_list.h" +-#define REMOTE_RO_LAYER_DIR "RO" +-#define OVERLAY_RO_DIR "RO" ++#include "map.h" ++#include "ro_symlink_maintain.h" + + #ifdef __cplusplus + extern "C" { + #endif + +-typedef struct { +- void *(*create)(const char *remote_home, const char *remote_ro); +- void (*destroy)(void *data); +- // populate the list contains all dirs +- int (*scan_remote_dir)(void *data); +- // consume the list contains all dirs +- int (*load_item)(void *data); +-} remote_support; ++struct remote_overlay_data { ++ const char *overlay_home; ++ const char *overlay_ro; ++}; + +-typedef struct { +- void *data; +- remote_support *handlers; +-} remote_supporter; ++struct remote_layer_data { ++ const char *layer_home; ++ const char *layer_ro; ++}; + +-// RemoteSupport *impl_remote_support(); +-remote_supporter *create_image_supporter(const char *remote_home, const char *remote_ro); ++struct remote_image_data { ++ const char *image_home; ++}; + +-remote_supporter *create_layer_supporter(const char *remote_home, const char *remote_ro); ++// image impl ++struct remote_image_data *remote_image_create(const char *image_home, const char *image_ro); + +-remote_supporter *create_overlay_supporter(const char *remote_home, const char *remote_ro); ++void remote_image_destroy(struct remote_image_data *data); + +-void destroy_suppoter(remote_supporter *supporter); ++void remote_image_refresh(struct remote_image_data *data); + +-int scan_remote_dir(remote_supporter *supporter); ++// layer impl ++struct remote_layer_data *remote_layer_create(const char *layer_home, const char *layer_ro); + +-int load_item(remote_supporter *supporter); ++void remote_layer_destroy(struct remote_layer_data *data); ++ ++void remote_layer_refresh(struct remote_layer_data *data); ++ ++bool remote_layer_layer_valid(const char *layer_id); ++ ++// overlay impl ++struct remote_overlay_data *remote_overlay_create(const char *overlay_home, const char *overlay_ro); ++ ++void remote_overlay_destroy(struct remote_overlay_data *data); ++ ++void remote_overlay_refresh(struct remote_overlay_data *data); ++ ++bool remote_overlay_layer_valid(const char *layer_id); ++ ++// start refresh remote ++int remote_start_refresh_thread(void); ++ ++// extra map utils ++char **remote_deleted_layers(const map_t *old, const map_t *new_l); ++ ++char **remote_added_layers(const map_t *old, const map_t *new_l); + + #ifdef __cplusplus + } +diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c b/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c +index 7df7a221..a3aa3aa4 100644 +--- a/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c ++++ b/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c +@@ -17,19 +17,15 @@ + + #include <sys/prctl.h> + #include <stdio.h> ++#include <stdlib.h> + #include <pthread.h> + #include <unistd.h> +-#include "map.h" ++ + #include "path.h" + #include "linked_list.h" +-#include "layer_store.h" +-#include "layer.h" + #include "isula_libutils/log.h" +-#include "image_store.h" +-#include "remote_support.h" + #include "utils.h" + #include "utils_file.h" +-#include "stdlib.h" + + #define REMOTE_RO_LAYER_DIR "RO" + +@@ -43,14 +39,6 @@ static char *layer_home; + static char *overlay_ro_dir; + static char *overlay_home; + +-struct supporters { +- remote_supporter *image_supporter; +- remote_supporter *layer_supporter; +- remote_supporter *overlay_supporter; +-}; +- +-static struct supporters supporters; +- + int remote_image_init(const char *root_dir) + { + if (root_dir == NULL) { +@@ -134,75 +122,11 @@ void remote_maintain_cleanup(void) + overlay_ro_dir = NULL; + } + +-// to maintain the symbol links, add new symbol link and delete invalid symbol link +-// arg is const char *driver_home +-// scanning driver->home/RO/ directory, build symlink in driver->home +-static void *remote_refresh_ro_symbol_link(void *arg) +-{ +- struct supporters *supporters = (struct supporters *)arg; +- prctl(PR_SET_NAME, "RoLayerRefresh"); +- +- while (true) { +- util_usleep_nointerupt(5 * 1000 * 1000); +- DEBUG("remote refresh start\n"); +- scan_remote_dir(supporters->overlay_supporter); +- load_item(supporters->overlay_supporter); +- scan_remote_dir(supporters->layer_supporter); +- load_item(supporters->layer_supporter); +- scan_remote_dir(supporters->image_supporter); +- load_item(supporters->image_supporter); +- DEBUG("remote refresh end\n"); +- } +- return NULL; +-} +- +-int start_refresh_thread(void) +-{ +- int res = 0; +- pthread_t a_thread; +- +- supporters.image_supporter = create_image_supporter(image_home, NULL); +- if (supporters.image_supporter == NULL) { +- goto free_out; +- } +- +- supporters.layer_supporter = create_layer_supporter(layer_home, layer_ro_dir); +- if (supporters.layer_supporter == NULL) { +- goto free_out; +- } +- +- supporters.overlay_supporter = create_overlay_supporter(overlay_home, overlay_ro_dir); +- if (supporters.overlay_supporter == NULL) { +- goto free_out; +- } +- +- res = pthread_create(&a_thread, NULL, remote_refresh_ro_symbol_link, (void *)&supporters); +- if (res != 0) { +- CRIT("Thread creation failed"); +- return -1; +- } +- +- if (pthread_detach(a_thread) != 0) { +- SYSERROR("Failed to detach 0x%lx", a_thread); +- return -1; +- } +- +- return 0; +- +-free_out: +- destroy_suppoter(supporters.image_supporter); +- destroy_suppoter(supporters.layer_supporter); +- destroy_suppoter(supporters.overlay_supporter); +- +- return -1; +-} +- + static int do_build_ro_dir(const char *home, const char *id) + { + char *ro_symlink = NULL; + char *ro_layer_dir = NULL; + int nret = 0; +- // bool ret = true; + int ret = 0; + + nret = asprintf(&ro_symlink, "%s/%s", home, id); +@@ -305,43 +229,15 @@ int remote_overlay_remove_ro_dir(const char *id) + return do_remove_ro_dir(overlay_home, id); + } + +-static char **map_diff(map_t *map_a, map_t *map_b) +-{ +- char **array = NULL; +- map_itor *itor = map_itor_new(map_a); +- bool *found = NULL; +- +- // iter new_map, every item not in old, append them to new_layers +- for (; map_itor_valid(itor); map_itor_next(itor)) { +- char *id = map_itor_key(itor); +- found = map_search(map_b, id); +- if (found == NULL) { +- util_array_append(&array, util_strdup_s(id)); +- } +- } +- +- map_itor_free(itor); +- +- return array; +-} +- +-char **deleted_layers(map_t *old, map_t *new) +-{ +- return map_diff(old, new); +-} +- +-char **added_layers(map_t *old, map_t *new) ++maintain_context get_maintain_context(void) + { +- return map_diff(new, old); +-} ++ maintain_context ctx = {0x0}; + +-int empty_map(map_t *mp) +-{ +- if (mp == NULL) { +- return -1; +- } ++ ctx.image_home = image_home; ++ ctx.layer_ro_dir = layer_ro_dir; ++ ctx.layer_home = layer_home; ++ ctx.overlay_ro_dir = overlay_ro_dir; ++ ctx.overlay_home = overlay_home; + +- map_clear(mp); +- mp->store->root = mp->store->nil; +- return 0; ++ return ctx; + } +diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.h b/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.h +index 25712d40..aa2036ea 100644 +--- a/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.h ++++ b/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.h +@@ -15,12 +15,21 @@ + #ifndef DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_RO_SYMLINK_MAINTAIN_H + #define DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_RO_SYMLINK_MAINTAIN_H + +-#include "map.h" ++#define REMOTE_RO_LAYER_DIR "RO" ++#define OVERLAY_RO_DIR "RO" + + #ifdef __cplusplus + extern "C" { + #endif + ++typedef struct { ++ const char *image_home; ++ const char *layer_ro_dir; ++ const char *layer_home; ++ const char *overlay_ro_dir; ++ const char *overlay_home; ++} maintain_context; ++ + int remote_image_init(const char *root_dir); + + int remote_layer_init(const char *root_dir); +@@ -29,8 +38,6 @@ int remote_overlay_init(const char *driver_home); + + void remote_maintain_cleanup(void); + +-int start_refresh_thread(void); +- + int remote_layer_build_ro_dir(const char *id); + + int remote_overlay_build_ro_dir(const char *id); +@@ -39,11 +46,7 @@ int remote_layer_remove_ro_dir(const char *id); + + int remote_overlay_remove_ro_dir(const char *id); + +-char **deleted_layers(map_t *old, map_t *new); +- +-char **added_layers(map_t *old, map_t *new); +- +-int empty_map(map_t *mp); ++maintain_context get_maintain_context(void); + + #ifdef __cplusplus + } +diff --git a/src/daemon/modules/image/oci/storage/storage.c b/src/daemon/modules/image/oci/storage/storage.c +index 31812a22..f9830ac3 100644 +--- a/src/daemon/modules/image/oci/storage/storage.c ++++ b/src/daemon/modules/image/oci/storage/storage.c +@@ -43,7 +43,7 @@ + #include "utils_verify.h" + #include "sha256.h" + #ifdef ENABLE_REMOTE_LAYER_STORE +-#include "ro_symlink_maintain.h" ++#include "remote_support.h" + #endif + + static pthread_rwlock_t g_storage_rwlock; +@@ -1874,7 +1874,7 @@ int storage_module_init(struct storage_module_init_options *opts) + } + + #ifdef ENABLE_REMOTE_LAYER_STORE +- if (opts->enable_remote_layer && start_refresh_thread() != 0) { ++ if (opts->enable_remote_layer && remote_start_refresh_thread() != 0) { + ERROR("Failed to start remote refresh thread"); + } + #endif +diff --git a/src/utils/cutils/map/rb_tree.c b/src/utils/cutils/map/rb_tree.c +index e933003a..b85371e3 100644 +--- a/src/utils/cutils/map/rb_tree.c ++++ b/src/utils/cutils/map/rb_tree.c +@@ -134,6 +134,7 @@ void rbtree_clear(rb_tree_t *tree) + return; + } + rbtree_destroy_all(tree, tree->root); ++ tree->root = tree->nil; + } + + void rbtree_free(rb_tree_t *tree) +diff --git a/test/image/oci/registry/CMakeLists.txt b/test/image/oci/registry/CMakeLists.txt +index 13ed95b2..5475b7b5 100644 +--- a/test/image/oci/registry/CMakeLists.txt ++++ b/test/image/oci/registry/CMakeLists.txt +@@ -26,6 +26,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/registry_type.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/sysinfo.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/storage/image_store/image_store.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/registry/registry.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/registry/registry_apiv2.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/registry/registry_apiv1.c +@@ -57,6 +58,7 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/storage + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/storage/image_store ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/storage/remote_layer_support + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/registry + ${CMAKE_CURRENT_SOURCE_DIR}/../../../mocks + ) +diff --git a/test/image/oci/storage/CMakeLists.txt b/test/image/oci/storage/CMakeLists.txt +index feb81b14..7a9d77fe 100644 +--- a/test/image/oci/storage/CMakeLists.txt ++++ b/test/image/oci/storage/CMakeLists.txt +@@ -3,3 +3,6 @@ project(iSulad_UT) + add_subdirectory(images) + add_subdirectory(rootfs) + add_subdirectory(layers) ++IF (ENABLE_REMOTE_LAYER_STORE) ++add_subdirectory(remote_layer_support) ++ENDIF() +diff --git a/test/image/oci/storage/images/CMakeLists.txt b/test/image/oci/storage/images/CMakeLists.txt +index 3e6b69a4..8446ebba 100644 +--- a/test/image/oci/storage/images/CMakeLists.txt ++++ b/test/image/oci/storage/images/CMakeLists.txt +@@ -21,6 +21,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/image_store/image_type.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/registry_type.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/image_store/image_store.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../mocks/storage_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../mocks/isulad_config_mock.cc + storage_images_ut.cc) +@@ -39,6 +40,7 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/image_store ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/registry + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../mocks + ) +diff --git a/test/image/oci/storage/layers/CMakeLists.txt b/test/image/oci/storage/layers/CMakeLists.txt +index 952e9483..ae0ac9c3 100644 +--- a/test/image/oci/storage/layers/CMakeLists.txt ++++ b/test/image/oci/storage/layers/CMakeLists.txt +@@ -30,6 +30,7 @@ add_executable(${DRIVER_EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/metadata_store.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/wrapper_devmapper.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/quota/project_quota.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../mocks/driver_quota_mock.cc + storage_driver_ut.cc) +@@ -52,6 +53,7 @@ target_include_directories(${DRIVER_EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2 ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/quota + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../mocks + ) +@@ -105,6 +107,7 @@ add_executable(${LAYER_EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/wrapper_devmapper.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/quota/project_quota.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../mocks/driver_quota_mock.cc + storage_layers_ut.cc) + +@@ -129,6 +132,7 @@ target_include_directories(${LAYER_EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2 + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/quota ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../mocks + ) + +diff --git a/test/image/oci/storage/remote_layer_support/CMakeLists.txt b/test/image/oci/storage/remote_layer_support/CMakeLists.txt +new file mode 100644 +index 00000000..c36d9049 +--- /dev/null ++++ b/test/image/oci/storage/remote_layer_support/CMakeLists.txt +@@ -0,0 +1,44 @@ ++project(iSulad_UT) ++ ++SET(EXE remote_layer_support_ut) ++ ++add_executable(${EXE} ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../mocks/remote_store_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils/map/map.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support/overlay_remote_impl.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support/image_remote_impl.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support/layer_remote_impl.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils/utils_file.c ++ remote_layer_ut.cc ++ ) ++ ++target_include_directories(${EXE} PUBLIC ++ ${GTEST_INCLUDE_DIR} ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../include ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/config ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/common ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/api ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/image_store ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2 ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../mocks ++ ) ++ ++target_link_libraries(${EXE} ++ ${GTEST_BOTH_LIBRARIES} ++ ${GMOCK_LIBRARY} ++ ${GMOCK_MAIN_LIBRARY} ++ ${CMAKE_THREAD_LIBS_INIT} ++ ${ISULA_LIBUTILS_LIBRARY} ++ -lgtest -lgtest_main libutils_ut -lcrypto -lyajl -lz) ++ ++add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/image/oci/storage/remote_layer_support/remote_layer_ut.cc b/test/image/oci/storage/remote_layer_support/remote_layer_ut.cc +new file mode 100644 +index 00000000..5f5e92fb +--- /dev/null ++++ b/test/image/oci/storage/remote_layer_support/remote_layer_ut.cc +@@ -0,0 +1,93 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: wangrunze ++ * Create: 2023-03-16 ++ * Description: provide remote layer support ut ++ ******************************************************************************/ ++#include <gtest/gtest.h> ++ ++#include "remote_store_mock.h" ++#include "ro_symlink_maintain.h" ++#include "remote_support.h" ++#include "map.h" ++ ++using ::testing::Invoke; ++ ++bool invokeOverlayRemoteLayerValid(const char *id) ++{ ++ return true; /* currently always valid overlay layer */ ++} ++ ++bool invokeLayerRemoteLayerValid(const char *id) ++{ ++ return true; ++} ++ ++int invokeLayerLoadOneLayer(const char *id) ++{ ++ return 0; ++} ++ ++int invokeLayerRemoveOneLayer(const char *id) ++{ ++ return 0; ++} ++ ++int invokeImageAppendOneImage(const char *id) ++{ ++ return 0; ++} ++ ++int invokeImageRemoveOneImage(const char *id) ++{ ++ return 0; ++} ++ ++char *invokeImageGetTopLayer(const char *id) ++{ ++ return NULL; ++} ++ ++int invokeImageValidSchemaV1(const char *path, bool *valid) ++{ ++ return 0; ++} ++ ++void mockCommonAll(MockRemoteStore *mock) ++{ ++ EXPECT_CALL(*mock, LayerLoadOneLayer(::testing::_)).WillRepeatedly(Invoke(invokeLayerLoadOneLayer)); ++ EXPECT_CALL(*mock, LayerRemoveOneLayer(::testing::_)).WillRepeatedly(Invoke(invokeLayerRemoveOneLayer)); ++ ++ EXPECT_CALL(*mock, ImageAppendOneImage(::testing::_)).WillRepeatedly(Invoke(invokeImageAppendOneImage)); ++ EXPECT_CALL(*mock, ImageRemoveOneImage(::testing::_)).WillRepeatedly(Invoke(invokeImageRemoveOneImage)); ++ EXPECT_CALL(*mock, ImageGetTopLayer(::testing::_)).WillRepeatedly(Invoke(invokeImageGetTopLayer)); ++ EXPECT_CALL(*mock, ImageValidSchemaV1(::testing::_, ::testing::_)).WillRepeatedly(Invoke(invokeImageValidSchemaV1)); ++} ++ ++TEST(remote_Layer_ut, test_map_diff) ++{ ++ // old: a b x ++ // new: x b c ++ map_t *old_one = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); ++ map_t *new_one = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); ++ bool exist = true; ++ ++ map_insert(old_one, (void *)"a", (void *)&exist); ++ map_insert(old_one, (void *)"b", (void *)&exist); ++ map_insert(new_one, (void *)"b", (void *)&exist); ++ map_insert(new_one, (void *)"c", (void *)&exist); ++ ++ char **added = remote_added_layers(old_one, new_one); ++ char **deleted = remote_deleted_layers(old_one, new_one); ++ ++ ASSERT_EQ(added[0][0], 'c'); ++ ASSERT_EQ(deleted[0][0], 'a'); ++} +diff --git a/test/image/oci/storage/rootfs/CMakeLists.txt b/test/image/oci/storage/rootfs/CMakeLists.txt +index 4d7d3533..5383fbac 100644 +--- a/test/image/oci/storage/rootfs/CMakeLists.txt ++++ b/test/image/oci/storage/rootfs/CMakeLists.txt +@@ -22,6 +22,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/utils_images.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/rootfs_store/rootfs.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/rootfs_store/rootfs_store.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../mocks/storage_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../mocks/isulad_config_mock.cc + storage_rootfs_ut.cc) +@@ -41,6 +42,7 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/rootfs_store ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support + ) + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lcrypto -lyajl -lz) +diff --git a/test/mocks/remote_store_mock.cc b/test/mocks/remote_store_mock.cc +new file mode 100644 +index 00000000..c6428623 +--- /dev/null ++++ b/test/mocks/remote_store_mock.cc +@@ -0,0 +1,68 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: wangrunze ++ * Create: 2023-03-09 ++ * Description: provide mock for image store, layer store and driver overlay ++ ******************************************************************************/ ++ ++#include "remote_store_mock.h" ++ ++namespace { ++MockRemoteStore *g_remote_store_mock = nullptr; ++} ++ ++int remote_load_one_layer(const char *id) ++{ ++ if (g_remote_store_mock != nullptr) { ++ return g_remote_store_mock->LayerLoadOneLayer(id); ++ } ++ return -1; ++} ++ ++int remote_layer_remove_memory_stores_with_lock(const char *id) ++{ ++ if (g_remote_store_mock != nullptr) { ++ return g_remote_store_mock->LayerRemoveOneLayer(id); ++ } ++ return -1; ++} ++ ++int image_store_validate_manifest_schema_version_1(const char *path, bool *valid) ++{ ++ if (g_remote_store_mock != nullptr) { ++ return g_remote_store_mock->ImageValidSchemaV1(path, valid); ++ } ++ return -1; ++} ++ ++int remote_append_image_by_directory_with_lock(const char *image_dir) ++{ ++ if (g_remote_store_mock != nullptr) { ++ return g_remote_store_mock->ImageAppendOneImage(image_dir); ++ } ++ return -1; ++} ++ ++int remote_remove_image_from_memory_with_lock(const char *id) ++{ ++ if (g_remote_store_mock != nullptr) { ++ return g_remote_store_mock->ImageRemoveOneImage(id); ++ } ++ return -1; ++} ++ ++char *remote_image_get_top_layer_from_json(const char *img_id) ++{ ++ if (g_remote_store_mock != nullptr) { ++ return g_remote_store_mock->ImageGetTopLayer(img_id); ++ } ++ return nullptr; ++} +diff --git a/test/mocks/remote_store_mock.h b/test/mocks/remote_store_mock.h +new file mode 100644 +index 00000000..e8d73ef1 +--- /dev/null ++++ b/test/mocks/remote_store_mock.h +@@ -0,0 +1,40 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: wangrunze ++ * Create: 2023-03-09 ++ * Description: provide mock for image store, layer store and driver overlay ++ ******************************************************************************/ ++ ++#ifndef _ISULAD_TEST_MOCKS_REMOTE_STORE_MOCK_H ++#define _ISULAD_TEST_MOCKS_REMOTE_STORE_MOCK_H ++ ++#include <gmock/gmock.h> ++ ++#include "image_store.h" ++#include "layer_store.h" ++#include "driver_overlay2.h" ++ ++class MockRemoteStore { ++public: ++ virtual ~MockRemoteStore() = default; ++ // MOCK_METHOD1(OverlayRemoteLayerValid, bool(const char *)); ++ ++ // MOCK_METHOD1(LayerRemoteLayerValid, bool(const char *)); ++ MOCK_METHOD1(LayerLoadOneLayer, int(const char *)); ++ MOCK_METHOD1(LayerRemoveOneLayer, int(const char *)); ++ ++ MOCK_METHOD1(ImageAppendOneImage, int(const char *)); ++ MOCK_METHOD1(ImageRemoveOneImage, int(const char *)); ++ MOCK_METHOD1(ImageGetTopLayer, char *(const char *)); ++ MOCK_METHOD2(ImageValidSchemaV1, int(const char *, bool *)); ++}; ++ ++#endif // _ISULAD_TEST_MOCKS_IMAGE_MOCK_H +-- +2.25.1 + diff --git a/0052-add-ci-for-runc.patch b/0052-add-ci-for-runc.patch new file mode 100644 index 0000000..24c4194 --- /dev/null +++ b/0052-add-ci-for-runc.patch @@ -0,0 +1,1666 @@ +From b7b26805535a73eb647111d247d0e7c5f2c81856 Mon Sep 17 00:00:00 2001 +From: zhongtao <zhongtao17@huawei.com> +Date: Wed, 22 Mar 2023 09:50:56 +0800 +Subject: [PATCH 52/53] add ci for runc + +Signed-off-by: zhongtao <zhongtao17@huawei.com> +--- + CI/test_cases/container_cases/annotaion.sh | 31 ++-- + .../container_cases/bind_special_dir.sh | 10 +- + .../check_fd_leakage_of_isulad.sh | 31 +++- + .../container_cases/cleanup_leftover.sh | 49 ++++--- + .../container_cases/cni_bridge_test.sh | 14 +- + CI/test_cases/container_cases/cpu_rt.sh | 88 ++++++----- + CI/test_cases/container_cases/create.sh | 41 ++++-- + .../container_cases/cri_default_namespace.sh | 28 ++-- + CI/test_cases/container_cases/cri_pod_ip.sh | 18 ++- + CI/test_cases/container_cases/cri_stream.sh | 58 ++++++-- + CI/test_cases/container_cases/kill.sh | 19 ++- + CI/test_cases/container_cases/log_test.sh | 138 ++++++++++-------- + CI/test_cases/container_cases/pause.sh | 21 ++- + CI/test_cases/container_cases/resume.sh | 25 ++-- + CI/test_cases/container_cases/rm.sh | 29 ++-- + CI/test_cases/container_cases/start.sh | 39 +++-- + CI/test_cases/container_cases/stats.sh | 16 +- + CI/test_cases/container_cases/ulimit.sh | 24 +-- + CI/test_cases/container_cases/update.sh | 46 ++++-- + CI/test_cases/helpers.sh | 5 +- + 20 files changed, 472 insertions(+), 258 deletions(-) + +diff --git a/CI/test_cases/container_cases/annotaion.sh b/CI/test_cases/container_cases/annotaion.sh +index 65c474e8..93b432e5 100755 +--- a/CI/test_cases/container_cases/annotaion.sh ++++ b/CI/test_cases/container_cases/annotaion.sh +@@ -21,13 +21,12 @@ + + declare -r curr_path=$(dirname $(readlink -f "$0")) + source ../helpers.sh +-test="annotation test => test_annotation" + + function test_label() + { + local ret=0 + +- isula run -tid --name annotation --label "test_long_label=111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111" busybox sh ++ isula run -tid --name annotation --runtime $1 --label "test_long_label=111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111" busybox sh + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container for long label" && ((ret++)) + + isula inspect annotation +@@ -52,7 +51,7 @@ function test_annotation() + { + local ret=0 + +- isula run -tid --name annotation --annotation "test_long_label=111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111" busybox sh ++ isula run -tid --name annotation --runtime $1 --annotation "test_long_label=111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111" busybox sh + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container for long label" && ((ret++)) + + isula inspect annotation +@@ -73,14 +72,28 @@ function test_annotation() + return ${ret} + } + +-declare -i ans=0 ++function do_test_t() ++{ ++ local ret=0 ++ local runtime=$1 ++ local test="annotation test => $runtime" ++ msg_info "${test} starting..." + +-msg_info "${test} starting..." +-[[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - start isulad failed" && ((ret++)) ++ test_label $runtime || ((ret++)) ++ test_annotation $runtime || ((ret++)) + +-test_label || ((ans++)) +-test_annotation || ((ans++)) ++ msg_info "${test} finished with return ${ret}..." ++ return $ret ++} ++ ++declare -i ans=0 + +-msg_info "${test} finished with return ${ans}..." ++for element in ${RUNTIME_LIST[@]}; ++do ++ do_test_t $element ++ if [ $? -ne 0 ];then ++ let "ans=$ans + 1" ++ fi ++done + + show_result ${ans} "${curr_path}/${0}" +diff --git a/CI/test_cases/container_cases/bind_special_dir.sh b/CI/test_cases/container_cases/bind_special_dir.sh +index e855404b..e2653685 100755 +--- a/CI/test_cases/container_cases/bind_special_dir.sh ++++ b/CI/test_cases/container_cases/bind_special_dir.sh +@@ -25,8 +25,9 @@ source ../helpers.sh + function test_bind_special_dir() + { + local ret=0 ++ local runtime=$1 + local image="busybox" +- local test="container bind special directory test => (${FUNCNAME[@]})" ++ local test="container bind special directory test with ($runtime) => (${FUNCNAME[@]})" + + msg_info "${test} starting..." + +@@ -36,7 +37,7 @@ function test_bind_special_dir() + isula images | grep busybox + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${image}" && ((ret++)) + +- c_id=`isula run -itd -v -itd -v /sys/fs:/sys/fs:rw,rshared -v /proc:/proc -v /dev:/dev:ro -v /dev/pts:/dev/pts:rw busybox sh` ++ c_id=`isula run --runtime=$runtime -itd -v -itd -v /sys/fs:/sys/fs:rw,rshared -v /proc:/proc -v /dev:/dev:ro -v /dev/pts:/dev/pts:rw busybox sh` + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++)) + + isula exec -it $c_id sh -c "ls -al /sys/fs" | grep "cgroup" +@@ -51,6 +52,9 @@ function test_bind_special_dir() + + declare -i ans=0 + +-test_bind_special_dir || ((ans++)) ++for element in ${RUNTIME_LIST[@]}; ++do ++ test_bind_special_dir $element || ((ans++)) ++done + + show_result ${ans} "${curr_path}/${0}" +diff --git a/CI/test_cases/container_cases/check_fd_leakage_of_isulad.sh b/CI/test_cases/container_cases/check_fd_leakage_of_isulad.sh +index 39c9f49f..1978b587 100755 +--- a/CI/test_cases/container_cases/check_fd_leakage_of_isulad.sh ++++ b/CI/test_cases/container_cases/check_fd_leakage_of_isulad.sh +@@ -26,6 +26,10 @@ connect="grpc" + + function do_test_t_grpc() + { ++ local runtime=$1 ++ local test="do_test_t_grpc test => $runtime" ++ msg_info "${test} starting..." ++ + if [ $connect != "grpc" ];then + echo "this test is designed for grpc version" + return 0 +@@ -34,7 +38,7 @@ function do_test_t_grpc() + containername=test_fds + isulad_pid=`cat /var/run/isulad.pid` + precount=`ls /proc/$isulad_pid/fd | wc -l` +- isula create -t --name $containername busybox ++ isula create -t --name $containername --runtime $runtime busybox + fn_check_eq "$?" "0" "create failed" + testcontainer $containername inited + +@@ -61,11 +65,15 @@ function do_test_t_grpc() + curcount=`ls /proc/$isulad_pid/fd | wc -l` + fn_check_eq "$precount" "$curcount" "test failed" + ++ msg_info "${test} finished with return ${TC_RET_T}..." + return $TC_RET_T + } + + function do_test_t_rest() + { ++ local runtime=$1 ++ local test="do_test_t_rest test => $runtime" ++ msg_info "${test} starting..." + if [ $connect != "rest" ];then + echo "this test is designed for rest version" + return 0 +@@ -76,7 +84,7 @@ function do_test_t_rest() + isulad_pid=`cat /var/run/isulad.pid` + precount=`ls /proc/$isulad_pid/fd | wc -l` + +- isula create -t --name $containername busybox ++ isula create -t --name $containername --runtime $runtime busybox + fn_check_eq "$?" "0" "create failed" + testcontainer $containername inited + +@@ -118,15 +126,24 @@ function do_test_t_rest() + TC_RET_T=$(($TC_RET_T+1)) + fi + ++ msg_info "${test} finished with return ${TC_RET_T}..." + return $TC_RET_T + } + + ret=0 + +-do_test_t_grpc +-do_test_t_rest +-if [ $? -ne 0 ];then +- let "ret=$ret + 1" +-fi ++for element in ${RUNTIME_LIST[@]}; ++do ++ do_test_t_grpc $element ++ if [ $? -ne 0 ];then ++ let "ret=$ret + 1" ++ fi ++ do_test_t_rest $element ++ if [ $? -ne 0 ];then ++ let "ret=$ret + 1" ++ fi ++done ++ ++ + + show_result $ret "basic check fd leak" +diff --git a/CI/test_cases/container_cases/cleanup_leftover.sh b/CI/test_cases/container_cases/cleanup_leftover.sh +index 054f7d48..6f29180c 100755 +--- a/CI/test_cases/container_cases/cleanup_leftover.sh ++++ b/CI/test_cases/container_cases/cleanup_leftover.sh +@@ -5,26 +5,41 @@ + declare -r curr_path=$(dirname $(readlink -f "$0")) + source ../helpers.sh + +-containerid=$(isula run -tid busybox ls) ++function test_cleanup() ++{ ++ local runtime=$1 ++ local test="cleanup_test => (${runtime})" ++ msg_info "${test} starting..." + +-check_valgrind_log +-rm -rf "$LCR_ROOT_PATH/$containerid" ++ containerid=$(isula run -tid --runtime $runtime busybox ls) + +-start_isulad_with_valgrind +-wait_isulad_running ++ check_valgrind_log ++ rm -rf "$RUNTIME_ROOT_PATH/$runtime/$containerid" ++ ++ start_isulad_with_valgrind ++ wait_isulad_running + +-ret=0 +-ls "/var/lib/isulad/storage/overlay-containers/$containerid" +-if [ $? != 0 ]; then +- echo "ls can't access which is expected" + ret=0 +-else +- ret=1; +-fi +-check_valgrind_log +-[[ $? -ne 0 ]] && msg_err "cleanup leftover - memory leak, please check...." && ((ret++)) +- +-start_isulad_with_valgrind +-wait_isulad_running ++ ls "/var/lib/isulad/storage/overlay-containers/$containerid" ++ if [ $? != 0 ]; then ++ echo "ls can't access which is expected" ++ ret=0 ++ else ++ ret=1; ++ fi ++ check_valgrind_log ++ [[ $? -ne 0 ]] && msg_err "cleanup leftover - memory leak, please check...." && ((ret++)) ++ ++ start_isulad_with_valgrind ++ wait_isulad_running ++ msg_info "${test} finished with return ${ret}..." ++} ++ ++declare -i ret=0 ++ ++for element in ${RUNTIME_LIST[@]}; ++do ++ test_cleanup $element || ((ret++)) ++done + + show_result $ret "${curr_path}/${0}" +\ No newline at end of file +diff --git a/CI/test_cases/container_cases/cni_bridge_test.sh b/CI/test_cases/container_cases/cni_bridge_test.sh +index 369b284d..55c6e37c 100755 +--- a/CI/test_cases/container_cases/cni_bridge_test.sh ++++ b/CI/test_cases/container_cases/cni_bridge_test.sh +@@ -72,8 +72,9 @@ function do_post() + function do_test_help() + { + local ret=0 ++ local runtime=$4 + +- msg_info "this is $0 do_test" ++ msg_info "this is $0 do_test with runtime $runtime" + + crictl pull busybox + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Failed to pull busybox image" && ((ret++)) +@@ -81,14 +82,14 @@ function do_test_help() + crictl images | grep "mirrorgooglecontainers/pause-amd64" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Failed to find mirrorgooglecontainers/pause-amd64 image" && ((ret++)) + +- sid1=`crictl runp ${data_path}/$1` ++ sid1=`crictl runp --runtime $runtime ${data_path}/$1` + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Failed to run sandbox1" && ((ret++)) + + spid1=`isula inspect -f '{{json .State.Pid}}' $sid1` + nsenter -t $spid1 -n ifconfig eth0 + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Sandbox1 network config failed" && ((ret++)) + +- sid2=`crictl runp ${data_path}/$2` ++ sid2=`crictl runp --runtime $runtime ${data_path}/$2` + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Failed to run sandbox2" && ((ret++)) + + spid2=`isula inspect -f '{{json .State.Pid}}' $sid2` +@@ -123,7 +124,7 @@ function do_test_help() + crictl rmp $sid1 $sid2 + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - rm sandbox failed" && ((ret++)) + +- msg_info "$0 do_test finished with return ${ret}..." ++ msg_info "$0 do_test with runtime $runtime finished with return ${ret}..." + return ${ret} + } + +@@ -131,7 +132,10 @@ declare -i ans=0 + + do_pre || ((ans++)) + +-do_test_help "sandbox-config.json" "sandbox-config2.json" "10\.2\." || ((ans++)) ++for element in ${RUNTIME_LIST[@]}; ++do ++ do_test_help "sandbox-config.json" "sandbox-config2.json" "10\.2\." $element || ((ans++)) ++done + + do_post + +diff --git a/CI/test_cases/container_cases/cpu_rt.sh b/CI/test_cases/container_cases/cpu_rt.sh +index 39c0b427..bdc43a5e 100755 +--- a/CI/test_cases/container_cases/cpu_rt.sh ++++ b/CI/test_cases/container_cases/cpu_rt.sh +@@ -47,10 +47,9 @@ function test_isula_update_normal() + msg_info "${test} starting..." + + #start isulad with cpu_rt +- isulad --cpu-rt-period 1000000 --cpu-rt-runtime 950000 -l DEBUG > /dev/null 2>&1 & +- wait_isulad_running ++ start_isulad_without_valgrind --cpu-rt-period 1000000 --cpu-rt-runtime 950000 + +- c_id=`isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 1000 ${image} sh` ++ c_id=`isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 1000 --runtime $1 ${image} sh` + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++)) + + isula update --cpu-rt-period 900000 --cpu-rt-runtime 2000 $c_id +@@ -67,8 +66,7 @@ function test_isula_update_normal() + + stop_isulad_without_valgrind + #set cpu-rt to the initial state +- isulad --cpu-rt-period 1000000 --cpu-rt-runtime 0 -l DEBUG > /dev/null 2>&1 & +- wait_isulad_running ++ start_isulad_without_valgrind --cpu-rt-period 1000000 --cpu-rt-runtime 0 + + msg_info "${test} finished with return ${ret}..." + return ${ret} +@@ -84,23 +82,18 @@ function test_isula_update_abnormal() + isulad --cpu-rt-period 1000000 --cpu-rt-runtime 950000 -l DEBUG > /dev/null 2>&1 & + wait_isulad_running + +- c_id=`isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 1000 ${image} sh` ++ c_id=`isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 1000 --runtime $1 ${image} sh` + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++)) + + isula update --cpu-rt-period 800000 --cpu-rt-runtime 900000 $c_id 2>&1 | grep "Invalid --cpu-rt-runtime: rt runtime cannot be higher than rt period" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to update container cpu-rt-runtime" && ((ret++)) + +- isula update --cpu-rt-runtime 1000000 $c_id 2>&1 | grep "updating cgroup cpu.rt_runtime_us to 1000000: Invalid argument" ++ isula update --cpu-rt-runtime 1000000 $c_id 2>&1 | grep -i "invalid argument" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to update container cpu-rt-runtime" && ((ret++)) + + isula rm -f $c_id + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to rm container ${c_id}" && ((ret++)) + +- stop_isulad_without_valgrind +- #set cpu-rt to the initial state +- isulad --cpu-rt-period 1000000 --cpu-rt-runtime 0 -l DEBUG > /dev/null 2>&1 & +- wait_isulad_running +- + msg_info "${test} finished with return ${ret}..." + return ${ret} + } +@@ -113,8 +106,7 @@ function test_kernel_without_cpurt() + + msg_info "${test} starting..." + +- isulad --cpu-rt-period 1000000 --cpu-rt-runtime 950000 -l DEBUG > /dev/null 2>&1 & +- wait_isulad_running ++ start_isulad_without_valgrind --cpu-rt-period 1000000 --cpu-rt-runtime 950000 + + isula pull ${image} + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to pull image: ${image}" && return ${FAILURE} +@@ -122,7 +114,7 @@ function test_kernel_without_cpurt() + isula images | grep busybox + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${image}" && ((ret++)) + +- isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 900000 $image /bin/sh 2>&1 | grep "Your kernel does not support cgroup rt" ++ isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 900000 --runtime $1 $image /bin/sh 2>&1 | grep "Your kernel does not support cgroup rt" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - kernel does not support cpu-rt" && ((ret++)) + + msg_info "${test} finished with return ${ret}..." +@@ -137,9 +129,8 @@ function test_isula_run_abnormal() + + msg_info "${test} starting..." + +- #start isulad without cpu_rt +- isulad --cpu-rt-period 1000000 --cpu-rt-runtime 950000 -l DEBUG > /dev/null 2>&1 & +- wait_isulad_running ++ #start isulad with cpu_rt ++ start_isulad_without_valgrind --cpu-rt-period 1000000 --cpu-rt-runtime 950000 + + isula pull ${image} + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to pull image: ${image}" && return ${FAILURE} +@@ -147,25 +138,25 @@ function test_isula_run_abnormal() + isula images | grep busybox + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${image}" && ((ret++)) + +- isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime -1 $image /bin/sh 2>&1 | grep "failed to write -1" | grep "cpu.rt_runtime_us: Invalid argument" ++ isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime -1 --runtime $1 $image /bin/sh 2>&1 | grep "failed to write" | grep -i "cpu.rt_runtime_us: Invalid argument" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Invalid argument for cpu-rt-runtime" && ((ret++)) + +- isula run -itd --cpu-rt-period xx --cpu-rt-runtime 10000 $image /bin/sh 2>&1 | grep 'Invalid value "xx" for flag --cpu-rt-period: Invalid argument' ++ isula run -itd --cpu-rt-period xx --cpu-rt-runtime 10000 --runtime $1 $image /bin/sh 2>&1 | grep 'Invalid value "xx" for flag --cpu-rt-period: Invalid argument' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Invalid argument for cpu-rt-period" && ((ret++)) + +- isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime xx $image /bin/sh 2>&1 | grep 'Invalid value "xx" for flag --cpu-rt-runtime: Invalid argument' ++ isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime xx --runtime $1 $image /bin/sh 2>&1 | grep 'Invalid value "xx" for flag --cpu-rt-runtime: Invalid argument' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Invalid argument for cpu-rt-runtime" && ((ret++)) + +- isula run -itd --cpu-rt-period xx --cpu-rt-runtime xx $image /bin/sh 2>&1 | grep 'Invalid value "xx" for flag --cpu-rt-period: Invalid argument' ++ isula run -itd --cpu-rt-period xx --cpu-rt-runtime xx --runtime $1 $image /bin/sh 2>&1 | grep 'Invalid value "xx" for flag --cpu-rt-period: Invalid argument' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Invalid argument for cpu-rt-period" && ((ret++)) + +- isula run -itd --cpu-rt-period -1 --cpu-rt-runtime 10000 $image /bin/sh 2>&1 | grep "Invalid --cpu-rt-runtime: rt runtime cannot be higher than rt period" ++ isula run -itd --cpu-rt-period -1 --cpu-rt-runtime 10000 --runtime $1 $image /bin/sh 2>&1 | grep "Invalid --cpu-rt-runtime: rt runtime cannot be higher than rt period" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cpu-rt-runtime cannot be higher than cpu-rt-period" && ((ret++)) + +- isula run -itd --cpu-rt-period 100 --cpu-rt-runtime 10000 $image /bin/sh 2>&1 | grep "Invalid --cpu-rt-runtime: rt runtime cannot be higher than rt period" ++ isula run -itd --cpu-rt-period 100 --cpu-rt-runtime 10000 --runtime $1 $image /bin/sh 2>&1 | grep "Invalid --cpu-rt-runtime: rt runtime cannot be higher than rt period" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cpu-rt-runtime cannot be higher than cpu-rt-period" && ((ret++)) + +- isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 960000 $image /bin/sh 2>&1 | grep "failed to write 960000" | grep "cpu.rt_runtime_us: Invalid argument" ++ isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 960000 --runtime $1 $image /bin/sh 2>&1 | grep "failed to write" | grep -i "cpu.rt_runtime_us: Invalid argument" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Invalid argument for cpu-rt-runtime" && ((ret++)) + + msg_info "${test} finished with return ${ret}..." +@@ -177,7 +168,7 @@ function test_isula_run_normal() + local ret=0 + local image="busybox" + +- isula run -itd -n box --cpu-rt-period 1000000 --cpu-rt-runtime 1000 $image /bin/sh 2>&1 ++ isula run -itd -n box --cpu-rt-period 1000000 --cpu-rt-runtime 1000 --runtime $1 $image /bin/sh 2>&1 + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container" && ((ret++)) + + isula rm -f box +@@ -187,18 +178,41 @@ function test_isula_run_normal() + return ${ret} + } + ++function do_test() ++{ ++ local ret=0 ++ local runtime=$1 ++ local test="cpu_rt_test => (${runtime})" ++ msg_info "${test} starting..." ++ ++ if [ -f "/sys/fs/cgroup/cpu/cpu.rt_runtime_us" ];then ++ test_isula_run_abnormal $runtime|| ((ret++)) ++ test_isula_run_normal $runtime || ((ret++)) ++ test_cpurt_isulad_abnormal $runtime || ((ret++)) ++ test_isula_update_normal $runtime || ((ret++)) ++ test_isula_update_abnormal $runtime || ((ret++)) ++ else ++ test_kernel_without_cpurt $runtime || ((ans++)) ++ fi ++ ++ msg_info "${test} finished with return ${ret}..." ++ ++ return ${ret} ++} ++ + declare -i ans=0 + +-if [ -f "/sys/fs/cgroup/cpu/cpu.rt_runtime_us" ];then +- test_isula_run_abnormal || ((ans++)) +- test_isula_run_normal || ((ans++)) +- test_cpurt_isulad_abnormal || ((ans++)) +- test_isula_update_normal || ((ans++)) +- test_isula_update_abnormal || ((ans++)) +-else +- test_kernel_without_cpurt || ((ans++)) +-fi +- +-isula rm -f $(isula ps -aq) ++for element in ${RUNTIME_LIST[@]}; ++do ++ check_valgrind_log ++ ++ do_test $element || ((ans++)) ++ ++ stop_isulad_without_valgrind ++ # set cpu-rt to the initial state ++ start_isulad_with_valgrind --cpu-rt-period 1000000 --cpu-rt-runtime 0 ++ ++ isula rm -f $(isula ps -aq) ++done + + show_result ${ans} "${curr_path}/${0}" +diff --git a/CI/test_cases/container_cases/create.sh b/CI/test_cases/container_cases/create.sh +index 2c28a4fa..868a9b22 100755 +--- a/CI/test_cases/container_cases/create.sh ++++ b/CI/test_cases/container_cases/create.sh +@@ -23,8 +23,12 @@ source ../helpers.sh + + function do_test_t() + { ++ local runtime=$1 ++ local test="create_test => (${runtime})" ++ msg_info "${test} starting..." ++ + containername=test_create +- isula run -itd --name $containername busybox ++ isula run -itd --name $containername --runtime $runtime busybox + fn_check_eq "$?" "0" "create failed" + testcontainer $containername running + +@@ -34,44 +38,44 @@ function do_test_t() + isula inspect $containername + fn_check_ne "$?" "0" "inspect should fail" + +- containerid=`isula run -itd --name $containername --cpu-shares 1024 busybox` ++ containerid=`isula run -itd --name $containername --cpu-shares 1024 --runtime $runtime busybox` + fn_check_eq "$?" "0" "create failed" + +- cat "$LCR_ROOT_PATH/$containerid/config" | grep "cpu.shares = 1024" ++ cat "$RUNTIME_ROOT_PATH/$runtime/$containerid/config.json" | grep "\"shares\": 1024" + fn_check_eq "$?" "0" "create failed" + + isula rm -f $containername + fn_check_eq "$?" "0" "rm failed" + +- containerid=`isula run -itd --name $containername --cpu-quota 50000 busybox` ++ containerid=`isula run -itd --name $containername --cpu-quota 50000 --runtime $runtime busybox` + fn_check_eq "$?" "0" "create failed" + +- cat "$LCR_ROOT_PATH/$containerid/config" | grep "cpu.cfs_quota_us = 50000" ++ cat "$RUNTIME_ROOT_PATH/$runtime/$containerid/config.json" | grep "\"quota\": 50000" + fn_check_eq "$?" "0" "create failed" + + isula rm -f $containername + fn_check_eq "$?" "0" "rm failed" + +- containerid=`isula run -itd --name $containername --cpuset-cpus 0-1 busybox` ++ containerid=`isula run -itd --name $containername --cpuset-cpus 0-1 --runtime $runtime busybox` + fn_check_eq "$?" "0" "create failed" + +- cat "$LCR_ROOT_PATH/$containerid/config" | grep "cpuset.cpus = 0-1" ++ cat "$RUNTIME_ROOT_PATH/$runtime/$containerid/config.json" | grep "\"cpus\": \"0-1\"" + fn_check_eq "$?" "0" "create failed" + + isula rm -f $containername + fn_check_eq "$?" "0" "rm failed" + +- containerid=`isula run -itd --name $containername --memory 1000000000 busybox` ++ containerid=`isula run -itd --name $containername --memory 1000000000 --runtime $runtime busybox` + fn_check_eq "$?" "0" "create failed" + +- cat "$LCR_ROOT_PATH/$containerid/config" | grep "memory.limit_in_bytes = 1000000000" ++ cat "$RUNTIME_ROOT_PATH/$runtime/$containerid/config.json" | grep "\"limit\": 1000000000" + fn_check_eq "$?" "0" "create failed" + + isula rm -f $containername + fn_check_eq "$?" "0" "rm failed" + + # validate --label +- containerid=`isula run -itd --name $containername --label "iSulad=lcrd" busybox` ++ containerid=`isula run -itd --name $containername --label "iSulad=lcrd" --runtime $runtime busybox` + fn_check_eq "$?" "0" "create failed" + + isula inspect -f "{{.Config.Labels}}" ${containerid} | grep iSulad | grep lcrd +@@ -82,7 +86,7 @@ function do_test_t() + + # validate --label-file + echo "iSulad=lcrd\n abc=kkk" > ./label_file +- containerid=`isula run -itd --name $containername --label-file ./label_file busybox` ++ containerid=`isula run -itd --name $containername --label-file ./label_file --runtime $runtime busybox` + fn_check_eq "$?" "0" "create failed" + + isula inspect -f "{{.Config.Labels}}" ${containerid} | grep iSulad | grep lcrd +@@ -97,7 +101,7 @@ function do_test_t() + fn_check_eq "$?" "0" "rm failed" + + # validate --dns --dns-search --dns-opt +- containerid=`isula run -itd --name $containername --dns 8.8.8.8 --dns-opt debug --dns-search example.com busybox` ++ containerid=`isula run -itd --name $containername --dns 8.8.8.8 --dns-opt debug --dns-search example.com --runtime $runtime busybox` + fn_check_eq "$?" "0" "create failed" + + isula exec -it ${containerid} cat /etc/resolv.conf | grep "nameserver 8.8.8.8" +@@ -112,14 +116,19 @@ function do_test_t() + isula rm -f $containername + fn_check_eq "$?" "0" "rm failed" + ++ msg_info "${test} finished with return ${TC_RET_T}..." ++ + return $TC_RET_T + } + + ret=0 + +-do_test_t +-if [ $? -ne 0 ];then +- let "ret=$ret + 1" +-fi ++for element in ${RUNTIME_LIST[@]}; ++do ++ do_test_t $element ++ if [ $? -ne 0 ];then ++ let "ret=$ret + 1" ++ fi ++done + + show_result $ret "basic create" +diff --git a/CI/test_cases/container_cases/cri_default_namespace.sh b/CI/test_cases/container_cases/cri_default_namespace.sh +index 6a17289b..d013c6fa 100755 +--- a/CI/test_cases/container_cases/cri_default_namespace.sh ++++ b/CI/test_cases/container_cases/cri_default_namespace.sh +@@ -43,10 +43,11 @@ function set_up() + function test_cri_default_namespace_in_pod_fun() + { + local ret=0 +- local test="test_cri_default_namespace_in_pod_fun => (${FUNCNAME[@]})" ++ local runtime=$1 ++ local test="test_cri_default_namespace_in_pod_fun => (${runtime})" + msg_info "${test} starting..." + +- sid=$(crictl runp ${data_path}/sandbox-config.json) ++ sid=$(crictl runp --runtime $runtime ${data_path}/sandbox-config.json) + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run sandbox" && ((ret++)) + + cid=$(crictl create $sid ${data_path}/container-config-default-namespace.json ${data_path}/sandbox-config.json) +@@ -73,14 +74,6 @@ function test_cri_default_namespace_in_pod_fun() + [[ x"$sandboxns" == x"$conatainerns" ]] && msg_err "${FUNCNAME[0]}:${LINENO} - $element namespace should be not shared in pod" && ((ret++)) + done + +- msg_info "${test} finished with return ${ret}..." +- return ${ret} +-} +- +-function tear_down() +-{ +- local ret=0 +- + crictl stop $cid + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to stop container" && ((ret++)) + +@@ -93,19 +86,26 @@ function tear_down() + crictl rmp $sid + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to rm sandbox" && ((ret++)) + ++ msg_info "${test} finished with return ${ret}..." ++ return ${ret} ++} ++ ++function tear_down() ++{ + cp -f /etc/isulad/daemon.bak /etc/isulad/daemon.json + check_valgrind_log + start_isulad_with_valgrind +- +- return ${ret} + } + + declare -i ans=0 + + set_up || ((ans++)) + +-test_cri_default_namespace_in_pod_fun || ((ans++)) ++for element in ${RUNTIME_LIST[@]}; ++do ++ test_cri_default_namespace_in_pod_fun $element || ((ans++)) ++done + +-tear_down || ((ans++)) ++tear_down + + show_result ${ans} "${curr_path}/${0}" +diff --git a/CI/test_cases/container_cases/cri_pod_ip.sh b/CI/test_cases/container_cases/cri_pod_ip.sh +index 4048ed74..7bf7833b 100755 +--- a/CI/test_cases/container_cases/cri_pod_ip.sh ++++ b/CI/test_cases/container_cases/cri_pod_ip.sh +@@ -7,7 +7,6 @@ + curr_path=$(dirname $(readlink -f "$0")) + data_path=$(realpath $curr_path/criconfigs) + pause_img_path=$(realpath $curr_path/test_data) +-work_path="/var/lib/isulad/engines/lcr" + pod_config="sandbox-config.json" + source ../helpers.sh + +@@ -41,7 +40,7 @@ function do_post() + + function do_test() + { +- msg_info "this is $0 do_test" ++ msg_info "this is $0 do_test -> ($1)" + + crictl pull busybox + if [ $? -ne 0 ]; then +@@ -55,7 +54,7 @@ function do_test() + TC_RET_T=$(($TC_RET_T+1)) + fi + +- pod_id=`crictl runp ${data_path}/$pod_config` ++ pod_id=`crictl runp --runtime $1 ${data_path}/$pod_config` + if [ $? -ne 0 ]; then + msg_err "Failed to run sandbox" + TC_RET_T=$(($TC_RET_T+1)) +@@ -83,7 +82,7 @@ function do_test() + TC_RET_T=$(($TC_RET_T+1)) + fi + +- cat ${work_path}/${pod_id}/network_settings.json | grep "$ip" ++ cat ${RUNTIME_ROOT_PATH}/${1}/${pod_id}/network_settings.json | grep "$ip" + if [ $? -ne 0 ];then + msg_err "expect ip: $ip, network_settings.json cannot get it" + TC_RET_T=$(($TC_RET_T+1)) +@@ -112,10 +111,13 @@ if [ $? -ne 0 ];then + let "ret=$ret + 1" + fi + +-do_test +-if [ $? -ne 0 ];then +- let "ret=$ret + 1" +-fi ++for element in ${RUNTIME_LIST[@]}; ++do ++ do_test $element ++ if [ $? -ne 0 ];then ++ let "ret=$ret + 1" ++ fi ++done + + do_post + +diff --git a/CI/test_cases/container_cases/cri_stream.sh b/CI/test_cases/container_cases/cri_stream.sh +index 6c89ab38..6124c783 100755 +--- a/CI/test_cases/container_cases/cri_stream.sh ++++ b/CI/test_cases/container_cases/cri_stream.sh +@@ -8,8 +8,9 @@ curr_path=$(dirname $(readlink -f "$0")) + data_path=$(realpath $curr_path/criconfigs) + pause_img_path=$(realpath $curr_path/test_data) + source ../helpers.sh ++export CONTAINER_RUNTIME_ENDPOINT=unix:///var/run/isulad.sock + +-function set_up() ++function do_pre() + { + local ret=0 + local image="busybox" +@@ -21,7 +22,8 @@ function set_up() + cp /etc/isulad/daemon.json /etc/isulad/daemon.bak + sed -i "s#\"pod-sandbox-image\": \"\"#\"pod-sandbox-image\": \"mirrorgooglecontainers/pause-amd64:3.0\"#g" /etc/isulad/daemon.json + +- check_valgrind_log ++ # check_valgrind_log ++ stop_isulad_without_valgrind + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to stop isulad" && return ${FAILURE} + + start_isulad_without_valgrind +@@ -36,7 +38,13 @@ function set_up() + crictl images | grep ${podimage} + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${podimage}" && ((ret++)) + +- sid=$(crictl runp ${data_path}/sandbox-config.json) ++ return ${ret} ++} ++ ++function set_up() ++{ ++ local ret=0 ++ sid=$(crictl runp --runtime $1 ${data_path}/sandbox-config.json) + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run sandbox" && ((ret++)) + + cid=$(crictl create $sid ${data_path}/container-config.json ${data_path}/sandbox-config.json) +@@ -135,22 +143,50 @@ function tear_down() + crictl rmp $sid + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to rm sandbox" && ((ret++)) + ++ return ${ret} ++} ++ ++function do_post() ++{ + cp -f /etc/isulad/daemon.bak /etc/isulad/daemon.json ++ # check_valgrind_log ++ + stop_isulad_without_valgrind +- start_isulad_with_valgrind ++ start_isulad_without_valgrind ++} + +- return ${ret} ++function do_test_t() ++{ ++ local ret=0 ++ local runtime=$1 ++ local test="cri_stream_test => (${runtime})" ++ msg_info "${test} starting..." ++ ++ set_up $runtime || ((ret++)) ++ ++ test_cri_exec_fun || ((ret++)) ++ test_cri_exec_abn || ((ret++)) ++ ++ # runc attach not support ++ if [ $runtime == "lcr" ]; then ++ test_cri_attach || ((ret++)) ++ fi ++ tear_down || ((ret++)) ++ ++ msg_info "${test} finished with return ${ret}..." ++ ++ return $ret + } + + declare -i ans=0 + +-set_up || ((ans++)) +- +-test_cri_exec_fun || ((ans++)) +-test_cri_exec_abn || ((ans++)) ++do_pre || ((ans++)) + +-test_cri_attach || ((ans++)) ++for element in ${RUNTIME_LIST[@]}; ++do ++ do_test_t $element || ((ans++)) ++done + +-tear_down || ((ans++)) ++do_post + + show_result ${ans} "${curr_path}/${0}" +diff --git a/CI/test_cases/container_cases/kill.sh b/CI/test_cases/container_cases/kill.sh +index b31714a7..7b50b094 100755 +--- a/CI/test_cases/container_cases/kill.sh ++++ b/CI/test_cases/container_cases/kill.sh +@@ -25,8 +25,12 @@ source ../helpers.sh + + function do_test_t() + { ++ local runtime=$1 ++ local test="kill_test => (${runtime})" ++ msg_info "${test} starting..." ++ + containername=test_kill +- isula create -t --name $containername busybox ++ isula create -t --name $containername --runtime $runtime busybox + fn_check_eq "$?" "0" "create failed" + testcontainer $containername inited + +@@ -42,14 +46,19 @@ function do_test_t() + isula rm $containername + fn_check_eq "$?" "0" "rm failed" + ++ msg_info "${test} finished with return ${TC_RET_T}..." ++ + return $TC_RET_T + } + + ret=0 + +-do_test_t +-if [ $? -ne 0 ];then +- let "ret=$ret + 1" +-fi ++for element in ${RUNTIME_LIST[@]}; ++do ++ do_test_t $element ++ if [ $? -ne 0 ];then ++ let "ret=$ret + 1" ++ fi ++done + + show_result $ret "basic kill" +diff --git a/CI/test_cases/container_cases/log_test.sh b/CI/test_cases/container_cases/log_test.sh +index 119a005a..d13dd702 100755 +--- a/CI/test_cases/container_cases/log_test.sh ++++ b/CI/test_cases/container_cases/log_test.sh +@@ -24,10 +24,10 @@ function do_post() + + function do_check_item() + { +- cat ${ISULAD_ROOT_PATH}/engines/lcr/$1/config | grep console | grep "$2" ++ cat ${RUNTIME_ROOT_PATH}/$1/$2/config.json | grep console | grep "$3" + if [ $? -ne 0 ]; then +- cat ${ISULAD_ROOT_PATH}/engines/lcr/$1/config | grep console +- msg_err "expect $2" ++ cat ${RUNTIME_ROOT_PATH}/$1/$2/config.json | grep console ++ msg_err "expect $3" + TC_RET_T=$(($TC_RET_T+1)) + fi + } +@@ -42,16 +42,16 @@ function do_test_syslog_helper() + TC_RET_T=$(($TC_RET_T+1)) + fi + +- cid=`isula run -tid busybox sh` ++ cid=`isula run -tid --runtime $2 busybox sh` + if [ $? -ne 0 ]; then + msg_err "Failed to run container" + TC_RET_T=$(($TC_RET_T+1)) + fi + +- do_check_item ${cid} "logdriver = syslog" ++ do_check_item $2 ${cid} "driver\": \"syslog" + + if [ "x$1" != "x" ]; then +- do_check_item ${cid} "syslog_tag = $1" ++ do_check_item $2 ${cid} "tag\": \"$1" + fi + + isula rm -f ${cid} +@@ -74,91 +74,91 @@ function do_test_syslog_tag() + TC_RET_T=$(($TC_RET_T+1)) + fi + +- isula run -ti --log-opt="syslog-tag={{.xxx}}" busybox date ++ isula run -ti --log-opt="syslog-tag={{.xxx}}" --runtime $1 busybox date + if [ $? -eq 0 ]; then + msg_err "run container success with invalid syslog-tag" + TC_RET_T=$(($TC_RET_T+1)) + fi + +- isula run -ti --log-opt="syslog-tag={{" busybox date ++ isula run -ti --log-opt="syslog-tag={{" --runtime $1 busybox date + if [ $? -eq 0 ]; then + msg_err "run container success with invalid syslog-tag" + TC_RET_T=$(($TC_RET_T+1)) + fi + +- isula run -ti --log-opt="syslog-tag=aab{{cd" busybox date ++ isula run -ti --log-opt="syslog-tag=aab{{cd" --runtime $1 busybox date + if [ $? -eq 0 ]; then + msg_err "run container success with invalid syslog-tag" + TC_RET_T=$(($TC_RET_T+1)) + fi + +- cid=$(isula run -tid --log-opt="syslog-tag={{.DaemonName}}" busybox sh) ++ cid=$(isula run -tid --log-opt="syslog-tag={{.DaemonName}}" --runtime $1 busybox sh) + if [ $? -ne 0 ]; then + msg_err "Failed to run container" + TC_RET_T=$(($TC_RET_T+1)) + fi +- do_check_item ${cid} "logdriver = syslog" +- do_check_item ${cid} "syslog_tag = iSulad" ++ do_check_item $1 ${cid} "driver\": \"syslog" ++ do_check_item $1 ${cid} "tag\": \"iSulad" + +- cid=`isula run -tid --log-opt="syslog-tag={{.ID}}" busybox sh` ++ cid=`isula run -tid --log-opt="syslog-tag={{.ID}}" --runtime $1 busybox sh` + if [ $? -ne 0 ]; then + msg_err "Failed to run container" + TC_RET_T=$(($TC_RET_T+1)) + fi +- do_check_item ${cid} "logdriver = syslog" +- do_check_item ${cid} "syslog_tag = ${cid: 0: 12}" ++ do_check_item $1 ${cid} "driver\": \"syslog" ++ do_check_item $1 ${cid} "tag\": \"${cid: 0: 12}" + +- cid=`isula run -tid --name=haozi --log-opt="syslog-tag={{.ID}}xx{{.Name}}" busybox sh` ++ cid=`isula run -tid --name=haozi --log-opt="syslog-tag={{.ID}}xx{{.Name}}" --runtime $1 busybox sh` + if [ $? -ne 0 ]; then + msg_err "Failed to run container" + TC_RET_T=$(($TC_RET_T+1)) + fi +- do_check_item ${cid} "logdriver = syslog" +- do_check_item ${cid} "syslog_tag = ${cid: 0: 12}xxhaozi" ++ do_check_item $1 ${cid} "driver\": \"syslog" ++ do_check_item $1 ${cid} "tag\": \"${cid: 0: 12}xxhaozi" + isula rm -f haozi + +- cid=`isula run -tid --log-opt="syslog-tag={{.FullID}}" busybox sh` ++ cid=`isula run -tid --log-opt="syslog-tag={{.FullID}}" --runtime $1 busybox sh` + if [ $? -ne 0 ]; then + msg_err "Failed to run container" + TC_RET_T=$(($TC_RET_T+1)) + fi +- do_check_item ${cid} "logdriver = syslog" +- do_check_item ${cid} "syslog_tag = ${cid}" ++ do_check_item $1 ${cid} "driver\": \"syslog" ++ do_check_item $1 ${cid} "tag\": \"${cid}" + +- cid=`isula run -tid --name haozi --log-opt="syslog-tag={{.Name}}" busybox sh` ++ cid=`isula run -tid --name haozi --log-opt="syslog-tag={{.Name}}" --runtime $1 busybox sh` + if [ $? -ne 0 ]; then + msg_err "Failed to run container" + TC_RET_T=$(($TC_RET_T+1)) + fi +- do_check_item ${cid} "logdriver = syslog" +- do_check_item ${cid} "syslog_tag = haozi" ++ do_check_item $1 ${cid} "driver\": \"syslog" ++ do_check_item $1 ${cid} "tag\": \"haozi" + isula rm -f haozi + +- cid=`isula run -tid --name haozi --log-opt="syslog-tag=xx{{.Name}}yy" busybox sh` ++ cid=`isula run -tid --name haozi --log-opt="syslog-tag=xx{{.Name}}yy" --runtime $1 busybox sh` + if [ $? -ne 0 ]; then + msg_err "Failed to run container" + TC_RET_T=$(($TC_RET_T+1)) + fi +- do_check_item ${cid} "logdriver = syslog" +- do_check_item ${cid} "syslog_tag = xxhaoziyy" ++ do_check_item $1 ${cid} "driver\": \"syslog" ++ do_check_item $1 ${cid} "tag\": \"xxhaoziyy" + isula rm -f haozi + +- cid=`isula run -tid --log-opt="syslog-tag={{.ImageName}}" busybox sh` ++ cid=`isula run -tid --log-opt="syslog-tag={{.ImageName}}" --runtime $1 busybox sh` + if [ $? -ne 0 ]; then + msg_err "Failed to run container" + TC_RET_T=$(($TC_RET_T+1)) + fi +- do_check_item ${cid} "logdriver = syslog" +- do_check_item ${cid} "syslog_tag = busybox" ++ do_check_item $1 ${cid} "driver\": \"syslog" ++ do_check_item $1 ${cid} "tag\": \"busybox" + +- cid=`isula run -tid --log-opt="syslog-tag={{.ImageID}}" busybox sh` ++ cid=`isula run -tid --log-opt="syslog-tag={{.ImageID}}" --runtime $1 busybox sh` + if [ $? -ne 0 ]; then + msg_err "Failed to run container" + TC_RET_T=$(($TC_RET_T+1)) + fi + img_id=`isula inspect -f '{{.image.id}}' busybox` +- do_check_item ${cid} "logdriver = syslog" +- do_check_item ${cid} "syslog_tag = sha256:${img_id:0:5}" ++ do_check_item $1 ${cid} "driver\": \"syslog" ++ do_check_item $1 ${cid} "tag\": \"sha256:${img_id:0:5}" + + isula rm -f `isula ps -aq` + if [ $? -ne 0 ]; then +@@ -182,15 +182,15 @@ function do_test_json_file_helper() + file_size=$2 + fi + +- cid=`isula run -tid busybox sh` ++ cid=`isula run -tid --runtime $3 busybox sh` + if [ $? -ne 0 ]; then + msg_err "Failed to run container" + TC_RET_T=$(($TC_RET_T+1)) + fi + +- do_check_item ${cid} "logdriver = json-file" +- do_check_item ${cid} "rotate = $file_cnt" +- do_check_item ${cid} "size = $file_size" ++ do_check_item $3 ${cid} "driver\": \"json-file" ++ do_check_item $3 ${cid} "rotate\": \"$file_cnt" ++ do_check_item $3 ${cid} "size\": \"$file_size" + + isula rm -f ${cid} + if [ $? -ne 0 ]; then +@@ -207,40 +207,40 @@ function do_test_container_log() + cat /etc/isulad/daemon.json + ps aux | grep -i isulad + +- cid=`isula run -tid --log-driver=json-file busybox sh` ++ cid=`isula run -tid --log-driver=json-file --runtime $1 busybox sh` + if [ $? -ne 0 ]; then + msg_err "Failed to run container" + TC_RET_T=$(($TC_RET_T+1)) + fi +- do_check_item ${cid} "logdriver = json-file" +- do_check_item ${cid} "rotate = 7" +- do_check_item ${cid} "size = 1MB" ++ do_check_item $1 ${cid} "driver\": \"json-file" ++ do_check_item $1 ${cid} "rotate\": \"7" ++ do_check_item $1 ${cid} "size\": \"1MB" + +- cid=`isula run -tid --log-driver=json-file --log-opt="max-file=8" busybox sh` ++ cid=`isula run -tid --log-driver=json-file --log-opt="max-file=8" --runtime $1 busybox sh` + if [ $? -ne 0 ]; then + msg_err "Failed to run container" + TC_RET_T=$(($TC_RET_T+1)) + fi +- do_check_item ${cid} "logdriver = json-file" +- do_check_item ${cid} "rotate = 8" +- do_check_item ${cid} "size = 1MB" ++ do_check_item $1 ${cid} "driver\": \"json-file" ++ do_check_item $1 ${cid} "rotate\": \"8" ++ do_check_item $1 ${cid} "size\": \"1MB" + +- cid=`isula run -tid --log-driver=json-file --log-opt="max-size=128KB" busybox sh` ++ cid=`isula run -tid --log-driver=json-file --log-opt="max-size=128KB" --runtime $1 busybox sh` + if [ $? -ne 0 ]; then + msg_err "Failed to run container" + TC_RET_T=$(($TC_RET_T+1)) + fi +- do_check_item ${cid} "logdriver = json-file" +- do_check_item ${cid} "rotate = 7" +- do_check_item ${cid} "size = 128KB" ++ do_check_item $1 ${cid} "driver\": \"json-file" ++ do_check_item $1 ${cid} "rotate\": \"7" ++ do_check_item $1 ${cid} "size\": \"128KB" + +- cid=`isula run -tid --log-driver=json-file --log-opt="disable-log=true" busybox sh` ++ cid=`isula run -tid --log-driver=json-file --log-opt="disable-log=true" --runtime $1 busybox sh` + if [ $? -ne 0 ]; then + msg_err "Failed to run container" + TC_RET_T=$(($TC_RET_T+1)) + fi +- cat ${ISULAD_ROOT_PATH}/engines/lcr/${cid}/config | grep console | grep "logfile =" +- if [ $? -eq 0 ]; then ++ cat ${RUNTIME_ROOT_PATH}/$1/$cid/config.json | grep console | grep "\"log.console.file\": \"none\"" ++ if [ $? -ne 0 ]; then + msg_err "Failed to disable log" + TC_RET_T=$(($TC_RET_T+1)) + fi +@@ -250,32 +250,48 @@ function do_test_container_log() + } + + function do_test_container_syslog() { +- do_test_syslog_helper "xxxx" ++ do_test_syslog_helper "xxxx" $1 + +- do_test_syslog_tag ++ do_test_syslog_tag $1 + } + + function do_test() { ++ local runtime=$1 ++ local test="log_test => (${runtime})" ++ msg_info "${test} starting..." ++ + check_valgrind_log + start_isulad_with_valgrind --container-log-opts="syslog-tag=xxxx" + +- do_test_container_syslog ++ do_test_container_syslog $runtime + + check_valgrind_log + start_isulad_with_valgrind --container-log-driver=json-file --container-log-opts="max-size=10MB" --container-log-opts="max-file=3" +- do_test_json_file_helper "3" "10MB" ++ ++ do_test_json_file_helper "3" "10MB" $runtime + + check_valgrind_log + start_isulad_with_valgrind +- do_test_container_log +-} + +-ret=0 ++ do_test_container_log $runtime ++ ++ msg_info "${test} finished with return ${TC_RET_T}..." ++ ++ return $TC_RET_T ++} + + do_pre + +-do_test ++ret=0 ++ ++for element in ${RUNTIME_LIST[@]}; ++do ++ do_test $element ++ if [ $? -ne 0 ];then ++ let "ret=$ret + 1" ++ fi ++done + + do_post + +-show_result $TC_RET_T "container log test" ++show_result $ret "container log test" +diff --git a/CI/test_cases/container_cases/pause.sh b/CI/test_cases/container_cases/pause.sh +index fe749cd1..5778642b 100755 +--- a/CI/test_cases/container_cases/pause.sh ++++ b/CI/test_cases/container_cases/pause.sh +@@ -25,10 +25,12 @@ source ../helpers.sh + + function do_test_t() + { +- echo "Do not support pause function now" +- return 0 ++ local runtime=$1 ++ local test="rm_test => (${runtime})" ++ msg_info "${test} starting..." ++ + containername=test_pause +- isula create -t --name $containername busybox ++ isula create -t --name $containername --runtime $runtime busybox + fn_check_eq "$?" "0" "create failed" + testcontainer $containername inited + +@@ -48,14 +50,19 @@ function do_test_t() + isula rm -f $containername + fn_check_eq "$?" "0" "rm failed" + ++ msg_info "${test} finished with return ${TC_RET_T}..." ++ + return $TC_RET_T + } + + ret=0 + +-do_test_t +-if [ $? -ne 0 ];then +- let "ret=$ret + 1" +-fi ++for element in ${RUNTIME_LIST[@]}; ++do ++ do_test_t $element ++ if [ $? -ne 0 ];then ++ let "ret=$ret + 1" ++ fi ++done + + show_result $ret "basic pause" +diff --git a/CI/test_cases/container_cases/resume.sh b/CI/test_cases/container_cases/resume.sh +index d0b6ba98..e7de41bb 100755 +--- a/CI/test_cases/container_cases/resume.sh ++++ b/CI/test_cases/container_cases/resume.sh +@@ -25,14 +25,16 @@ source ../helpers.sh + + function do_test_t() + { +- echo "Do not support resume function now" +- return 0 ++ local runtime=$1 ++ local test="kill_test => (${runtime})" ++ msg_info "${test} starting..." ++ + containername=test_resume +- isula create -t --name $containername busybox ++ isula create -t --name $containername --runtime $runtime busybox + fn_check_eq "$?" "0" "create failed" + testcontainer $containername inited + +- isula resume $containername ++ isula unpause $containername + fn_check_ne "$?" "0" "resume should fail" + testcontainer $containername inited + +@@ -45,21 +47,26 @@ function do_test_t() + + testcontainer $containername paused + +- isula resume $containername ++ isula unpause $containername + fn_check_eq "$?" "0" "resume failed" + testcontainer $containername running + + isula rm -f $containername + fn_check_eq "$?" "0" "rm failed" + ++ msg_info "${test} finished with return ${TC_RET_T}..." ++ + return $TC_RET_T + } + + ret=0 + +-do_test_t +-if [ $? -ne 0 ];then +- let "ret=$ret + 1" +-fi ++for element in ${RUNTIME_LIST[@]}; ++do ++ do_test_t $element ++ if [ $? -ne 0 ];then ++ let "ret=$ret + 1" ++ fi ++done + + show_result $ret "basic resume" +diff --git a/CI/test_cases/container_cases/rm.sh b/CI/test_cases/container_cases/rm.sh +index 2903edab..3deae453 100755 +--- a/CI/test_cases/container_cases/rm.sh ++++ b/CI/test_cases/container_cases/rm.sh +@@ -26,7 +26,7 @@ source ../helpers.sh + function rm_stopped_container() + { + containername=test_rm_stopped +- isula create -t --name $containername busybox ++ isula create -t --name $containername --runtime $1 busybox + fn_check_eq "$?" "0" "create failed" + testcontainer $containername inited + +@@ -40,7 +40,7 @@ function rm_stopped_container() + function rm_running_container() + { + containername=test_rm_running +- isula run -td --name $containername busybox ++ isula run -td --name $containername --runtime $1 busybox + fn_check_eq "$?" "0" "run failed" + testcontainer $containername running + +@@ -61,7 +61,7 @@ function rm_running_container() + function rm_running_container_force() + { + containername=test_rm_running_force +- conID=$(isula run -td --name $containername busybox) ++ conID=$(isula run -td --name $containername --runtime $1 busybox) + fn_check_eq "$?" "0" "run failed" + testcontainer $containername running + +@@ -83,18 +83,27 @@ function rm_running_container_force() + + function do_test_t() + { +- rm_stopped_container +- rm_running_container +- rm_running_container_force ++ local runtime=$1 ++ local test="rm_test => (${runtime})" ++ msg_info "${test} starting..." ++ ++ rm_stopped_container $runtime ++ rm_running_container $runtime ++ rm_running_container_force $runtime ++ ++ msg_info "${test} finished with return ${TC_RET_T}..." + + return $TC_RET_T + } + + ret=0 + +-do_test_t +-if [ $? -ne 0 ];then +- let "ret=$ret + 1" +-fi ++for element in ${RUNTIME_LIST[@]}; ++do ++ do_test_t $element ++ if [ $? -ne 0 ];then ++ let "ret=$ret + 1" ++ fi ++done + + show_result $ret "basic rm" +diff --git a/CI/test_cases/container_cases/start.sh b/CI/test_cases/container_cases/start.sh +index c3dc9c27..c2d0d02f 100755 +--- a/CI/test_cases/container_cases/start.sh ++++ b/CI/test_cases/container_cases/start.sh +@@ -26,7 +26,7 @@ source ../helpers.sh + function do_test_t() + { + containername=test_start +- isula create -t --name $containername busybox ++ isula create -t --name $containername --runtime $1 busybox + fn_check_eq "$?" "0" "create failed" + testcontainer $containername inited + +@@ -51,7 +51,7 @@ function do_attach_local_test_t() + local test="container start with --attach local test => (${FUNCNAME[@]})" + + containername=start_attach +- isula create -ti --name $containername busybox /bin/sh -c 'echo "hello"' ++ isula create -ti --name $containername --runtime $1 busybox /bin/sh -c 'echo "hello"' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to create ${containername}" && ((ret++)) + testcontainer $containername inited + +@@ -62,7 +62,7 @@ function do_attach_local_test_t() + isula rm -f $containername + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to rm container" && ((ret++)) + +- id=`isula create -ti busybox /bin/sh -c 'ech "hello"'` ++ id=`isula create -ti --runtime $1 busybox /bin/sh -c 'ech "hello"'` + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to create container" && ((ret++)) + + isula start -a $id +@@ -88,19 +88,19 @@ function do_attach_remote_test_t() + start_isulad_with_valgrind -H "$config" + + containername=start_attach +- isula create -ti -H "$config" --name $containername busybox /bin/sh -c 'echo "hello"' ++ isula create -ti -H "$config" --name $containername --runtime $1 busybox /bin/sh -c 'echo "hello"' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to create ${containername} remote" && ((ret++)) +- testcontainer $containername inited ++ testcontainer $containername inited "-H $config" + + result=`isula start -a -H "$config" $containername` + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to start -a ${containername} remote" && ((ret++)) +- testcontainer $containername exited ++ testcontainer $containername exited "-H $config" + + isula rm -f -H "$config" $containername + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to rm container remote" && ((ret++)) + + containername=start_exit +- isula run -it -H "$config" --name $containername busybox /bin/sh -c 'exit 5' ++ isula run -it -H "$config" --name $containername --runtime $1 busybox /bin/sh -c 'exit 5' + [[ $? -ne 5 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - invalid exit code with remote start" && ((ret++)) + + isula start -a -H "$config" $containername +@@ -119,12 +119,29 @@ function do_attach_remote_test_t() + return ${ret} + } + +-declare -i ans=0 ++function test_start() ++{ ++ local ret=0 ++ local runtime=$1 ++ local test="start_test => (${runtime})" ++ msg_info "${test} starting..." ++ ++ do_test_t $runtime || ((ret++)) + +-do_test_t || ((ans++)) ++ do_attach_local_test_t $runtime || ((ret++)) + +-do_attach_local_test_t || ((ans++)) ++ do_attach_remote_test_t $runtime || ((ret++)) ++ ++ msg_info "${test} finished with return ${ret}..." ++ ++ return $ret ++} ++ ++declare -i ans=0 + +-do_attach_remote_test_t || ((ans++)) ++for element in ${RUNTIME_LIST[@]}; ++do ++ test_start $element || ((ans++)) ++done + + show_result ${ans} "${curr_path}/${0}" +diff --git a/CI/test_cases/container_cases/stats.sh b/CI/test_cases/container_cases/stats.sh +index 705f7a7c..df241b90 100755 +--- a/CI/test_cases/container_cases/stats.sh ++++ b/CI/test_cases/container_cases/stats.sh +@@ -26,7 +26,8 @@ function test_stats_spec() + { + local ret=0 + local image="busybox" +- local test="container stats test => (${FUNCNAME[@]})" ++ local runtime=$1 ++ local test="container stats test with (${runtime}) => (${FUNCNAME[@]})" + statslog=/tmp/stats.log + + msg_info "${test} starting..." +@@ -44,11 +45,11 @@ function test_stats_spec() + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++)) + + container_name_init=stats_inited +- id_init=`isula create -t -n $container_name_init $image /bin/sh` ++ id_init=`isula create -t -n $container_name_init --runtime $runtime $image /bin/sh` + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++)) + + container_name_running=stats_running +- id_running=`isula run -td -n $container_name_running $image /bin/sh` ++ id_running=`isula run -td -n $container_name_running --runtime $runtime $image /bin/sh` + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++)) + + isula stats --no-stream > $statslog +@@ -73,14 +74,14 @@ function test_stats_spec() + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++)) + + container_name_pause=stats_paused +- id_pause=`isula run -td -n $container_name_pause $image /bin/sh` ++ id_pause=`isula run -td -n $container_name_pause --runtime $runtime $image /bin/sh` + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++)) + + isula pause $id_pause + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to pause running container" && ((ret++)) + + container_name_stop=stats_stopped +- id_stop=`isula run -td -n $container_name_stop $image /bin/sh` ++ id_stop=`isula run -td -n $container_name_stop --runtime $runtime $image /bin/sh` + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++)) + + isula stop -t 0 $id_stop +@@ -129,6 +130,9 @@ function test_stats_spec() + + declare -i ans=0 + +-test_stats_spec || ((ans++)) ++for element in ${RUNTIME_LIST[@]}; ++do ++ test_stats_spec $element || ((ans++)) ++done + + show_result ${ans} "${curr_path}/${0}" +diff --git a/CI/test_cases/container_cases/ulimit.sh b/CI/test_cases/container_cases/ulimit.sh +index c134a71a..29868041 100755 +--- a/CI/test_cases/container_cases/ulimit.sh ++++ b/CI/test_cases/container_cases/ulimit.sh +@@ -26,10 +26,11 @@ source ../helpers.sh + function test_ulimit() + { + local ret=0 ++ local runtime=$1 + local image="busybox" + ulimitlog=/tmp/ulimit.log + +- local test="ulimit test => (${FUNCNAME[@]})" ++ local test="ulimit test with (${runtime})=> (${FUNCNAME[@]})" + msg_info "${test} starting..." + + check_valgrind_log +@@ -38,37 +39,37 @@ function test_ulimit() + start_isulad_with_valgrind --default-ulimit nproc=2048:4096 --default-ulimit nproc=2048:8192 --default-ulimit nofile=1024:4096 + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - start isulad failed" && ((ret++)) + +- isula run --ulimit nproc= $image /bin/sh > $ulimitlog 2>&1 ++ isula run --ulimit nproc= $image --runtime $runtime /bin/sh > $ulimitlog 2>&1 + cat $ulimitlog | grep "delimiter '=' can't be the first or the last character" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check failed" && ((ret++)) + +- isula run --ulimit nproc=1024: $image /bin/sh > $ulimitlog 2>&1 ++ isula run --ulimit nproc=1024: --runtime $runtime $image /bin/sh > $ulimitlog 2>&1 + cat $ulimitlog | grep "delimiter ':' can't be the first or the last character" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check failed" && ((ret++)) + +- isula run --ulimit npro=1024:2048 $image /bin/sh > $ulimitlog 2>&1 ++ isula run --ulimit npro=1024:2048 --runtime $runtime $image /bin/sh > $ulimitlog 2>&1 + cat $ulimitlog | grep "Invalid ulimit type" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check failed" && ((ret++)) + +- isula run --ulimit nproc=4096:2048 $image /bin/sh > $ulimitlog 2>&1 ++ isula run --ulimit nproc=4096:2048 --runtime $runtime $image /bin/sh > $ulimitlog 2>&1 + cat $ulimitlog | grep "Ulimit soft limit must be less than or equal to hard limit" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check failed" && ((ret++)) + +- isula run --ulimit nproc=2048:4096.5 $image /bin/sh > $ulimitlog 2>&1 ++ isula run --ulimit nproc=2048:4096.5 --runtime $runtime $image /bin/sh > $ulimitlog 2>&1 + cat $ulimitlog | grep "Invalid ulimit hard value" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check failed" && ((ret++)) + +- isula run --ulimit nproc==2048:4096 $image /bin/sh > $ulimitlog 2>&1 ++ isula run --ulimit nproc==2048:4096 --runtime $runtime $image /bin/sh > $ulimitlog 2>&1 + cat $ulimitlog | grep "Invalid ulimit argument" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check failed" && ((ret++)) + +- isula run --ulimit nproc=2048::4096 $image /bin/sh > $ulimitlog 2>&1 ++ isula run --ulimit nproc=2048::4096 --runtime $runtime $image /bin/sh > $ulimitlog 2>&1 + cat $ulimitlog | grep "Too many limit value arguments" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check failed" && ((ret++)) + + container_name="ulimit_test" + +- isula run -td -n $container_name --ulimit nofile=20480:40960 --ulimit core=1024:2048 $image /bin/sh ++ isula run -td -n $container_name --ulimit nofile=20480:40960 --ulimit core=1024:2048 --runtime $runtime $image /bin/sh + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check failed" && ((ret++)) + + isula exec $container_name /bin/sh -c "cat /proc/self/limits" | grep "Max open files" |awk '{ print $(NF-1) }' |grep 40960 +@@ -106,6 +107,9 @@ function test_ulimit() + + declare -i ans=0 + +-test_ulimit || ((ans++)) ++for element in ${RUNTIME_LIST[@]}; ++do ++ test_ulimit $element || ((ans++)) ++done + + show_result ${ans} "${curr_path}/${0}" +diff --git a/CI/test_cases/container_cases/update.sh b/CI/test_cases/container_cases/update.sh +index 1e1e9cdc..d379acc2 100755 +--- a/CI/test_cases/container_cases/update.sh ++++ b/CI/test_cases/container_cases/update.sh +@@ -26,7 +26,7 @@ source ../helpers.sh + function do_test_t() + { + containername=test_update +- containerid=`isula create -t --name $containername busybox` ++ containerid=`isula create -t --runtime $1 --name $containername busybox` + fn_check_eq "$?" "0" "create failed" + testcontainer $containername inited + +@@ -101,7 +101,24 @@ function do_test_t() + + main=$(uname -r | awk -F . '{print $1}') + minor=$(uname -r | awk -F . '{print $2}') +- if [[ ${main} -lt 5 ]] || [[ ${main} -eq 5 ]] && [[ ${minor} -lt 11 ]]; then ++ enable=1 ++ if [ $1 == "runc" ]; then ++ version=$(runc --version | grep runc) ++ # Runc does not support '--kernel-memory' options from v1.0.0-rc94 version ++ limit=(1 0 0 93) ++ array=`echo $version |egrep -o "[0-9]*"` ++ index=0 ++ for i in $(echo $array| awk '{print $1,$2}') ++ do ++ echo $i ++ if [[ $i -gt ${limit[index]} ]]; then ++ enable=0 ++ break ++ fi ++ let "index+=1" ++ done ++ fi ++ if [[ ${main} -lt 5 ]] || [[ ${main} -eq 5 ]] && [[ ${minor} -lt 11 ]] && [[ ${enable} -eq 1 ]]; then + isula update --kernel-memory 2000000000 $containername + fn_check_eq "$?" "0" "update failed" + +@@ -124,7 +141,7 @@ function do_test_t() + function do_test_t1() + { + containername=test_update1 +- containerid=`isula run -itd --memory 500M --name $containername busybox` ++ containerid=`isula run -itd --runtime $1 --memory 500M --name $containername busybox` + fn_check_eq "$?" "0" "run failed" + + isula inspect $containerid | grep "MemorySwap" | grep "1048576000" +@@ -146,16 +163,23 @@ function do_test_t1() + + ret=0 + +-do_test_t +-if [ $? -ne 0 ];then +- let "ret=$ret + 1" +-fi ++for element in ${RUNTIME_LIST[@]}; ++do ++ test="update test => (${element})" ++ msg_info "${test} starting..." + +-if [ -f "/sys/fs/cgroup/memory/memory.memsw.usage_in_bytes" ];then +- do_test_t1 ++ do_test_t $element + if [ $? -ne 0 ];then +- let "ret=$ret + 1" ++ let "ret=$ret + 1" ++ fi ++ ++ if [ -f "/sys/fs/cgroup/memory/memory.memsw.usage_in_bytes" ];then ++ do_test_t1 $element ++ if [ $? -ne 0 ];then ++ let "ret=$ret + 1" ++ fi + fi +-fi ++ msg_info "${test} finished with return ${ret}..." ++done + + show_result $ret "basic update" +diff --git a/CI/test_cases/helpers.sh b/CI/test_cases/helpers.sh +index 5ea4ff94..6b4eea24 100755 +--- a/CI/test_cases/helpers.sh ++++ b/CI/test_cases/helpers.sh +@@ -21,11 +21,14 @@ declare -a lines + + # Root directory of integration tests. + LCR_ROOT_PATH="/var/lib/isulad/engines/lcr" ++RUNTIME_ROOT_PATH="/var/lib/isulad/engines" + valgrind_log="/tmp/valgrind.log" + ISUALD_LOG="/var/lib/isulad/isulad.log" + ISULAD_ROOT_PATH="/var/lib/isulad" + ISULAD_RUN_ROOT_PATH="/var/run/isulad" + ++RUNTIME_LIST=(lcr runc) ++ + enable_native_network=0 + + declare -r -i FAILURE=1 +@@ -60,7 +63,7 @@ function fn_check_ne() { + } + + function testcontainer() { +- st=`isula inspect -f '{{json .State.Status}}' "$1"` ++ st=`isula inspect -f '{{json .State.Status}}' $3 "$1"` + if ! [[ "${st}" =~ "$2" ]];then + echo "expect status $2, but get ${st}" + TC_RET_T=$(($TC_RET_T+1)) +-- +2.25.1 + diff --git a/0053-bugfix-when-refresh-can-t-load-or-pull-images.patch b/0053-bugfix-when-refresh-can-t-load-or-pull-images.patch new file mode 100644 index 0000000..da4bffe --- /dev/null +++ b/0053-bugfix-when-refresh-can-t-load-or-pull-images.patch @@ -0,0 +1,319 @@ +From 9d6df0b3065867d5ca1a597bedb10eab5a1c9235 Mon Sep 17 00:00:00 2001 +From: "Neil.wrz" <wangrunze13@huawei.com> +Date: Mon, 20 Mar 2023 23:47:25 -0700 +Subject: [PATCH 53/53] bugfix when refresh can't load or pull images + +Signed-off-by: Neil.wrz <wangrunze13@huawei.com> +--- + src/daemon/modules/image/oci/oci_image.c | 105 +++++++++++++++++- + .../remote_layer_support/remote_support.c | 34 +++++- + .../remote_layer_support/remote_support.h | 4 +- + .../modules/image/oci/storage/storage.c | 2 +- + .../modules/image/oci/storage/storage.h | 2 + + 5 files changed, 143 insertions(+), 4 deletions(-) + +diff --git a/src/daemon/modules/image/oci/oci_image.c b/src/daemon/modules/image/oci/oci_image.c +index fa92a861..40e9a88f 100644 +--- a/src/daemon/modules/image/oci/oci_image.c ++++ b/src/daemon/modules/image/oci/oci_image.c +@@ -44,6 +44,39 @@ + + struct oci_image_module_data g_oci_image_module_data = { 0 }; + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++// intend to make remote refresh and oci ops exlusive ++static bool g_enable_remote; ++static pthread_rwlock_t g_remote_lock = PTHREAD_RWLOCK_INITIALIZER; ++ ++static inline bool oci_remote_lock(pthread_rwlock_t *remote_lock, bool writable) ++{ ++ int nret = 0; ++ ++ if (writable) { ++ nret = pthread_rwlock_wrlock(remote_lock); ++ } else { ++ nret = pthread_rwlock_rdlock(remote_lock); ++ } ++ if (nret != 0) { ++ ERROR("Lock memory store failed: %s", strerror(nret)); ++ return false; ++ } ++ ++ return true; ++} ++ ++static inline void oci_remote_unlock(pthread_rwlock_t *remote_lock) ++{ ++ int nret = 0; ++ ++ nret = pthread_rwlock_unlock(remote_lock); ++ if (nret != 0) { ++ FATAL("Unlock memory store failed: %s", strerror(nret)); ++ } ++} ++#endif ++ + static void free_oci_image_data(void) + { + free(g_oci_image_module_data.root_dir); +@@ -220,6 +253,7 @@ static int storage_module_init_helper(const isulad_daemon_configs *args) + + #ifdef ENABLE_REMOTE_LAYER_STORE + storage_opts->enable_remote_layer = args->storage_enable_remote_layer; ++ storage_opts->remote_lock = &g_remote_lock; + #endif + + if (util_dup_array_of_strings((const char **)args->storage_opts, args->storage_opts_len, &storage_opts->driver_opts, +@@ -303,6 +337,10 @@ int oci_init(const isulad_daemon_configs *args) + goto out; + } + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ g_enable_remote = args->storage_enable_remote_layer; ++#endif ++ + if (storage_module_init_helper(args) != 0) { + ret = -1; + goto out; +@@ -321,6 +359,7 @@ void oci_exit() + + int oci_pull_rf(const im_pull_request *request, im_pull_response *response) + { ++ int ret = 0; + if (request == NULL || request->image == NULL || response == NULL) { + ERROR("Invalid NULL param"); + return -1; +@@ -331,8 +370,24 @@ int oci_pull_rf(const im_pull_request *request, im_pull_response *response) + isulad_try_set_error_message("Invalid image name: %s", request->image); + return -1; + } ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ // read lock here because pull have exclusive access against remote refresh ++ // pull can work concurrently with other oci operations. ++ if (g_enable_remote && !oci_remote_lock(&g_remote_lock, false)) { ++ ERROR("Failed to lock oci remote lock when load image"); ++ return -1; ++ } ++#endif ++ ++ ret = oci_do_pull_image(request, response); ++ ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ if (g_enable_remote) { ++ oci_remote_unlock(&g_remote_lock); ++ } ++#endif + +- return oci_do_pull_image(request, response); ++ return ret; + } + + int oci_prepare_rf(const im_prepare_request *request, char **real_rootfs) +@@ -441,6 +496,15 @@ int oci_rmi(const im_rmi_request *request) + return -1; + } + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ // read lock here because load have exclusive access against remote refresh ++ // load can work concurrently with other oci operations. ++ if (g_enable_remote && !oci_remote_lock(&g_remote_lock, false)) { ++ ERROR("Failed to lock oci remote lock when load image"); ++ return -1; ++ } ++#endif ++ + if (!util_valid_image_name(request->image.image)) { + ERROR("Invalid image name: %s", request->image.image); + isulad_try_set_error_message("Invalid image name: %s", request->image.image); +@@ -502,6 +566,11 @@ int oci_rmi(const im_rmi_request *request) + } + + out: ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ if (g_enable_remote) { ++ oci_remote_unlock(&g_remote_lock); ++ } ++#endif + free(real_image_name); + free(image_ID); + util_free_array_by_len(image_names, image_names_len); +@@ -527,7 +596,24 @@ int oci_import(const im_import_request *request, char **id) + goto err_out; + } + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ // read lock here because import have exclusive access against remote refresh ++ // import can work concurrently with other oci operations. ++ if (g_enable_remote && !oci_remote_lock(&g_remote_lock, false)) { ++ ERROR("Failed to lock oci remote lock when load image"); ++ ret = -1; ++ goto err_out; ++ } ++#endif ++ + ret = oci_do_import(request->file, dest_name, id); ++ ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ if (g_enable_remote) { ++ oci_remote_unlock(&g_remote_lock); ++ } ++#endif ++ + if (ret != 0) { + goto err_out; + } +@@ -677,7 +763,24 @@ int oci_load_image(const im_load_request *request) + goto out; + } + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ // read lock here because load have exclusive access against remote refresh ++ // load can work concurrently with other oci operations. ++ if (g_enable_remote && !oci_remote_lock(&g_remote_lock, false)) { ++ ERROR("Failed to lock oci remote lock when load image"); ++ ret = -1; ++ goto out; ++ } ++#endif ++ + ret = oci_do_load(request); ++ ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ if (g_enable_remote) { ++ oci_remote_unlock(&g_remote_lock); ++ } ++#endif ++ + if (ret != 0) { + ERROR("Failed to load image"); + goto out; +diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c +index 3c7d0f54..7d457755 100644 +--- a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c ++++ b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c +@@ -24,10 +24,38 @@ struct supporters { + struct remote_image_data *image_data; + struct remote_layer_data *layer_data; + struct remote_overlay_data *overlay_data; ++ pthread_rwlock_t *remote_lock; + }; + + static struct supporters supporters; + ++static inline bool remote_refresh_lock(pthread_rwlock_t *remote_lock, bool writable) ++{ ++ int nret = 0; ++ ++ if (writable) { ++ nret = pthread_rwlock_wrlock(remote_lock); ++ } else { ++ nret = pthread_rwlock_rdlock(remote_lock); ++ } ++ if (nret != 0) { ++ ERROR("Lock memory store failed: %s", strerror(nret)); ++ return false; ++ } ++ ++ return true; ++} ++ ++static inline void remote_refresh_unlock(pthread_rwlock_t *remote_lock) ++{ ++ int nret = 0; ++ ++ nret = pthread_rwlock_unlock(remote_lock); ++ if (nret != 0) { ++ FATAL("Unlock memory store failed: %s", strerror(nret)); ++ } ++} ++ + static void *remote_refresh_ro_symbol_link(void *arg) + { + struct supporters *refresh_supporters = (struct supporters *)arg; +@@ -37,16 +65,18 @@ static void *remote_refresh_ro_symbol_link(void *arg) + util_usleep_nointerupt(5 * 1000 * 1000); + DEBUG("remote refresh start\n"); + ++ remote_refresh_lock(supporters.remote_lock, true); + remote_overlay_refresh(refresh_supporters->overlay_data); + remote_layer_refresh(refresh_supporters->layer_data); + remote_image_refresh(refresh_supporters->image_data); ++ remote_refresh_unlock(supporters.remote_lock); + + DEBUG("remote refresh end\n"); + } + return NULL; + } + +-int remote_start_refresh_thread(void) ++int remote_start_refresh_thread(pthread_rwlock_t *remote_lock) + { + int res = 0; + pthread_t a_thread; +@@ -67,6 +97,8 @@ int remote_start_refresh_thread(void) + goto free_out; + } + ++ supporters.remote_lock = remote_lock; ++ + res = pthread_create(&a_thread, NULL, remote_refresh_ro_symbol_link, (void *)&supporters); + if (res != 0) { + CRIT("Thread creation failed"); +diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h +index 892a9155..30e3ebb0 100644 +--- a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h ++++ b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h +@@ -16,6 +16,8 @@ + #ifndef DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_REMOTE_SUPPORT_H + #define DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_REMOTE_SUPPORT_H + ++#include <pthread.h> ++ + #include "linked_list.h" + #include "map.h" + #include "ro_symlink_maintain.h" +@@ -64,7 +66,7 @@ void remote_overlay_refresh(struct remote_overlay_data *data); + bool remote_overlay_layer_valid(const char *layer_id); + + // start refresh remote +-int remote_start_refresh_thread(void); ++int remote_start_refresh_thread(pthread_rwlock_t *remote_lock); + + // extra map utils + char **remote_deleted_layers(const map_t *old, const map_t *new_l); +diff --git a/src/daemon/modules/image/oci/storage/storage.c b/src/daemon/modules/image/oci/storage/storage.c +index f9830ac3..836ccf4d 100644 +--- a/src/daemon/modules/image/oci/storage/storage.c ++++ b/src/daemon/modules/image/oci/storage/storage.c +@@ -1874,7 +1874,7 @@ int storage_module_init(struct storage_module_init_options *opts) + } + + #ifdef ENABLE_REMOTE_LAYER_STORE +- if (opts->enable_remote_layer && remote_start_refresh_thread() != 0) { ++ if (opts->enable_remote_layer && remote_start_refresh_thread(opts->remote_lock) != 0) { + ERROR("Failed to start remote refresh thread"); + } + #endif +diff --git a/src/daemon/modules/image/oci/storage/storage.h b/src/daemon/modules/image/oci/storage/storage.h +index 7404ee54..df9fd761 100644 +--- a/src/daemon/modules/image/oci/storage/storage.h ++++ b/src/daemon/modules/image/oci/storage/storage.h +@@ -18,6 +18,7 @@ + #include <stdint.h> + #include <stdbool.h> + #include <stddef.h> ++#include <pthread.h> + #include <isula_libutils/imagetool_image.h> + #include <isula_libutils/json_common.h> + +@@ -72,6 +73,7 @@ struct storage_module_init_options { + bool integration_check; + #ifdef ENABLE_REMOTE_LAYER_STORE + bool enable_remote_layer; ++ pthread_rwlock_t *remote_lock; + #endif + }; + +-- +2.25.1 + diff --git a/iSulad.spec b/iSulad.spec new file mode 100644 index 0000000..8779b85 --- /dev/null +++ b/iSulad.spec @@ -0,0 +1,877 @@ +%global _version 2.1.1 +%global _release 6 +%global is_systemd 1 +%global enable_shimv2 1 +%global is_embedded 1 + +Name: iSulad +Version: %{_version} +Release: %{_release} +Summary: Lightweight Container Runtime Daemon +License: Mulan PSL v2 +URL: https://gitee.com/openeuler/iSulad +Source: https://gitee.com/openeuler/iSulad/repository/archive/v%{version}.tar.gz +BuildRoot: {_tmppath}/iSulad-%{version} + +Patch0001: 0001-modify-dependence-from-lcr-to-libisula.patch +Patch0002: 0002-Add-unified-memory_swap_limit_in_bytes-fields-into-C.patch +Patch0003: 0003-Add-macro-for-protoc-cmake.patch +Patch0004: 0004-fix-design-typo.patch +Patch0005: 0005-fix-cpu-rt-review-comments.patch +Patch0006: 0006-fix-inspect.sh-failed.patch +Patch0007: 0007-add-CRI-ContainerStats-Service.patch +Patch0008: 0008-fix-isula-cpu-rt-CI.patch +Patch0009: 0009-fix-cpu-rt-CI.patch +Patch0010: 0010-fix-cpu-rt-CI.patch +Patch0011: 0011-Bugfix-in-config-and-executor.patch +Patch0012: 0012-fix-cpu-rt-disable-after-reboot-machine.patch +Patch0013: 0013-fix-selinux_label_ut-timeout-and-add-timeout-for-all.patch +Patch0014: 0014-add-retry-for-read-write.patch +Patch0015: 0015-support-pull-image-with-digest.patch +Patch0016: 0016-isulad-shim-support-execSync-with-timeout.patch +Patch0017: 0017-Refine-the-commit-info.patch +Patch0018: 0018-Refine-typo-of-word-container.patch +Patch0019: 0019-cleancode-for-read-write.patch +Patch0020: 0020-add-crictl-timeout-and-sync-for-CI.patch +Patch0021: 0021-unlock-m_podsLock-if-new-failed.patch +Patch0022: 0022-Update-CRI.patch +Patch0023: 0023-add-cgroup-cpu-ut.patch +Patch0024: 0024-remove-temp-variables.patch +Patch0025: 0025-fix-read-member-error-from-struct.patch +Patch0026: 0026-Fix-PR-runc.patch +Patch0027: 0027-allow-the-paused-container-to-be-stopped.patch +Patch0028: 0028-Refine.patch +Patch0029: 0029-support-isula-update-when-runtime-is-runc.patch +Patch0030: 0030-Refine-as-others-feedback.patch +Patch0031: 0031-fix-CRI-SetupPod-and-TearDownPod-deadlock.patch +Patch0032: 0032-remote-layer-store-demo.patch +Patch0033: 0033-add-ci-for-remote-ro.patch +Patch0034: 0034-change-sleep-to-usleep-to-avoid-lossing-of-accuracy.patch +Patch0035: 0035-fix-compile-error-when-not-enable-remote-ro.patch +Patch0036: 0036-adapt-to-repo-of-openeuler-url-changed.patch +Patch0037: 0037-change-goto-branch.patch +Patch0038: 0038-CI-not-enable-remote-ro-for-ut.patch +Patch0039: 0039-use-auto-free-to-proc_t.patch +Patch0040: 0040-modifying-cpurt-file-permissions.patch +Patch0041: 0041-use-CURLOPT_XFERINFOFUNCTION-instead-of-deprecated-C.patch +Patch0042: 0042-bugfix-remote-ro-try-add-or-remove-image-layer-twice.patch +Patch0043: 0043-bugfix-can-t-delete-layers-under-dir-overlay-layers.patch +Patch0044: 0044-doc-add-document-about-support-remote-ro-directory.patch +Patch0045: 0045-Refine-a-minor-log-message.patch +Patch0046: 0046-modify-the-return-value-of-the-util_waitpid_with_tim.patch +Patch0047: 0047-fix-util_getgrent_r-overflow.patch +Patch0048: 0048-add-ut-for-runc.patch +Patch0049: 0049-add-runc-doc.patch +Patch0050: 0050-fix-isula_rt_ops_ut-bugs.patch +Patch0051: 0051-refactor-remote-ro-code.patch +Patch0052: 0052-add-ci-for-runc.patch +Patch0053: 0053-bugfix-when-refresh-can-t-load-or-pull-images.patch + +%ifarch x86_64 aarch64 +Provides: libhttpclient.so()(64bit) +Provides: libisula_client.so()(64bit) +Provides: libisulad_img.so()(64bit) +Provides: libisulad_tools.so()(64bit) +%endif + +%if 0%{?is_systemd} +# Systemd 230 and up no longer have libsystemd-journal +BuildRequires: pkgconfig(systemd) +Requires: systemd-units +%else +Requires(post): chkconfig +Requires(preun): chkconfig +# This is for /sbin/service +Requires(preun): initscripts +%endif + +%if 0%{?is_embedded} +BuildRequires: sqlite-devel +Requires: sqlite +%endif + +%if %{defined openeuler} +BuildRequires: gtest-devel gmock-devel +%endif + +%define lcrver_lower 2.1.1-0 +%define lcrver_upper 2.1.2-0 + +BuildRequires: lcr-devel > %{lcrver_lower} lcr-devel < %{lcrver_upper} +BuildRequires: libisula-devel > %{lcrver_lower} libisula-devel < %{lcrver_upper} +BuildRequires: cmake gcc-c++ yajl-devel lxc lxc-devel +BuildRequires: grpc grpc-plugins grpc-devel protobuf-devel +BuildRequires: libcurl libcurl-devel libarchive-devel device-mapper-devel +BuildRequires: http-parser-devel +BuildRequires: libseccomp-devel libcap-devel libselinux-devel libwebsockets libwebsockets-devel +BuildRequires: systemd-devel git +%if 0%{?enable_shimv2} +BuildRequires: lib-shim-v2 lib-shim-v2-devel +%endif + + +Requires: libisula > %{lcrver_lower} libisula < %{lcrver_upper} +Requires: lcr > %{lcrver_lower} lcr < %{lcrver_upper} +Requires: grpc protobuf lxc +Requires: libcurl +Requires: http-parser libseccomp +Requires: libcap libselinux libwebsockets libarchive device-mapper +Requires: systemd +%if 0%{?enable_shimv2} +Requires: lib-shim-v2 +%endif + +%description +This is a umbrella project for gRPC-services based Lightweight Container +Runtime Daemon, written by C. + +%prep +%autosetup -n iSulad-v%{_version} -Sgit -p1 + +%build +mkdir -p build +cd build +%if 0%{?enable_shimv2} +%if %{defined openeuler} +%cmake -DDEBUG=ON -DCMAKE_SKIP_RPATH=TRUE -DLIB_INSTALL_DIR=%{_libdir} -DCMAKE_INSTALL_PREFIX=/usr -DENABLE_SHIM_V2=ON -DENABLE_UT=ON ../ +%else +%cmake -DDEBUG=ON -DCMAKE_SKIP_RPATH=TRUE -DLIB_INSTALL_DIR=%{_libdir} -DCMAKE_INSTALL_PREFIX=/usr -DENABLE_SHIM_V2=ON ../ +%endif +%else +%if %{defined openeuler} +%cmake -DDEBUG=ON -DCMAKE_SKIP_RPATH=TRUE -DLIB_INSTALL_DIR=%{_libdir} -DCMAKE_INSTALL_PREFIX=/usr -DENABLE_UT=ON ../ +%else +%cmake -DDEBUG=ON -DCMAKE_SKIP_RPATH=TRUE -DLIB_INSTALL_DIR=%{_libdir} -DCMAKE_INSTALL_PREFIX=/usr ../ +%endif +%endif +%make_build + +%check +%if %{defined openeuler} +cd build +# registry_images_ut and volume_ut must run with root user +ctest -E "registry_images_ut|volume_ut" +%endif + +%install +rm -rf %{buildroot} +cd build +install -d $RPM_BUILD_ROOT/%{_libdir} +install -m 0755 ./src/libisula_client.so %{buildroot}/%{_libdir}/libisula_client.so +install -m 0755 ./src/utils/http/libhttpclient.so %{buildroot}/%{_libdir}/libhttpclient.so +install -m 0755 ./src/libisulad_tools.so %{buildroot}/%{_libdir}/libisulad_tools.so +install -m 0755 ./src/daemon/modules/image/libisulad_img.so %{buildroot}/%{_libdir}/libisulad_img.so + +install -d $RPM_BUILD_ROOT/%{_libdir}/pkgconfig +install -m 0640 ./conf/isulad.pc %{buildroot}/%{_libdir}/pkgconfig/isulad.pc + +install -d $RPM_BUILD_ROOT/%{_bindir} + +install -m 0755 ./src/isula %{buildroot}/%{_bindir}/isula +install -m 0755 ./src/isulad-shim %{buildroot}/%{_bindir}/isulad-shim + +install -m 0755 ./src/isulad %{buildroot}/%{_bindir}/isulad + +install -d $RPM_BUILD_ROOT/%{_includedir}/isulad +install -m 0644 ../src/daemon/modules/api/image_api.h %{buildroot}/%{_includedir}/isulad/image_api.h + +install -d $RPM_BUILD_ROOT/%{_sysconfdir}/isulad +install -m 0640 ../src/contrib/config/daemon.json %{buildroot}/%{_sysconfdir}/isulad/daemon.json +install -m 0440 ../src/contrib/config/daemon_constants.json %{buildroot}/%{_sysconfdir}/isulad/daemon_constants.json +install -m 0640 ../src/contrib/config/seccomp_default.json %{buildroot}/%{_sysconfdir}/isulad/seccomp_default.json + +install -d $RPM_BUILD_ROOT/%{_sysconfdir}/default/isulad +install -m 0640 ../src/contrib/config/config.json %{buildroot}/%{_sysconfdir}/default/isulad/config.json +install -m 0640 ../src/contrib/config/systemcontainer_config.json %{buildroot}/%{_sysconfdir}/default/isulad/systemcontainer_config.json +install -m 0550 ../src/contrib/sysmonitor/isulad-check.sh %{buildroot}/%{_sysconfdir}/default/isulad/isulad-check.sh + +mkdir -p $RPM_BUILD_ROOT/%{_sysconfdir}/sysmonitor/process +cp ../src/contrib/sysmonitor/isulad-monit $RPM_BUILD_ROOT/etc/sysmonitor/process + +install -d $RPM_BUILD_ROOT/%{_sysconfdir}/default/isulad/hooks +install -m 0640 ../src/contrib/config/hooks/default.json %{buildroot}/%{_sysconfdir}/default/isulad/hooks/default.json + +install -d $RPM_BUILD_ROOT/%{_sysconfdir}/sysconfig +install -p -m 0640 ../src/contrib/config/iSulad.sysconfig $RPM_BUILD_ROOT/%{_sysconfdir}/sysconfig/iSulad + +%if 0%{?is_systemd} +install -d $RPM_BUILD_ROOT/%{_unitdir} +install -p -m 0640 ../src/contrib/init/isulad.service $RPM_BUILD_ROOT/%{_unitdir}/isulad.service +%else +install -d $RPM_BUILD_ROOT/%{_initddir} +install -p -m 0640 ../src/contrib/init/isulad.init $RPM_BUILD_ROOT/%{_initddir}/isulad.init +%endif + +%clean +rm -rf %{buildroot} + +%pre +# support update from lcrd to isulad, will remove in next version +if [ "$1" = "2" ]; then +%if 0%{?is_systemd} +systemctl stop lcrd &>/dev/null +systemctl disable lcrd &>/dev/null +if [ -e %{_sysconfdir}/isulad/daemon.json ];then + sed -i 's#/etc/default/lcrd/hooks#/etc/default/isulad/hooks#g' %{_sysconfdir}/isulad/daemon.json +fi +%else +/sbin/chkconfig --del lcrd &>/dev/null +%endif +fi + +%post +if ! getent group isula > /dev/null; then + groupadd --system isula +fi + +if [ "$1" = "1" ]; then +%if 0%{?is_systemd} +systemctl enable isulad +systemctl start isulad +%else +/sbin/chkconfig --add isulad +%endif +elif [ "$1" = "2" ]; then +%if 0%{?is_systemd} +# support update from lcrd to isulad, will remove in next version +if [ -e %{_unitdir}/lcrd.service.rpmsave ]; then + mv %{_unitdir}/lcrd.service.rpmsave %{_unitdir}/isulad.service + sed -i 's/lcrd/isulad/g' %{_unitdir}/isulad.service +fi +systemctl status isulad | grep 'Active:' | grep 'running' +if [ $? -eq 0 ]; then + systemctl restart isulad +else + systemctl start isulad +fi +%else +/sbin/service isulad status | grep 'Active:' | grep 'running' +if [ $? -eq 0 ]; then + /sbin/service isulad restart +fi +%endif +fi + +if ! getent group isula > /dev/null; then + groupadd --system isula +fi + +%preun +%if 0%{?is_systemd} +%systemd_preun isulad +%else +if [ $1 -eq 0 ] ; then + /sbin/service isulad stop >/dev/null 2>&1 + /sbin/chkconfig --del isulad +fi +%endif + +%postun +%if 0%{?is_systemd} +%systemd_postun_with_restart isulad +%else +if [ "$1" -ge "1" ] ; then + /sbin/service isulad condrestart >/dev/null 2>&1 || : +fi +%endif + +%files +%attr(0600,root,root) %{_sysconfdir}/sysmonitor/process/isulad-monit +%attr(0550,root,root) %{_sysconfdir}/default/isulad/isulad-check.sh +%defattr(0640,root,root,0750) +%{_sysconfdir}/isulad +%{_sysconfdir}/isulad/* +%{_sysconfdir}/default/* +%defattr(-,root,root,-) +%if 0%{?is_systemd} +%{_unitdir}/isulad.service +%attr(0640,root,root) %{_unitdir}/isulad.service +%else +%{_initddir}/isulad.init +%attr(0640,root,root) %{_initddir}/isulad.init +%endif +%{_includedir}/isulad/* +%attr(0755,root,root) %{_libdir}/pkgconfig +%attr(0640,root,root) %{_libdir}/pkgconfig/isulad.pc +%defattr(0755,root,root,0755) +%{_bindir}/* +%{_libdir}/* +%attr(0640,root,root) %{_sysconfdir}/sysconfig/iSulad +%attr(0640,root,root) %{_sysconfdir}/isulad/daemon.json + +%config(noreplace,missingok) %{_sysconfdir}/sysconfig/iSulad +%config(noreplace,missingok) %{_sysconfdir}/isulad/daemon.json +%if 0%{?is_systemd} +%config(noreplace,missingok) %{_unitdir}/isulad.service +%else +%config(noreplace,missingok) %{_initddir}/isulad.init +%endif + +%changelog +* Fri Mar 24 2023 wangrunze <wangrunze13@huawei.com> - 2.1.1-6 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: update from upstream to include remote feature + +* Thu Mar 16 2023 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.1.1-5 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: use CURLOPT_XFERINFOFUNCTION instead of deprecated CURLOPT_PROGRESSFUNCTION since curl 7.32.0 + +* Wed Feb 22 2023 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.1.1-4 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: upgrade from upstream + +* Thu Feb 16 2023 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.1.1-3 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: add check + +* Mon Feb 06 2023 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.1.1-2 +- Type: update +- ID: NA +- SUG: NA +- DESC: modify dependence from lcr to libisula + +* Mon Feb 06 2023 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.1.1-1 +- Type: update +- ID: NA +- SUG: NA +- DESC: update to v2.1.1 + +* Tue Jan 03 2023 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.18-1 +- Type: update +- ID: NA +- SUG: NA +- DESC: update to v2.0.18 + +* Thu Dec 22 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.17-14 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: remove clean_module_fill_ctx for libisulad_img.so + +* Mon Dec 19 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.17-13 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: update lcr dependence version + +* Fri Dec 16 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.17-12 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: update from upstream + +* Tue Dec 06 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.17-11 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: update from upstream + +* Mon Nov 28 2022 yangjiaqi <yangjiaqi16@huawei.com> - 2.0.17-10 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: remove chmod 751 permission for dirs by engine when user-remap enabled + +* Fri Nov 25 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.17-9 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: add primary group to additional groups + +* Mon Nov 21 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.17-8 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fix cannot install isulad and unknown option + +* Wed Nov 16 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.17-7 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: update tar package from source + +* Wed Nov 02 2022 wangrunze <wangrunze13@huawei.com> - 2.0.17-6 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fix cleanup module memory leak + +* Tue Nov 01 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.17-5 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: add ut and bugfix for device mapper and websocket + +* Mon Oct 31 2022 wujing <wujing50@huawei.com> - 2.0.17-4 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sync from openEuler + +* Wed Oct 19 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.17-3 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: add required package lcr clibcni lower and upper version + +* Mon Oct 10 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.17-2 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: use epoll instead of select for wait_exit_fifo + +* Sun Oct 09 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.17-1 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: update iSulad version to 2.0.17-1 + +* Thu Sep 29 2022 haozi007 <liuhao27@huawei.com> - 2.0.16-8 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sync from openEuler + +* Tue Sep 20 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.16-7 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: change libisulad_tools.so mode + +* Thu Sep 15 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.16-6 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: shield upgrade error if lcrd not exist + +* Tue Sep 13 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.16-5 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: calculate timezone by tm_gmtoff + +* Thu Sep 08 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.16-4 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: invalid free default-runtime and cri-runtime after free json-confs + +* Wed Sep 07 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.16-3 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sycn patches from openeuler/iSulad + +* Tue Aug 30 2022 leizhongkai <leizhongkai@huawei.com> - 2.0.16-2 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sync from upstream openeuler/iSulad + +* Tue Aug 23 2022 wangfengtu <wangfengtu@huawei.com> - 2.0.16-1 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: update iSulad version to 2.0.16-1 + +* Mon Aug 22 2022 zhongtao <zhongtao17@huawei.com> - 2.0.15-6 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: remove rpath by cmake + +* Wed Aug 17 2022 haozi007 <liuhao27@huawei.com> - 2.0.15-5 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sycn patches from openeuler + +* Mon Aug 15 2022 wangfengtu <wangfengtu@huawei.com> - 2.0.15-4 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: change default umask to 0022 + +* Tue Aug 9 2022 haozi007 <liuhao27@huawei.com> - 2.0.15-3 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sycn patches from openeuler + +* Mon Aug 1 2022 chengzeruizhi <chengzeruizhi@huawei.com> - 2.0.15-2 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sycn patches from openeuler branch + +* Fri Jul 8 2022 haozi007 <liuhao27@huawei.com> - 2.0.15-1 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: update version to v2.0.15 + +* Fri Jul 8 2022 haozi007 <liuhao27@huawei.com> - 2.0.14-11 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: add limit to lcr version + +* Wed Jun 22 2022 yangjiaqi <yangjiaqi16@huawei.com> - 2.0.14-10 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sync from upstream openeuler/iSulad + +* Tue Jun 21 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.14-9 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sync from upstream openeuler/iSulad + +* Wed Jun 15 2022 chengzeruizhi <chengzeruizhi@huawei.com> - 2.0.14-8 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sync from upstream openeuler/iSulad + +* Tue May 31 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.14-7 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: fix type convert, add null pointer check, remove unuse macro + +* Tue May 31 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.14-6 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: fix different type convert and add check to arguments + +* Mon May 30 2022 chengzrz <chengzeruizhi@huawei.com> - 2.0.14-5 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: seccomp optimization + +* Fri May 27 2022 haozi007 <liuhao27@huawei.com> - 2.0.14-4 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: improve fuzz test for pw and gr parser + +* Tue May 24 2022 wangfengtu <wangfengtu@huawei.com> - 2.0.14-3 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: fix install error when android + +* Tue May 24 2022 wangfengtu <wangfengtu@huawei.com> - 2.0.14-2 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: do not mkdir of isulad if no controller found + +* Mon May 23 2022 haozi007 <liuhao27@huawei.com> - 2.0.14-1 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: update version to v2.0.14 + +* Mon May 16 2022 haozi007<liuhao27@huawei.com> - 2.0.13-5 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sync from upstream openEuler/iSulad + +* Tue May 10 2022 hejunjie<hejunjie10@huawei.com> - 2.0.13-4 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: bionic adaptation, increase lcov coverage + +* Thu May 5 2022 hejunjie<hejunjie10@huawei.com> - 2.0.13-3 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: bionic adaptation for pwgr obj parser + +* Mon Apr 25 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.13-2 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: sync from upstream + +* Mon Apr 18 2022 wangfengtu <wangfengtu@huawei.com> - 2.0.13-1 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: update version to v2.0.13 + +* Fri Mar 25 2022 wangfengtu <wangfengtu@huawei.com> - 2.0.12-1 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: update version to v2.0.12 + +* Thu Mar 17 2022 haozi007 <liuhao27@huawei.com> - 2.0.11-6 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: remove unnecessary error message + +* Thu Mar 17 2022 wangfengtu <wangfengtu@huawei.com> - 2.0.11-5 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fix uid/gid error when load image + +* Wed Mar 09 2022 wangfengtu <wangfengtu@huawei.com> - 2.0.11-4 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: add provides of libisulad_tools.so + +* Thu Mar 03 2022 wangfengtu <wangfengtu@huawei.com> - 2.0.11-3 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: Add the function of isolating the user namespaces + +* Thu Mar 03 2022 wangfengtu <wangfengtu@huawei.com> - 2.0.11-2 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: add libisulad_tools.so + +* Thu Feb 24 2022 wangfengtu <wangfengtu@huawei.com> - 2.0.11-1 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: update version to v2.0.11 + +* Wed Jan 12 2022 wangfengtu <wangfengtu@huawei.com> - 2.0.10-15 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fix compile error of isula-transform + +* Wed Jan 12 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.10-14 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fix compile error with grpc 1.41.x + +* Tue Jan 4 2022 wangfengtu <wangfengtu@huawei.com> - 2.0.10-13 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fix compile error when building embedded image + +* Mon Dec 27 2021 wangfengtu <wangfengtu@huawei.com> - 2.0.10-12 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: sync patches from upstream + +* Thu Dec 09 2021 chengzeruizhi <chengzeruizhi@huawei.com> - 2.0.10-11 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fixed a bug that occurs when starting a container in host mode + +* Thu Dec 09 2021 wangfengtu <wagnfengtu@huawei.com> - 2.0.10-10 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: remove dependance of sqlite + +* Mon Dec 06 2021 gaohuatao <gaohuatao@huawei.com> - 2.0.10-9 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: specify version + +* Fri Dec 03 2021 wangfengtu <wangfengtu@huawei.com> - 2.0.10-8 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fix undefined reference to `service_arguments_free' in libisulad_img.so + +* Thu Dec 02 2021 wangfengtu <wangfengtu@huawei.com> - 2.0.10-7 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: sync patches from upstream + +* Tue Nov 23 2021 chengzeruizhi <chengzeruizhi@huawei.com> - 2.0.10-6 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: modify the procedure of running a pod + +* Fri Nov 19 2021 gaohuatao <gaohuatao@huawei.com> - 2.0.10-5 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sync from upstream + +* Fri Nov 19 2021 wangfengtu <wangfengtu@huawei.com> - 2.0.10-4 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fix memleak when use multiple --volumes-from + +* Tue Nov 16 2021 wujing <wujing50@huawei.com> - 2.0.10-3 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: add shimv2 build switch + +* Tue Nov 16 2021 wujing <wujing50@huawei.com> - 2.0.10-2 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: remove build platform restrictions + +* Tue Nov 09 2021 gaohuatao <gaohuatao@huawei.com> - 2.0.10-1 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: update from openeuler + +* Tue Oct 19 2021 wangfengtu <wangfengtu@huawei.com> - 2.0.9-20211019.121837.gitf067b3ce +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: strip sha256 prefix when decrease hold references + +* Fri Jun 25 2021 wujing <wujing50@huawei.com> - 2.0.9-20210625.165022.git5a088d9c +- Type: update to v2.0.9 +- ID: NA +- SUG: NA +- DESC: update from master + +* Tue May 18 2021 wangfengtu <wangfengtu@huawei.com> - 2.0.8-20210518.144540.git5288ed92 +- Type: sync from upstream +- ID: NA +- SUG: NA +- DESC: update from master + +* Fri Mar 26 2021 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.8-20210326.094027.gitac974aa6 +- Type: sync from upstream +- ID: NA +- SUG: NA +- DESC: update from master + +* Tue Mar 23 2021 haozi007 <liuhao27@huawei.com> - 20210323.094917.git7e6aa593 +- Type: sync from upstream +- ID: NA +- SUG: NA +- DESC: update from master + +* Tue Feb 2 2021 lifeng <lifeng68@huawei.com> - 2.0.8-20210202.153251.gite082dcf3 +- Type: sync from upstream +- ID: NA +- SUG: NA +- DESC: update from master + +* Mon Jan 18 2021 lifeng <lifeng68@huawei.com> - 2.0.8-20210118.195254.git077e10f2 +- Type: sync from upstream +- ID: NA +- SUG: NA +- DESC: update from master + +* Wed Dec 30 2020 lifeng <lifeng68@huawei.com> - 2.0.8-20201230.155843.git6557a6eb +- Type: update to v2.0.8 +- ID: NA +- SUG: NA +- DESC: update from master + +* Mon Dec 7 2020 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.7-20201207.151847.gita1fce123 +- Type: update +- ID: NA +- SUG: NA +- DESC: update from master + +* Sat Dec 5 2020 lifeng <lifeng68@huawei.com> - 2.0.7-20201205.145752.gita461cc51 +- Type: bugfix +- ID:NA +- SUG:NA +- DESC: ignore list containers errors + +* Thu Dec 3 2020 haozi007 <liuhao27@huawei.com> - 2.0.7-20201203.190902.git48f598fd +- Type:update from master +- ID:NA +- SUG:NA +- DESC: update from master + +* Sat Nov 28 2020 lifeng<lifeng68@huawei.com> - 2.0.7-20201128.095506.git1e1623a5 +- Type: bugfix +- ID:NA +- SUG:NA +- DESC: Mounts: only qsort the configed mounts and make possible to bind mount /proc and /sys/fs. +- related lxc PR fixed: +- 1.add check whether have /proc mounts entry, if has, skip the auto +- 2.mount cgroup before do mount entrys +- 3.pass if the mount on top of /proc and the source of the mount is a proc filesystem + +* Wed Nov 25 2020 wangfengtu<wangfengtu@huawei.com> - 2.0.7-20201125.165149.git7d150c3c +- Type: bugfix +- ID:NA +- SUG:NA +- DESC: update from openeuler + +* Wed Nov 25 2020 wangfengtu<wangfengtu@huawei.com> - 2.0.6-20201125.160534.git9fb5e75d +- Type: bugfix +- ID:NA +- SUG:NA +- DESC: fix rpath not work + +* Thu Nov 12 2020 gaohuatao<gaohuatao@huawei.com> - 2.0.6-20201112.193005.git8a6b73c8 +- Type: update from openeuler +- ID:NA +- SUG:NA +- DESC: update from openeuler + +* Wed Oct 14 2020 lifeng68<lifeng68@huawei.com> - 2.0.6-20201014.152749.gitc8a43925 +- Type: upgrade to v2.0.6 +- ID:NA +- SUG:NA +- DESC: upgrade to v2.0.6 + +* Fri Sep 18 2020 <lifeng68@huawei.com> - 2.0.5-20200918.112827.git9aea9b75 +- Type:bugfix +- ID:NA +- SUG:NA +- DESC: modify log level to warn + +* Mon Sep 14 2020 <lifeng68@huawei.com> - 2.0.5-20200914.172527.gitae86920a +- Type:bugfix +- ID:NA +- SUG:NA +- DESC: remove unused config + +* Thu Sep 10 2020 <yangjiaqi11@huawei.com> - 2.0.5-20200910.144345.git71b1055b +- Type:enhancement +- ID:NA +- SUG:NA +- DESC: add chrpath + +* Fri Sep 04 2020 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.5-20200904.114315.gitff1761c3 +- Type:enhancement +- ID:NA +- SUG:NA +- DESC: upgrade from v2.0.3 to v2.0.5 + +* Wed Sep 02 2020 YoungJQ <yangjiaqi11@huawei.com> - 2.0.3-20200902.114727.git6d945f26 +- Type:enhancement +- ID:NA +- SUG:NA +- DESC: modify source0 address @@ -0,0 +1 @@ +667c9f751daaac3b991659c27722724d v2.1.1.tar.gz |