From 5de4bc645d087edfbc848d43ac0019a6535a3c73 Mon Sep 17 00:00:00 2001 From: CoprDistGit Date: Mon, 17 Apr 2023 07:36:31 +0000 Subject: automatic import of iSulad --- .gitignore | 1 + 0001-modify-dependence-from-lcr-to-libisula.patch | 206 ++ ...-memory_swap_limit_in_bytes-fields-into-C.patch | 243 +++ 0003-Add-macro-for-protoc-cmake.patch | 135 ++ 0004-fix-design-typo.patch | 25 + 0005-fix-cpu-rt-review-comments.patch | 44 + 0006-fix-inspect.sh-failed.patch | 31 + 0007-add-CRI-ContainerStats-Service.patch | 490 +++++ 0008-fix-isula-cpu-rt-CI.patch | 183 ++ 0009-fix-cpu-rt-CI.patch | 105 + 0010-fix-cpu-rt-CI.patch | 54 + 0011-Bugfix-in-config-and-executor.patch | 52 + 0012-fix-cpu-rt-disable-after-reboot-machine.patch | 823 +++++++ ..._label_ut-timeout-and-add-timeout-for-all.patch | 465 ++++ 0014-add-retry-for-read-write.patch | 360 +++ 0015-support-pull-image-with-digest.patch | 302 +++ ...isulad-shim-support-execSync-with-timeout.patch | 186 ++ 0017-Refine-the-commit-info.patch | 30 + 0018-Refine-typo-of-word-container.patch | 85 + 0019-cleancode-for-read-write.patch | 54 + 0020-add-crictl-timeout-and-sync-for-CI.patch | 127 ++ 0021-unlock-m_podsLock-if-new-failed.patch | 27 + 0022-Update-CRI.patch | 73 + 0023-add-cgroup-cpu-ut.patch | 70 + 0024-remove-temp-variables.patch | 179 ++ 0025-fix-read-member-error-from-struct.patch | 34 + 0026-Fix-PR-runc.patch | 108 + ...-allow-the-paused-container-to-be-stopped.patch | 68 + 0028-Refine.patch | 29 + ...support-isula-update-when-runtime-is-runc.patch | 291 +++ 0030-Refine-as-others-feedback.patch | 41 + ...fix-CRI-SetupPod-and-TearDownPod-deadlock.patch | 134 ++ 0032-remote-layer-store-demo.patch | 2296 ++++++++++++++++++++ 0033-add-ci-for-remote-ro.patch | 119 + ...ep-to-usleep-to-avoid-lossing-of-accuracy.patch | 26 + ...x-compile-error-when-not-enable-remote-ro.patch | 62 + 0036-adapt-to-repo-of-openeuler-url-changed.patch | 33 + 0037-change-goto-branch.patch | 27 + 0038-CI-not-enable-remote-ro-for-ut.patch | 41 + 0039-use-auto-free-to-proc_t.patch | 43 + 0040-modifying-cpurt-file-permissions.patch | 72 + ..._XFERINFOFUNCTION-instead-of-deprecated-C.patch | 131 ++ ...te-ro-try-add-or-remove-image-layer-twice.patch | 99 + ...-t-delete-layers-under-dir-overlay-layers.patch | 34 + ...ocument-about-support-remote-ro-directory.patch | 396 ++++ 0045-Refine-a-minor-log-message.patch | 34 + ...return-value-of-the-util_waitpid_with_tim.patch | 44 + 0047-fix-util_getgrent_r-overflow.patch | 74 + 0048-add-ut-for-runc.patch | 188 ++ 0049-add-runc-doc.patch | 506 +++++ 0050-fix-isula_rt_ops_ut-bugs.patch | 28 + 0051-refactor-remote-ro-code.patch | 2089 ++++++++++++++++++ 0052-add-ci-for-runc.patch | 1666 ++++++++++++++ ...ix-when-refresh-can-t-load-or-pull-images.patch | 319 +++ iSulad.spec | 877 ++++++++ sources | 1 + 56 files changed, 14260 insertions(+) create mode 100644 0001-modify-dependence-from-lcr-to-libisula.patch create mode 100644 0002-Add-unified-memory_swap_limit_in_bytes-fields-into-C.patch create mode 100644 0003-Add-macro-for-protoc-cmake.patch create mode 100644 0004-fix-design-typo.patch create mode 100644 0005-fix-cpu-rt-review-comments.patch create mode 100644 0006-fix-inspect.sh-failed.patch create mode 100644 0007-add-CRI-ContainerStats-Service.patch create mode 100644 0008-fix-isula-cpu-rt-CI.patch create mode 100644 0009-fix-cpu-rt-CI.patch create mode 100644 0010-fix-cpu-rt-CI.patch create mode 100644 0011-Bugfix-in-config-and-executor.patch create mode 100644 0012-fix-cpu-rt-disable-after-reboot-machine.patch create mode 100644 0013-fix-selinux_label_ut-timeout-and-add-timeout-for-all.patch create mode 100644 0014-add-retry-for-read-write.patch create mode 100644 0015-support-pull-image-with-digest.patch create mode 100644 0016-isulad-shim-support-execSync-with-timeout.patch create mode 100644 0017-Refine-the-commit-info.patch create mode 100644 0018-Refine-typo-of-word-container.patch create mode 100644 0019-cleancode-for-read-write.patch create mode 100644 0020-add-crictl-timeout-and-sync-for-CI.patch create mode 100644 0021-unlock-m_podsLock-if-new-failed.patch create mode 100644 0022-Update-CRI.patch create mode 100644 0023-add-cgroup-cpu-ut.patch create mode 100644 0024-remove-temp-variables.patch create mode 100644 0025-fix-read-member-error-from-struct.patch create mode 100644 0026-Fix-PR-runc.patch create mode 100644 0027-allow-the-paused-container-to-be-stopped.patch create mode 100644 0028-Refine.patch create mode 100644 0029-support-isula-update-when-runtime-is-runc.patch create mode 100644 0030-Refine-as-others-feedback.patch create mode 100644 0031-fix-CRI-SetupPod-and-TearDownPod-deadlock.patch create mode 100644 0032-remote-layer-store-demo.patch create mode 100644 0033-add-ci-for-remote-ro.patch create mode 100644 0034-change-sleep-to-usleep-to-avoid-lossing-of-accuracy.patch create mode 100644 0035-fix-compile-error-when-not-enable-remote-ro.patch create mode 100644 0036-adapt-to-repo-of-openeuler-url-changed.patch create mode 100644 0037-change-goto-branch.patch create mode 100644 0038-CI-not-enable-remote-ro-for-ut.patch create mode 100644 0039-use-auto-free-to-proc_t.patch create mode 100644 0040-modifying-cpurt-file-permissions.patch create mode 100644 0041-use-CURLOPT_XFERINFOFUNCTION-instead-of-deprecated-C.patch create mode 100644 0042-bugfix-remote-ro-try-add-or-remove-image-layer-twice.patch create mode 100644 0043-bugfix-can-t-delete-layers-under-dir-overlay-layers.patch create mode 100644 0044-doc-add-document-about-support-remote-ro-directory.patch create mode 100644 0045-Refine-a-minor-log-message.patch create mode 100644 0046-modify-the-return-value-of-the-util_waitpid_with_tim.patch create mode 100644 0047-fix-util_getgrent_r-overflow.patch create mode 100644 0048-add-ut-for-runc.patch create mode 100644 0049-add-runc-doc.patch create mode 100644 0050-fix-isula_rt_ops_ut-bugs.patch create mode 100644 0051-refactor-remote-ro-code.patch create mode 100644 0052-add-ci-for-runc.patch create mode 100644 0053-bugfix-when-refresh-can-t-load-or-pull-images.patch create mode 100644 iSulad.spec create mode 100644 sources diff --git a/.gitignore b/.gitignore index e69de29..ac28aec 100644 --- a/.gitignore +++ b/.gitignore @@ -0,0 +1 @@ +/v2.1.1.tar.gz diff --git a/0001-modify-dependence-from-lcr-to-libisula.patch b/0001-modify-dependence-from-lcr-to-libisula.patch new file mode 100644 index 0000000..83d70eb --- /dev/null +++ b/0001-modify-dependence-from-lcr-to-libisula.patch @@ -0,0 +1,206 @@ +From d3d13f374bb341e7e4d389feb1780e43fec91945 Mon Sep 17 00:00:00 2001 +From: zhangxiaoyu +Date: Tue, 7 Feb 2023 14:27:51 +0800 +Subject: [PATCH 01/53] modify dependence from lcr to libisula + +Signed-off-by: zhangxiaoyu +--- + cmake/checker.cmake | 2 +- + iSulad.spec | 10 ++++----- + isulad.pc.in | 6 +++--- + src/CMakeLists.txt | 32 ++++++++++++++-------------- + src/daemon/modules/runtime/runtime.c | 14 ++---------- + 5 files changed, 27 insertions(+), 37 deletions(-) + +diff --git a/cmake/checker.cmake b/cmake/checker.cmake +index 0a4b6fb6..a1ca9360 100644 +--- a/cmake/checker.cmake ++++ b/cmake/checker.cmake +@@ -92,7 +92,7 @@ if (ENABLE_SELINUX) + endif() + + # check iSula libutils +-pkg_check_modules(PC_ISULA_LIBUTILS REQUIRED "lcr") ++pkg_check_modules(PC_ISULA_LIBUTILS REQUIRED "libisula") + find_path(ISULA_LIBUTILS_INCLUDE_DIR isula_libutils/log.h + HINTS ${PC_ISULA_LIBUTILS_INCLUDEDIR} ${PC_ISULA_LIBUTILS_INCLUDE_DIRS}) + _CHECK(ISULA_LIBUTILS_INCLUDE_DIR "ISULA_LIBUTILS_INCLUDE_DIR-NOTFOUND" "isula_libutils/log.h") +diff --git a/iSulad.spec b/iSulad.spec +index 79275885..3cff9c25 100644 +--- a/iSulad.spec ++++ b/iSulad.spec +@@ -16,7 +16,7 @@ ExclusiveArch: x86_64 aarch64 + + %ifarch x86_64 aarch64 + Provides: libhttpclient.so()(64bit) +-Provides: libisula.so()(64bit) ++Provides: libisula_client.so()(64bit) + Provides: libisulad_img.so()(64bit) + Provides: libisulad_tools.so()(64bit) + %endif +@@ -42,14 +42,14 @@ BuildRequires: lib-shim-v2-devel + Requires: lib-shim-v2 + %endif + +-BuildRequires: cmake gcc-c++ lxc-devel lcr-devel yajl-devel ++BuildRequires: cmake gcc-c++ lxc-devel lcr-devel yajl-devel libisula-devel + BuildRequires: grpc-plugins grpc-devel protobuf-devel + BuildRequires: libcurl-devel libarchive-devel device-mapper-devel + BuildRequires: http-parser-devel + BuildRequires: libselinux-devel libwebsockets-devel + BuildRequires: systemd-devel git + +-Requires: lcr lxc ++Requires: libisula lxc + Requires: grpc libcurl http-parser + Requires: libselinux libwebsockets libarchive device-mapper + Requires: systemd +@@ -78,13 +78,13 @@ cd build + rm -rf %{buildroot} + cd build + install -d $RPM_BUILD_ROOT/%{_libdir} +-install -m 0644 ./src/libisula.so %{buildroot}/%{_libdir}/libisula.so ++install -m 0644 ./src/libisula_client.so %{buildroot}/%{_libdir}/libisula_client.so + install -m 0644 ./src/utils/http/libhttpclient.so %{buildroot}/%{_libdir}/libhttpclient.so + chrpath -d ./src/libisulad_tools.so + install -m 0644 ./src/libisulad_tools.so %{buildroot}/%{_libdir}/libisulad_tools.so + chrpath -d ./src/daemon/modules/image/libisulad_img.so + install -m 0644 ./src/daemon/modules/image/libisulad_img.so %{buildroot}/%{_libdir}/libisulad_img.so +-chmod +x %{buildroot}/%{_libdir}/libisula.so ++chmod +x %{buildroot}/%{_libdir}/libisula_client.so + chmod +x %{buildroot}/%{_libdir}/libhttpclient.so + chmod +x %{buildroot}/%{_libdir}/libisulad_img.so + +diff --git a/isulad.pc.in b/isulad.pc.in +index 695ba364..016e406d 100644 +--- a/isulad.pc.in ++++ b/isulad.pc.in +@@ -3,10 +3,10 @@ libdir=@CMAKE_INSTALL_PREFIX@/lib + localstatedir=@CMAKE_INSTALL_PREFIX@/var + includedir=@CMAKE_INSTALL_PREFIX@/include + +-Name: libisula +-Description: light-weighted container runtime daemon library ++Name: libisula_client ++Description: light-weighted container client library + Version: @ISULAD_VERSION@ + URL: iSulad +-Libs: -L@CMAKE_INSTALL_PREFIX@/lib -lisula ++Libs: -L@CMAKE_INSTALL_PREFIX@/lib -lisula_client + Cflags: -I@CMAKE_INSTALL_PREFIX@/include + +diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt +index abce1284..8de18082 100644 +--- a/src/CMakeLists.txt ++++ b/src/CMakeLists.txt +@@ -60,40 +60,40 @@ endif() + # get all c and header files + list(REMOVE_DUPLICATES SHARED_INCS) + +-# ------ build libisula ------ ++# ------ build libisula_client ------ + + add_subdirectory(client) + if (OPENSSL_VERIFY) + list(APPEND CLIENT_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/utils/http/certificate.c) + endif() + +-add_library(libisula ${LIBTYPE} ++add_library(libisula_client ${LIBTYPE} + ${CLIENT_SRCS} + ) + +-target_include_directories(libisula PUBLIC ++target_include_directories(libisula_client PUBLIC + ${SHARED_INCS} + ${CLIENT_INCS} + ${CMAKE_CURRENT_SOURCE_DIR}/utils/http + ) + +-# set libisula FLAGS +-set_target_properties(libisula PROPERTIES PREFIX "") ++# set libisula_client FLAGS ++set_target_properties(libisula_client PROPERTIES PREFIX "") + +-target_link_libraries(libisula libisulad_tools) ++target_link_libraries(libisula_client libisulad_tools) + + if (GRPC_CONNECTOR) +- target_link_libraries(libisula -Wl,--as-needed -lstdc++) +- target_link_libraries(libisula -Wl,--as-needed ${PROTOBUF_LIBRARY}) +- target_link_libraries(libisula -Wl,--no-as-needed ${GRPC_PP_REFLECTION_LIBRARY} ${GRPC_PP_LIBRARY} ${GRPC_LIBRARY} ${GPR_LIBRARY}) ++ target_link_libraries(libisula_client -Wl,--as-needed -lstdc++) ++ target_link_libraries(libisula_client -Wl,--as-needed ${PROTOBUF_LIBRARY}) ++ target_link_libraries(libisula_client -Wl,--no-as-needed ${GRPC_PP_REFLECTION_LIBRARY} ${GRPC_PP_LIBRARY} ${GRPC_LIBRARY} ${GPR_LIBRARY}) + if(ABSL_SYNC_LIB) +- target_link_libraries(libisula -Wl,--no-as-needed ${ABSL_SYNC_LIB}) ++ target_link_libraries(libisula_client -Wl,--no-as-needed ${ABSL_SYNC_LIB}) + endif() + else() +- target_link_libraries(libisula -ldl libhttpclient) +- set_target_properties(libisula PROPERTIES LINKER_LANGUAGE "C") ++ target_link_libraries(libisula_client -ldl libhttpclient) ++ set_target_properties(libisula_client PROPERTIES LINKER_LANGUAGE "C") + endif() +-# ------ build libisula finish ----- ++# ------ build libisula_client finish ----- + + add_subdirectory(cmd) + # ------ build isula ------- +@@ -101,7 +101,7 @@ add_executable(isula + ${ISULA_SRCS} + ) + target_include_directories(isula PUBLIC ${ISULA_INCS} ${SHARED_INCS}) +-target_link_libraries(isula libisula ${LIBYAJL_LIBRARY}) ++target_link_libraries(isula libisula_client ${LIBYAJL_LIBRARY}) + if (ANDROID OR MUSL) + target_link_libraries(isula ${LIBSSL_LIBRARY}) + else() +@@ -192,7 +192,7 @@ endif() + + if (ISULAD_GCOV) + target_link_libraries(isula -lgcov) +- target_link_libraries(libisula -lgcov) ++ target_link_libraries(libisula_client -lgcov) + target_link_libraries(isulad -lgcov) + endif() + +@@ -201,7 +201,7 @@ endif() + # ------ install binary -------- + install(TARGETS libisulad_tools + ${INSTALL_TYPE} DESTINATION ${LIB_INSTALL_DIR_DEFAULT} PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) +-install(TARGETS libisula ++install(TARGETS libisula_client + ${INSTALL_TYPE} DESTINATION ${LIB_INSTALL_DIR_DEFAULT} PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) + install(TARGETS isula + RUNTIME DESTINATION bin PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) +diff --git a/src/daemon/modules/runtime/runtime.c b/src/daemon/modules/runtime/runtime.c +index 23a4d60f..7a3ed87f 100644 +--- a/src/daemon/modules/runtime/runtime.c ++++ b/src/daemon/modules/runtime/runtime.c +@@ -542,20 +542,10 @@ bool is_default_runtime(const char *name) + + int runtime_init() + { +- int ret = 0; +- + if (engines_global_init()) { + ERROR("Init engines global failed"); +- ret = -1; +- goto out; ++ return -1; + } + +- /* Init default engine, now is lcr */ +- if (engines_discovery(DEFAULT_RUNTIME_NAME)) { +- ERROR("Failed to discovery engine %s", DEFAULT_RUNTIME_NAME); +- ret = -1; +- } +- +-out: +- return ret; ++ return 0; + } +-- +2.25.1 + diff --git a/0002-Add-unified-memory_swap_limit_in_bytes-fields-into-C.patch b/0002-Add-unified-memory_swap_limit_in_bytes-fields-into-C.patch new file mode 100644 index 0000000..b492de1 --- /dev/null +++ b/0002-Add-unified-memory_swap_limit_in_bytes-fields-into-C.patch @@ -0,0 +1,243 @@ +From 27c3d00f74c5641685d5781fe0c02c5eead92d23 Mon Sep 17 00:00:00 2001 +From: "ilya.kuksenok" +Date: Thu, 2 Feb 2023 14:41:16 +0300 +Subject: [PATCH 02/53] Add unified, memory_swap_limit_in_bytes fields into + ContainerStats; add unified and memory_swap_limit_in_bytes into + UpdateCreateConfig add nullptr for unified. + +--- + src/daemon/common/sysinfo.c | 3 +- + .../cri/cri_container_manager_service.cc | 19 ++++++++++++ + src/daemon/entry/cri/cri_helpers.cc | 17 ++++++++++- + src/daemon/modules/runtime/runtime.c | 30 +++++++++---------- + 4 files changed, 52 insertions(+), 17 deletions(-) + +diff --git a/src/daemon/common/sysinfo.c b/src/daemon/common/sysinfo.c +index 8b5768db..38416db4 100644 +--- a/src/daemon/common/sysinfo.c ++++ b/src/daemon/common/sysinfo.c +@@ -743,7 +743,8 @@ static void check_cgroup_mem(struct layer **layers, bool quiet, cgroup_mem_info_ + return; + } + +- meminfo->limit = true; ++ meminfo->limit = cgroup_enabled(mountpoint, CGROUP_MEMORY_LIMIT); ++ cgroup_do_log(quiet, !(meminfo->limit), "Your kernel does not support memory limit"); + + meminfo->swap = cgroup_enabled(mountpoint, CGROUP_MEMORY_SWAP); + cgroup_do_log(quiet, !(meminfo->swap), "Your kernel does not support swap memory limit"); +diff --git a/src/daemon/entry/cri/cri_container_manager_service.cc b/src/daemon/entry/cri/cri_container_manager_service.cc +index d2f486cf..d044cca8 100644 +--- a/src/daemon/entry/cri/cri_container_manager_service.cc ++++ b/src/daemon/entry/cri/cri_container_manager_service.cc +@@ -1084,6 +1084,12 @@ void ContainerManagerService::UpdateContainerResources(const std::string &contai + struct parser_context ctx { + OPT_GEN_SIMPLIFY, 0 + }; ++ json_map_string_string *unified = nullptr; ++ unified = (json_map_string_string *)util_common_calloc_s(sizeof(json_map_string_string)); ++ if (unified == nullptr) { ++ error.SetError("Out of memory"); ++ goto cleanup; ++ } + request = (container_update_request *)util_common_calloc_s(sizeof(container_update_request)); + if (request == nullptr) { + error.SetError("Out of memory"); +@@ -1100,6 +1106,18 @@ void ContainerManagerService::UpdateContainerResources(const std::string &contai + hostconfig->cpu_period = resources.cpu_period(); + hostconfig->cpu_quota = resources.cpu_quota(); + hostconfig->cpu_shares = resources.cpu_shares(); ++ hostconfig->memory_swap_limit_in_bytes = resources.memory_swap_limit_in_bytes(); ++ ++ if (!resources.unified().empty()) { ++ for (auto &iter : resources.unified()) { ++ if (append_json_map_string_string(unified, iter.first.c_str(), iter.second.c_str()) != 0) { ++ error.SetError("Failed to append string"); ++ goto cleanup; ++ } ++ } ++ } ++ hostconfig->unified = unified; ++ unified = nullptr; + hostconfig->memory = resources.memory_limit_in_bytes(); + if (!resources.cpuset_cpus().empty()) { + hostconfig->cpuset_cpus = util_strdup_s(resources.cpuset_cpus().c_str()); +@@ -1126,6 +1144,7 @@ cleanup: + free_container_update_request(request); + free_container_update_response(response); + free_host_config(hostconfig); ++ free_json_map_string_string(unified); + free(perror); + } + +diff --git a/src/daemon/entry/cri/cri_helpers.cc b/src/daemon/entry/cri/cri_helpers.cc +index ddcc153f..2f6dcf78 100644 +--- a/src/daemon/entry/cri/cri_helpers.cc ++++ b/src/daemon/entry/cri/cri_helpers.cc +@@ -445,8 +445,23 @@ void UpdateCreateConfig(container_config *createConfig, host_config *hc, + hc->cpuset_mems = util_strdup_s(rOpts.cpuset_mems().c_str()); + } + hc->oom_score_adj = rOpts.oom_score_adj(); ++ hc->memory_swap_limit_in_bytes = rOpts.memory_swap_limit_in_bytes(); ++ auto *unified = (json_map_string_string *)util_common_calloc_s(sizeof(json_map_string_string)); ++ if (unified == nullptr) { ++ error.SetError("Out of memory"); ++ return; ++ } ++ if (!rOpts.unified().empty()) { ++ for (auto &iter : rOpts.unified()) { ++ if (append_json_map_string_string(unified, iter.first.c_str(), iter.second.c_str()) != 0) { ++ error.SetError("Failed to append string"); ++ free_json_map_string_string(unified); ++ return; ++ } ++ } ++ } ++ hc->unified = unified; + } +- + createConfig->open_stdin = config.stdin(); + createConfig->tty = config.tty(); + } +diff --git a/src/daemon/modules/runtime/runtime.c b/src/daemon/modules/runtime/runtime.c +index 7a3ed87f..29a64ac1 100644 +--- a/src/daemon/modules/runtime/runtime.c ++++ b/src/daemon/modules/runtime/runtime.c +@@ -122,7 +122,7 @@ int runtime_create(const char *name, const char *runtime, const rt_create_params + const struct rt_ops *ops = NULL; + + if (name == NULL || runtime == NULL) { +- ERROR("Invalide arguments for runtime create"); ++ ERROR("Invalid arguments for runtime create"); + ret = -1; + goto out; + } +@@ -146,7 +146,7 @@ int runtime_start(const char *name, const char *runtime, const rt_start_params_t + const struct rt_ops *ops = NULL; + + if (name == NULL || runtime == NULL || pid_info == NULL) { +- ERROR("Invalide arguments for runtime start"); ++ ERROR("Invalid arguments for runtime start"); + ret = -1; + goto out; + } +@@ -194,7 +194,7 @@ int runtime_restart(const char *name, const char *runtime, const rt_restart_para + const struct rt_ops *ops = NULL; + + if (name == NULL || runtime == NULL) { +- ERROR("Invalide arguments for runtime restart"); ++ ERROR("Invalid arguments for runtime restart"); + ret = -1; + goto out; + } +@@ -218,7 +218,7 @@ int runtime_clean_resource(const char *name, const char *runtime, const rt_clean + const struct rt_ops *ops = NULL; + + if (name == NULL || runtime == NULL) { +- ERROR("Invalide arguments for runtime clean"); ++ ERROR("Invalid arguments for runtime clean"); + ret = -1; + goto out; + } +@@ -242,7 +242,7 @@ int runtime_rm(const char *name, const char *runtime, const rt_rm_params_t *para + const struct rt_ops *ops = NULL; + + if (name == NULL || runtime == NULL) { +- ERROR("Invalide arguments for runtime rm"); ++ ERROR("Invalid arguments for runtime rm"); + ret = -1; + goto out; + } +@@ -267,7 +267,7 @@ int runtime_status(const char *name, const char *runtime, const rt_status_params + const struct rt_ops *ops = NULL; + + if (name == NULL || runtime == NULL || status == NULL) { +- ERROR("Invalide arguments for runtime status"); ++ ERROR("Invalid arguments for runtime status"); + ret = -1; + goto out; + } +@@ -292,7 +292,7 @@ int runtime_resources_stats(const char *name, const char *runtime, const rt_stat + const struct rt_ops *ops = NULL; + + if (name == NULL || runtime == NULL || rs_stats == NULL) { +- ERROR("Invalide arguments for runtime stats"); ++ ERROR("Invalid arguments for runtime stats"); + ret = -1; + goto out; + } +@@ -316,7 +316,7 @@ int runtime_exec(const char *name, const char *runtime, const rt_exec_params_t * + const struct rt_ops *ops = NULL; + + if (name == NULL || runtime == NULL || exit_code == NULL) { +- ERROR("Invalide arguments for runtime exec"); ++ ERROR("Invalid arguments for runtime exec"); + ret = -1; + goto out; + } +@@ -340,7 +340,7 @@ int runtime_pause(const char *name, const char *runtime, const rt_pause_params_t + const struct rt_ops *ops = NULL; + + if (name == NULL || runtime == NULL || params == NULL) { +- ERROR("Invalide arguments for runtime pause"); ++ ERROR("Invalid arguments for runtime pause"); + ret = -1; + goto out; + } +@@ -364,7 +364,7 @@ int runtime_resume(const char *name, const char *runtime, const rt_resume_params + const struct rt_ops *ops = NULL; + + if (name == NULL || runtime == NULL || params == NULL) { +- ERROR("Invalide arguments for runtime resume"); ++ ERROR("Invalid arguments for runtime resume"); + ret = -1; + goto out; + } +@@ -388,7 +388,7 @@ int runtime_attach(const char *name, const char *runtime, const rt_attach_params + const struct rt_ops *ops = NULL; + + if (name == NULL || runtime == NULL || params == NULL) { +- ERROR("Invalide arguments for runtime attach"); ++ ERROR("Invalid arguments for runtime attach"); + ret = -1; + goto out; + } +@@ -412,7 +412,7 @@ int runtime_update(const char *name, const char *runtime, const rt_update_params + const struct rt_ops *ops = NULL; + + if (name == NULL || runtime == NULL || params == NULL) { +- ERROR("Invalide arguments for runtime update"); ++ ERROR("Invalid arguments for runtime update"); + ret = -1; + goto out; + } +@@ -447,7 +447,7 @@ int runtime_listpids(const char *name, const char *runtime, const rt_listpids_pa + const struct rt_ops *ops = NULL; + + if (name == NULL || runtime == NULL || params == NULL || out == NULL) { +- ERROR("Invalide arguments for runtime listpids"); ++ ERROR("Invalid arguments for runtime listpids"); + ret = -1; + goto out; + } +@@ -471,7 +471,7 @@ int runtime_resize(const char *name, const char *runtime, const rt_resize_params + const struct rt_ops *ops = NULL; + + if (name == NULL || runtime == NULL || params == NULL) { +- ERROR("Invalide arguments for runtime resize"); ++ ERROR("Invalid arguments for runtime resize"); + ret = -1; + goto out; + } +@@ -495,7 +495,7 @@ int runtime_exec_resize(const char *name, const char *runtime, const rt_exec_res + const struct rt_ops *ops = NULL; + + if (name == NULL || runtime == NULL || params == NULL) { +- ERROR("Invalide arguments for runtime exec resize"); ++ ERROR("Invalid arguments for runtime exec resize"); + ret = -1; + goto out; + } +-- +2.25.1 + diff --git a/0003-Add-macro-for-protoc-cmake.patch b/0003-Add-macro-for-protoc-cmake.patch new file mode 100644 index 0000000..accf473 --- /dev/null +++ b/0003-Add-macro-for-protoc-cmake.patch @@ -0,0 +1,135 @@ +From 466309bc0aafe61ebed5c71012e28b9912783b60 Mon Sep 17 00:00:00 2001 +From: Xuepeng Xu +Date: Thu, 9 Feb 2023 14:32:59 +0800 +Subject: [PATCH 03/53] Add macro for protoc cmake + +Signed-off-by: Xuepeng Xu +--- + cmake/protoc.cmake | 98 +++++++++++++--------------------------------- + 1 file changed, 28 insertions(+), 70 deletions(-) + +diff --git a/cmake/protoc.cmake b/cmake/protoc.cmake +index 5c433e5c..23b8c077 100644 +--- a/cmake/protoc.cmake ++++ b/cmake/protoc.cmake +@@ -11,89 +11,47 @@ if (ENABLE_NATIVE_NETWORK) + set(NETWORK_PROTOS_OUT_PATH ${GRPC_OUT_PRE_PATH}/src/api/services/network) + endif() + ++macro(PROTOC_CPP_GEN proto_name cpp_out_path proto_path) ++ execute_process(COMMAND ${CMD_PROTOC} -I ${PROTOS_PATH}/${proto_name} --cpp_out=${cpp_out_path} ${proto_path} ERROR_VARIABLE cpp_err) ++ if (cpp_err) ++ message("Parse ${proto_path} failed: ") ++ message(FATAL_ERROR ${cpp_err}) ++ endif() ++endmacro(PROTOC_CPP_GEN) ++ ++macro(PROTOC_GRPC_GEN proto_name grpc_out_path proto_path) ++ execute_process(COMMAND ${CMD_PROTOC} -I ${PROTOS_PATH}/${proto_name} --grpc_out=${grpc_out_path} --plugin=protoc-gen-grpc=${CMD_GRPC_CPP_PLUGIN} ${proto_path} ERROR_VARIABLE grpc_err) ++ if (grpc_err) ++ message("Parse ${proto_path} failed: ") ++ message(FATAL_ERROR ${grpc_err}) ++ endif() ++endmacro(PROTOC_GRPC_GEN) ++ + if (GRPC_CONNECTOR) + execute_process(COMMAND mkdir -p ${CONTAINER_PROTOS_OUT_PATH}) + execute_process(COMMAND mkdir -p ${IMAGE_PROTOS_OUT_PATH}) + execute_process(COMMAND mkdir -p ${VOLUME_PROTOS_OUT_PATH}) + execute_process(COMMAND mkdir -p ${CRI_PROTOS_OUT_PATH}) +- execute_process(COMMAND ${CMD_PROTOC} -I ${PROTOS_PATH}/containers --cpp_out=${CONTAINER_PROTOS_OUT_PATH} +- ${PROTOS_PATH}/containers/container.proto ERROR_VARIABLE containers_err) +- if (containers_err) +- message("Parse ${PROTOS_PATH}/containers/container.proto failed: ") +- message(FATAL_ERROR ${containers_err}) +- endif() + +- execute_process(COMMAND ${CMD_PROTOC} -I ${PROTOS_PATH}/containers --grpc_out=${CONTAINER_PROTOS_OUT_PATH} --plugin=protoc-gen-grpc=${CMD_GRPC_CPP_PLUGIN} ${PROTOS_PATH}/containers/container.proto ERROR_VARIABLE containers_err) +- if (containers_err) +- message("Parse ${PROTOS_PATH}/containers/container.proto plugin failed: ") +- message(FATAL_ERROR ${containers_err}) +- endif() ++ PROTOC_CPP_GEN(containers ${CONTAINER_PROTOS_OUT_PATH} ${PROTOS_PATH}/containers/container.proto) ++ PROTOC_GRPC_GEN(containers ${CONTAINER_PROTOS_OUT_PATH} ${PROTOS_PATH}/containers/container.proto) + +- execute_process(COMMAND ${CMD_PROTOC} -I ${PROTOS_PATH}/images --cpp_out=${IMAGE_PROTOS_OUT_PATH} ${PROTOS_PATH}/images/images.proto ERROR_VARIABLE images_err) +- if (images_err) +- message("Parse ${PROTOS_PATH}/images/images.proto failed: ") +- message(FATAL_ERROR ${images_err}) +- endif() ++ PROTOC_CPP_GEN(images ${IMAGE_PROTOS_OUT_PATH} ${PROTOS_PATH}/images/images.proto) ++ PROTOC_GRPC_GEN(images ${IMAGE_PROTOS_OUT_PATH} ${PROTOS_PATH}/images/images.proto) + +- execute_process(COMMAND ${CMD_PROTOC} -I ${PROTOS_PATH}/images --grpc_out=${IMAGE_PROTOS_OUT_PATH} --plugin=protoc-gen-grpc=${CMD_GRPC_CPP_PLUGIN} ${PROTOS_PATH}/images/images.proto ERROR_VARIABLE images_err) +- if (images_err) +- message("Parse ${PROTOS_PATH}/images/images.proto plugin failed: ") +- message(FATAL_ERROR ${images_err}) +- endif() ++ PROTOC_CPP_GEN(volumes ${VOLUME_PROTOS_OUT_PATH} ${PROTOS_PATH}/volumes/volumes.proto) ++ PROTOC_GRPC_GEN(volumes ${VOLUME_PROTOS_OUT_PATH} ${PROTOS_PATH}/volumes/volumes.proto) + +- execute_process(COMMAND ${CMD_PROTOC} -I ${PROTOS_PATH}/volumes --cpp_out=${VOLUME_PROTOS_OUT_PATH} ${PROTOS_PATH}/volumes/volumes.proto ERROR_VARIABLE volumes_err) +- if (volumes_err) +- message("Parse ${PROTOS_PATH}/volumes/volumes.proto failed: ") +- message(FATAL_ERROR ${volumes_err}) +- endif() ++ PROTOC_CPP_GEN(cri ${CRI_PROTOS_OUT_PATH} ${PROTOS_PATH}/cri/api.proto) ++ PROTOC_GRPC_GEN(cri ${CRI_PROTOS_OUT_PATH} ${PROTOS_PATH}/cri/api.proto) + +- execute_process(COMMAND ${CMD_PROTOC} -I ${PROTOS_PATH}/volumes --grpc_out=${VOLUME_PROTOS_OUT_PATH} --plugin=protoc-gen-grpc=${CMD_GRPC_CPP_PLUGIN} ${PROTOS_PATH}/volumes/volumes.proto ERROR_VARIABLE volumes_err) +- if (volumes_err) +- message("Parse ${PROTOS_PATH}/volumes/volumes.proto plugin failed: ") +- message(FATAL_ERROR ${volumes_err}) +- endif() +- +- execute_process(COMMAND ${CMD_PROTOC} -I ${PROTOS_PATH}/cri --cpp_out=${CRI_PROTOS_OUT_PATH} ${PROTOS_PATH}/cri/api.proto +- ERROR_VARIABLE cri_err) +- if (cri_err) +- message("Parse ${PROTOS_PATH}/cri/api.proto failed: ") +- message(FATAL_ERROR ${cri_err}) +- endif() +- +- execute_process(COMMAND ${CMD_PROTOC} -I ${PROTOS_PATH}/cri --grpc_out=${CRI_PROTOS_OUT_PATH} +- --plugin=protoc-gen-grpc=${CMD_GRPC_CPP_PLUGIN} ${PROTOS_PATH}/cri/api.proto ERROR_VARIABLE cri_err) +- if (cri_err) +- message("Parse ${PROTOS_PATH}/cri/api.proto plugin failed: ") +- message(FATAL_ERROR ${cri_err}) +- endif() +- execute_process(COMMAND ${CMD_PROTOC} -I ${PROTOS_PATH}/cri --cpp_out=${CRI_PROTOS_OUT_PATH} ${PROTOS_PATH}/cri/gogo.proto +- ERROR_VARIABLE cri_err) +- if (cri_err) +- message("Parse ${PROTOS_PATH}/cri/gogo.proto failed: ") +- message(FATAL_ERROR ${cri_err}) +- endif() +- execute_process(COMMAND ${CMD_PROTOC} -I ${PROTOS_PATH}/cri --grpc_out=${CRI_PROTOS_OUT_PATH} +- --plugin=protoc-gen-grpc=${CMD_GRPC_CPP_PLUGIN} ${PROTOS_PATH}/cri/gogo.proto ERROR_VARIABLE cri_err) +- if (cri_err) +- message("Parse ${PROTOS_PATH}/cri/gogo.proto plugin failed: ") +- message(FATAL_ERROR ${cri_err}) +- endif() ++ PROTOC_CPP_GEN(cri ${CRI_PROTOS_OUT_PATH} ${PROTOS_PATH}/cri/gogo.proto) ++ PROTOC_GRPC_GEN(cri ${CRI_PROTOS_OUT_PATH} ${PROTOS_PATH}/cri/gogo.proto) + + if (ENABLE_NATIVE_NETWORK) + execute_process(COMMAND mkdir -p ${NETWORK_PROTOS_OUT_PATH}) +- execute_process(COMMAND ${CMD_PROTOC} -I ${PROTOS_PATH}/network +- --cpp_out=${NETWORK_PROTOS_OUT_PATH} ${PROTOS_PATH}/network/network.proto ERROR_VARIABLE network_err) +- if (network_err) +- message("Parse ${PROTOS_PATH}/network/network.proto failed: ") +- message(FATAL_ERROR ${network_err}) +- endif() +- +- execute_process(COMMAND ${CMD_PROTOC} -I ${PROTOS_PATH}/network --grpc_out=${NETWORK_PROTOS_OUT_PATH} +- --plugin=protoc-gen-grpc=${CMD_GRPC_CPP_PLUGIN} ${PROTOS_PATH}/network/network.proto ERROR_VARIABLE network_err) +- if (network_err) +- message("Parse ${PROTOS_PATH}/network/network.proto plugin failed: ") +- message(FATAL_ERROR ${network_err}) +- endif() ++ PROTOC_CPP_GEN(network ${NETWORK_PROTOS_OUT_PATH} ${PROTOS_PATH}/network/network.proto) ++ PROTOC_GRPC_GEN(network ${NETWORK_PROTOS_OUT_PATH} ${PROTOS_PATH}/network/network.proto) + endif() + endif() + +-- +2.25.1 + diff --git a/0004-fix-design-typo.patch b/0004-fix-design-typo.patch new file mode 100644 index 0000000..dafc2c7 --- /dev/null +++ b/0004-fix-design-typo.patch @@ -0,0 +1,25 @@ +From a7df50dc3b51f961f3d2e48dd968cfb115c39fec Mon Sep 17 00:00:00 2001 +From: zhushy +Date: Sat, 11 Feb 2023 00:05:53 +0800 +Subject: [PATCH 04/53] fix design typo + +--- + README.md | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/README.md b/README.md +index 3d1fc7cb..7e4b6de1 100644 +--- a/README.md ++++ b/README.md +@@ -70,7 +70,7 @@ CRI interface is implemented based on gRPC. iSulad implemented CRI gRPC Server f + + - [user manual](./docs/manual/README.md) + +-- [desgin docs](./docs/design/README.md) ++- [design docs](./docs/design/README.md) + + ### Installing + +-- +2.25.1 + diff --git a/0005-fix-cpu-rt-review-comments.patch b/0005-fix-cpu-rt-review-comments.patch new file mode 100644 index 0000000..d5f812d --- /dev/null +++ b/0005-fix-cpu-rt-review-comments.patch @@ -0,0 +1,44 @@ +From 257054b234debb7b1fcafce6f2ec3df828370aed Mon Sep 17 00:00:00 2001 +From: songbuhuang <544824346@qq.com> +Date: Sun, 12 Feb 2023 15:23:37 +0800 +Subject: [PATCH 05/53] fix cpu rt review comments + +Signed-off-by: songbuhuang <544824346@qq.com> +--- + src/daemon/executor/container_cb/execution_create.c | 10 +++++----- + 1 file changed, 5 insertions(+), 5 deletions(-) + +diff --git a/src/daemon/executor/container_cb/execution_create.c b/src/daemon/executor/container_cb/execution_create.c +index feaa3064..cc9ae716 100644 +--- a/src/daemon/executor/container_cb/execution_create.c ++++ b/src/daemon/executor/container_cb/execution_create.c +@@ -1327,7 +1327,7 @@ static int save_container_config_before_create(const char *id, const char *runti + static int maybe_create_cpu_realtime_file(int64_t value, const char *file, const char *path) + { + int ret; +- int fd = 0; ++ int fd = -1; + ssize_t nwrite; + char fpath[PATH_MAX] = { 0 }; + char buf[ISULAD_NUMSTRLEN64] = { 0 }; +@@ -1342,13 +1342,13 @@ static int maybe_create_cpu_realtime_file(int64_t value, const char *file, const + return -1; + } + +- int nret = snprintf(fpath, sizeof(fpath), "%s/%s", path, file); +- if (nret < 0 || nret >= sizeof(fpath)) { ++ ret = snprintf(fpath, sizeof(fpath), "%s/%s", path, file); ++ if (ret < 0 || ret >= sizeof(fpath)) { + ERROR("Failed to print string"); + return -1; + } +- nret = snprintf(buf, sizeof(buf), "%lld", (long long int)value); +- if (nret < 0 || (size_t)nret >= sizeof(buf)) { ++ ret = snprintf(buf, sizeof(buf), "%lld", (long long int)value); ++ if (ret < 0 || (size_t)ret >= sizeof(buf)) { + ERROR("Failed to print string"); + return -1; + } +-- +2.25.1 + diff --git a/0006-fix-inspect.sh-failed.patch b/0006-fix-inspect.sh-failed.patch new file mode 100644 index 0000000..17f5510 --- /dev/null +++ b/0006-fix-inspect.sh-failed.patch @@ -0,0 +1,31 @@ +From e4993d0e89ca853d74d8b23895de0967b4379441 Mon Sep 17 00:00:00 2001 +From: zhangxiaoyu +Date: Mon, 13 Feb 2023 17:42:30 +0800 +Subject: [PATCH 06/53] fix inspect.sh failed + +Signed-off-by: zhangxiaoyu +--- + CI/test_cases/container_cases/inspect.sh | 7 +------ + 1 file changed, 1 insertion(+), 6 deletions(-) + +diff --git a/CI/test_cases/container_cases/inspect.sh b/CI/test_cases/container_cases/inspect.sh +index 0d4ccb02..cde9ea1f 100755 +--- a/CI/test_cases/container_cases/inspect.sh ++++ b/CI/test_cases/container_cases/inspect.sh +@@ -103,12 +103,7 @@ function test_inspect_spec() + isula inspect --format='{{.Image}}' $containername 2>&1 | grep "sha256:${image_id}" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container with image: ${image}" && ((ret++)) + +- if [ -d /sys/fs/cgroup/files ];then +- grepval="100" +- else +- grepval="0" +- fi +- isula inspect --format='{{json .HostConfig.FilesLimit}}' $containername 2>&1 | grep "$grepval" ++ isula inspect --format='{{json .HostConfig.FilesLimit}}' $containername 2>&1 | grep 0 + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container with image: ${image}" && ((ret++)) + + isula inspect --format='{{json .Config.Env}}' $containername 2>&1 | grep "a=1" +-- +2.25.1 + diff --git a/0007-add-CRI-ContainerStats-Service.patch b/0007-add-CRI-ContainerStats-Service.patch new file mode 100644 index 0000000..e719086 --- /dev/null +++ b/0007-add-CRI-ContainerStats-Service.patch @@ -0,0 +1,490 @@ +From 9bd02c394110180ac7d7cbe80c1f4abe18146ebb Mon Sep 17 00:00:00 2001 +From: zhangxiaoyu +Date: Fri, 10 Feb 2023 17:43:11 +0800 +Subject: [PATCH 07/53] add CRI ContainerStats Service + +Signed-off-by: zhangxiaoyu +--- + .../connect/grpc/runtime_image_service.cc | 12 ++--- + .../connect/grpc/runtime_runtime_service.cc | 45 +++++++++++----- + .../connect/grpc/runtime_runtime_service.h | 3 ++ + .../cri/cri_container_manager_service.cc | 54 +++++++++++++++++++ + .../entry/cri/cri_container_manager_service.h | 3 ++ + src/daemon/entry/cri/cri_runtime_service.h | 3 ++ + .../entry/cri/cri_runtime_service_impl.cc | 6 +++ + .../entry/cri/cri_runtime_service_impl.h | 3 ++ + src/daemon/modules/events/collector.c | 4 +- + src/daemon/modules/image/image.c | 16 +++--- + .../modules/image/oci/oci_common_operators.c | 8 +-- + 11 files changed, 125 insertions(+), 32 deletions(-) + +diff --git a/src/daemon/entry/connect/grpc/runtime_image_service.cc b/src/daemon/entry/connect/grpc/runtime_image_service.cc +index 23447baf..e593a9c6 100644 +--- a/src/daemon/entry/connect/grpc/runtime_image_service.cc ++++ b/src/daemon/entry/connect/grpc/runtime_image_service.cc +@@ -54,7 +54,7 @@ grpc::Status RuntimeImageServiceImpl::ListImages(grpc::ServerContext *context, + std::vector> images; + Errors error; + +- WARN("Event: {Object: CRI, Type: Listing all images}"); ++ INFO("Event: {Object: CRI, Type: Listing all images}"); + + rService->ListImages(request->filter(), &images, error); + if (!error.Empty()) { +@@ -70,7 +70,7 @@ grpc::Status RuntimeImageServiceImpl::ListImages(grpc::ServerContext *context, + *image = *(iter->get()); + } + +- WARN("Event: {Object: CRI, Type: Listed all images}"); ++ INFO("Event: {Object: CRI, Type: Listed all images}"); + + return grpc::Status::OK; + } +@@ -82,7 +82,7 @@ grpc::Status RuntimeImageServiceImpl::ImageStatus(grpc::ServerContext *context, + std::unique_ptr image_info = nullptr; + Errors error; + +- WARN("Event: {Object: CRI, Type: Statusing image %s}", request->image().image().c_str()); ++ INFO("Event: {Object: CRI, Type: Statusing image %s}", request->image().image().c_str()); + + image_info = rService->ImageStatus(request->image(), error); + if (!error.Empty() && !CRIHelpers::IsImageNotFoundError(error.GetMessage())) { +@@ -96,7 +96,7 @@ grpc::Status RuntimeImageServiceImpl::ImageStatus(grpc::ServerContext *context, + *image = *image_info; + } + +- WARN("Event: {Object: CRI, Type: Statused image %s}", request->image().image().c_str()); ++ INFO("Event: {Object: CRI, Type: Statused image %s}", request->image().image().c_str()); + + return grpc::Status::OK; + } +@@ -108,7 +108,7 @@ grpc::Status RuntimeImageServiceImpl::ImageFsInfo(grpc::ServerContext *context, + std::vector> usages; + Errors error; + +- WARN("Event: {Object: CRI, Type: Statusing image fs info}"); ++ INFO("Event: {Object: CRI, Type: Statusing image fs info}"); + + rService->ImageFsInfo(&usages, error); + if (!error.Empty()) { +@@ -125,7 +125,7 @@ grpc::Status RuntimeImageServiceImpl::ImageFsInfo(grpc::ServerContext *context, + *fs_info = *(iter->get()); + } + +- WARN("Event: {Object: CRI, Type: Statused image fs info}"); ++ INFO("Event: {Object: CRI, Type: Statused image fs info}"); + return grpc::Status::OK; + } + +diff --git a/src/daemon/entry/connect/grpc/runtime_runtime_service.cc b/src/daemon/entry/connect/grpc/runtime_runtime_service.cc +index b6f9e751..8fed162b 100644 +--- a/src/daemon/entry/connect/grpc/runtime_runtime_service.cc ++++ b/src/daemon/entry/connect/grpc/runtime_runtime_service.cc +@@ -171,7 +171,7 @@ grpc::Status RuntimeRuntimeServiceImpl::ListContainers(grpc::ServerContext *cont + { + Errors error; + +- WARN("Event: {Object: CRI, Type: Listing all Container}"); ++ INFO("Event: {Object: CRI, Type: Listing all Container}"); + + std::vector> containers; + m_rService->ListContainers(request->has_filter() ? &request->filter() : nullptr, &containers, error); +@@ -189,7 +189,28 @@ grpc::Status RuntimeRuntimeServiceImpl::ListContainers(grpc::ServerContext *cont + *container = *(iter->get()); + } + +- WARN("Event: {Object: CRI, Type: Listed all Container}"); ++ INFO("Event: {Object: CRI, Type: Listed all Container}"); ++ ++ return grpc::Status::OK; ++} ++ ++grpc::Status RuntimeRuntimeServiceImpl::ContainerStats(grpc::ServerContext *context, ++ const runtime::v1alpha2::ContainerStatsRequest *request, ++ runtime::v1alpha2::ContainerStatsResponse *reply) ++{ ++ Errors error; ++ ++ INFO("Event: {Object: CRI, Type: Getting Container Stats: %s}", request->container_id().c_str()); ++ ++ std::unique_ptr contStats = ++ m_rService->ContainerStats(request->container_id(), error); ++ if (!error.Empty() || !contStats) { ++ ERROR("Object: CRI, Type: Failed to get container stats %s", request->container_id().c_str()); ++ return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ } ++ *(reply->mutable_stats()) = *contStats; ++ ++ INFO("Event: {Object: CRI, Type: Got Container stats: %s}", request->container_id().c_str()); + + return grpc::Status::OK; + } +@@ -200,7 +221,7 @@ grpc::Status RuntimeRuntimeServiceImpl::ListContainerStats(grpc::ServerContext * + { + Errors error; + +- WARN("Event: {Object: CRI, Type: Listing all Container stats}"); ++ INFO("Event: {Object: CRI, Type: Listing all Container stats}"); + + std::vector> containers; + m_rService->ListContainerStats(request->has_filter() ? &request->filter() : nullptr, &containers, error); +@@ -218,7 +239,7 @@ grpc::Status RuntimeRuntimeServiceImpl::ListContainerStats(grpc::ServerContext * + *container = *(iter->get()); + } + +- WARN("Event: {Object: CRI, Type: Listed all Container stats}"); ++ INFO("Event: {Object: CRI, Type: Listed all Container stats}"); + + return grpc::Status::OK; + } +@@ -229,7 +250,7 @@ grpc::Status RuntimeRuntimeServiceImpl::ContainerStatus(grpc::ServerContext *con + { + Errors error; + +- WARN("Event: {Object: CRI, Type: Statusing Container: %s}", request->container_id().c_str()); ++ INFO("Event: {Object: CRI, Type: Statusing Container: %s}", request->container_id().c_str()); + + std::unique_ptr contStatus = + m_rService->ContainerStatus(request->container_id(), error); +@@ -239,7 +260,7 @@ grpc::Status RuntimeRuntimeServiceImpl::ContainerStatus(grpc::ServerContext *con + } + *(reply->mutable_status()) = *contStatus; + +- WARN("Event: {Object: CRI, Type: Statused Container: %s}", request->container_id().c_str()); ++ INFO("Event: {Object: CRI, Type: Statused Container: %s}", request->container_id().c_str()); + + return grpc::Status::OK; + } +@@ -329,7 +350,7 @@ grpc::Status RuntimeRuntimeServiceImpl::PodSandboxStatus(grpc::ServerContext *co + { + Errors error; + +- WARN("Event: {Object: CRI, Type: Status Pod: %s}", request->pod_sandbox_id().c_str()); ++ INFO("Event: {Object: CRI, Type: Status Pod: %s}", request->pod_sandbox_id().c_str()); + + std::unique_ptr podStatus; + podStatus = m_rService->PodSandboxStatus(request->pod_sandbox_id(), error); +@@ -340,7 +361,7 @@ grpc::Status RuntimeRuntimeServiceImpl::PodSandboxStatus(grpc::ServerContext *co + } + *(reply->mutable_status()) = *podStatus; + +- WARN("Event: {Object: CRI, Type: Statused Pod: %s}", request->pod_sandbox_id().c_str()); ++ INFO("Event: {Object: CRI, Type: Statused Pod: %s}", request->pod_sandbox_id().c_str()); + + return grpc::Status::OK; + } +@@ -351,7 +372,7 @@ grpc::Status RuntimeRuntimeServiceImpl::ListPodSandbox(grpc::ServerContext *cont + { + Errors error; + +- WARN("Event: {Object: CRI, Type: Listing all Pods}"); ++ INFO("Event: {Object: CRI, Type: Listing all Pods}"); + + std::vector> pods; + m_rService->ListPodSandbox(request->has_filter() ? &request->filter() : nullptr, &pods, error); +@@ -368,7 +389,7 @@ grpc::Status RuntimeRuntimeServiceImpl::ListPodSandbox(grpc::ServerContext *cont + *pod = *(iter->get()); + } + +- WARN("Event: {Object: CRI, Type: Listed all Pods}"); ++ INFO("Event: {Object: CRI, Type: Listed all Pods}"); + + return grpc::Status::OK; + } +@@ -460,7 +481,7 @@ grpc::Status RuntimeRuntimeServiceImpl::Status(grpc::ServerContext *context, + { + Errors error; + +- WARN("Event: {Object: CRI, Type: Statusing daemon}"); ++ INFO("Event: {Object: CRI, Type: Statusing daemon}"); + + std::unique_ptr status = m_rService->Status(error); + if (status == nullptr || error.NotEmpty()) { +@@ -469,7 +490,7 @@ grpc::Status RuntimeRuntimeServiceImpl::Status(grpc::ServerContext *context, + } + *(reply->mutable_status()) = *status; + +- WARN("Event: {Object: CRI, Type: Statused daemon}"); ++ INFO("Event: {Object: CRI, Type: Statused daemon}"); + + return grpc::Status::OK; + } +diff --git a/src/daemon/entry/connect/grpc/runtime_runtime_service.h b/src/daemon/entry/connect/grpc/runtime_runtime_service.h +index 6e8c1009..cb3c5425 100644 +--- a/src/daemon/entry/connect/grpc/runtime_runtime_service.h ++++ b/src/daemon/entry/connect/grpc/runtime_runtime_service.h +@@ -52,6 +52,9 @@ public: + const runtime::v1alpha2::ListContainerStatsRequest *request, + runtime::v1alpha2::ListContainerStatsResponse *reply) override; + ++ grpc::Status ContainerStats(grpc::ServerContext *context, const runtime::v1alpha2::ContainerStatsRequest *request, ++ runtime::v1alpha2::ContainerStatsResponse *reply) override; ++ + grpc::Status ContainerStatus(grpc::ServerContext *context, const runtime::v1alpha2::ContainerStatusRequest *request, + runtime::v1alpha2::ContainerStatusResponse *reply) override; + +diff --git a/src/daemon/entry/cri/cri_container_manager_service.cc b/src/daemon/entry/cri/cri_container_manager_service.cc +index d044cca8..710556a3 100644 +--- a/src/daemon/entry/cri/cri_container_manager_service.cc ++++ b/src/daemon/entry/cri/cri_container_manager_service.cc +@@ -893,6 +893,60 @@ cleanup: + free_container_stats_response(response); + } + ++auto ContainerManagerService::ContainerStats(const std::string &containerID, Errors &error) ++-> std::unique_ptr ++{ ++ container_stats_request *request { nullptr }; ++ container_stats_response *response { nullptr }; ++ std::unique_ptr contStats { nullptr }; ++ std::vector> contStatsVec; ++ ++ if (containerID.empty()) { ++ error.SetError("Empty container id"); ++ return nullptr; ++ } ++ ++ if (m_cb == nullptr || m_cb->container.stats == nullptr) { ++ error.SetError("Unimplemented callback"); ++ return nullptr; ++ } ++ ++ request = (container_stats_request *)util_common_calloc_s(sizeof(container_stats_request)); ++ if (request == nullptr) { ++ error.SetError("Out of memory"); ++ return nullptr; ++ } ++ ++ request->containers = (char **)util_smart_calloc_s(sizeof(char *), 1); ++ if (request->containers == nullptr) { ++ error.SetError("Out of memory"); ++ goto cleanup; ++ } ++ ++ request->containers[0] = util_strdup_s(containerID.c_str()); ++ request->containers_len = 1; ++ ++ if (m_cb->container.stats(request, &response) != 0) { ++ if (response != nullptr && response->errmsg != nullptr) { ++ error.SetError(response->errmsg); ++ } else { ++ error.SetError("Failed to call stats container callback"); ++ } ++ goto cleanup; ++ } ++ ++ ContainerStatsToGRPC(response, &contStatsVec, error); ++ if (error.NotEmpty()) { ++ goto cleanup; ++ } ++ contStats = std::move(contStatsVec[0]); ++ ++cleanup: ++ free_container_stats_request(request); ++ free_container_stats_response(response); ++ return contStats; ++} ++ + void ContainerManagerService::PackContainerImageToStatus( + container_inspect *inspect, std::unique_ptr &contStatus, Errors &error) + { +diff --git a/src/daemon/entry/cri/cri_container_manager_service.h b/src/daemon/entry/cri/cri_container_manager_service.h +index 8002b77d..6ec1f21d 100644 +--- a/src/daemon/entry/cri/cri_container_manager_service.h ++++ b/src/daemon/entry/cri/cri_container_manager_service.h +@@ -53,6 +53,9 @@ public: + std::vector> *containerstats, + Errors &error); + ++ auto ContainerStats(const std::string &containerID, Errors &error) ++ -> std::unique_ptr; ++ + auto ContainerStatus(const std::string &containerID, Errors &error) + -> std::unique_ptr; + +diff --git a/src/daemon/entry/cri/cri_runtime_service.h b/src/daemon/entry/cri/cri_runtime_service.h +index 4727230f..5e4740cb 100644 +--- a/src/daemon/entry/cri/cri_runtime_service.h ++++ b/src/daemon/entry/cri/cri_runtime_service.h +@@ -49,6 +49,9 @@ public: + std::vector> *containerstats, + Errors &error) = 0; + ++ virtual auto ContainerStats(const std::string &containerID, ++ Errors &error) -> std::unique_ptr = 0; ++ + virtual auto ContainerStatus(const std::string &containerID, + Errors &error) -> std::unique_ptr = 0; + +diff --git a/src/daemon/entry/cri/cri_runtime_service_impl.cc b/src/daemon/entry/cri/cri_runtime_service_impl.cc +index 241d4316..c4b84828 100644 +--- a/src/daemon/entry/cri/cri_runtime_service_impl.cc ++++ b/src/daemon/entry/cri/cri_runtime_service_impl.cc +@@ -70,6 +70,12 @@ void CRIRuntimeServiceImpl::ListContainerStats( + m_containerManager->ListContainerStats(filter, containerstats, error); + } + ++auto CRIRuntimeServiceImpl::ContainerStats(const std::string &containerID, Errors &error) ++-> std::unique_ptr ++{ ++ return m_containerManager->ContainerStats(containerID, error); ++} ++ + auto CRIRuntimeServiceImpl::ContainerStatus(const std::string &containerID, Errors &error) + -> std::unique_ptr + { +diff --git a/src/daemon/entry/cri/cri_runtime_service_impl.h b/src/daemon/entry/cri/cri_runtime_service_impl.h +index f2e25e42..7355c5cb 100644 +--- a/src/daemon/entry/cri/cri_runtime_service_impl.h ++++ b/src/daemon/entry/cri/cri_runtime_service_impl.h +@@ -51,6 +51,9 @@ public: + std::vector> *containerstats, + Errors &error) override; + ++ auto ContainerStats(const std::string &containerID, Errors &error) ++ -> std::unique_ptr override; ++ + auto ContainerStatus(const std::string &containerID, Errors &error) + -> std::unique_ptr override; + +diff --git a/src/daemon/modules/events/collector.c b/src/daemon/modules/events/collector.c +index fb4e4a98..f53ad319 100644 +--- a/src/daemon/modules/events/collector.c ++++ b/src/daemon/modules/events/collector.c +@@ -484,7 +484,7 @@ static int write_events_log(const struct isulad_events_format *events) + + len = calculate_annaotation_info_len(events); + if (len == 1) { +- WARN("Event: {Object: %s, Type: %s}", events->id, events->opt); ++ INFO("Event: {Object: %s, Type: %s}", events->id, events->opt); + } else { + annotation = (char *)util_common_calloc_s(len); + if (annotation == NULL) { +@@ -502,7 +502,7 @@ static int write_events_log(const struct isulad_events_format *events) + } + (void)strcat(annotation, ")"); + +- WARN("Event: {Object: %s, Type: %s %s}", events->id, events->opt, annotation); ++ INFO("Event: {Object: %s, Type: %s %s}", events->id, events->opt, annotation); + } + + out: +diff --git a/src/daemon/modules/image/image.c b/src/daemon/modules/image/image.c +index d5fbc8dc..a9b9523e 100644 +--- a/src/daemon/modules/image/image.c ++++ b/src/daemon/modules/image/image.c +@@ -416,7 +416,7 @@ int im_get_filesystem_info(const char *image_type, im_fs_info_response **respons + goto out; + } + +- WARN("Event: {Object: get image filesystem info, Type: inspecting}"); ++ INFO("Event: {Object: get image filesystem info, Type: inspecting}"); + ret = q->ops->get_filesystem_info(response); + if (ret != 0) { + if (response != NULL && *response != NULL) { +@@ -426,7 +426,7 @@ int im_get_filesystem_info(const char *image_type, im_fs_info_response **respons + } + goto out; + } +- WARN("Event: {Object: get image filesystem info, Type: inspected}"); ++ INFO("Event: {Object: get image filesystem info, Type: inspected}"); + + out: + return ret; +@@ -466,7 +466,7 @@ int im_get_container_filesystem_usage(const char *image_type, const char *id, im + request->name_id = util_strdup_s(id); + } + +- WARN("Event: {Object: container \'%s\' filesystem info, Type: inspecting}", id != NULL ? id : ""); ++ INFO("Event: {Object: container \'%s\' filesystem info, Type: inspecting}", id != NULL ? id : ""); + ret = q->ops->container_fs_usage(request, &filesystemusage); + if (ret != 0) { + ERROR("Failed to get filesystem usage for container %s", id); +@@ -476,7 +476,7 @@ int im_get_container_filesystem_usage(const char *image_type, const char *id, im + + *fs_usage = filesystemusage; + filesystemusage = NULL; +- WARN("Event: {Object: container \'%s\' filesystem info, Type: inspected}", id != NULL ? id : ""); ++ INFO("Event: {Object: container \'%s\' filesystem info, Type: inspected}", id != NULL ? id : ""); + + out: + free_im_container_fs_usage_request(request); +@@ -916,7 +916,7 @@ int im_list_images(const im_list_request *ctx, im_list_response **response) + return -1; + } + +- WARN("Event: {Object: list images, Type: listing}"); ++ INFO("Event: {Object: list images, Type: listing}"); + + for (i = 0; i < g_numbims; i++) { + if (g_bims[i].ops->list_ims == NULL) { +@@ -936,7 +936,7 @@ int im_list_images(const im_list_request *ctx, im_list_response **response) + images_tmp = NULL; + } + +- WARN("Event: {Object: list images, Type: listed}"); ++ INFO("Event: {Object: list images, Type: listed}"); + + if (g_isulad_errmsg != NULL) { + (*response)->errmsg = util_strdup_s(g_isulad_errmsg); +@@ -1671,7 +1671,7 @@ int im_inspect_image(const im_inspect_request *request, im_inspect_response **re + + image_ref = util_strdup_s(request->image.image); + +- WARN("Event: {Object: %s, Type: image inspecting}", image_ref); ++ INFO("Event: {Object: %s, Type: image inspecting}", image_ref); + + bim_type = bim_query(image_ref); + if (bim_type == NULL) { +@@ -1693,7 +1693,7 @@ int im_inspect_image(const im_inspect_request *request, im_inspect_response **re + goto pack_response; + } + +- WARN("Event: {Object: %s, Type: image inspected}", image_ref); ++ INFO("Event: {Object: %s, Type: image inspected}", image_ref); + + pack_response: + if (g_isulad_errmsg != NULL) { +diff --git a/src/daemon/modules/image/oci/oci_common_operators.c b/src/daemon/modules/image/oci/oci_common_operators.c +index 3d200e09..83cccbe6 100644 +--- a/src/daemon/modules/image/oci/oci_common_operators.c ++++ b/src/daemon/modules/image/oci/oci_common_operators.c +@@ -429,7 +429,7 @@ int oci_summary_image(im_summary_request *request, im_summary_response *response + goto pack_response; + } + +- WARN("Event: {Object: %s, Type: statusing image summary}", resolved_name); ++ INFO("Event: {Object: %s, Type: statusing image summary}", resolved_name); + + image_summary = storage_img_get_summary(resolved_name); + if (image_summary == NULL) { +@@ -442,7 +442,7 @@ int oci_summary_image(im_summary_request *request, im_summary_response *response + response->image_summary = image_summary; + image_summary = NULL; + +- WARN("Event: {Object: %s, Type: statused image summary}", resolved_name); ++ INFO("Event: {Object: %s, Type: statused image summary}", resolved_name); + + pack_response: + free(resolved_name); +@@ -493,7 +493,7 @@ int oci_status_image(im_status_request *request, im_status_response *response) + goto pack_response; + } + +- WARN("Event: {Object: %s, Type: statusing image}", resolved_name); ++ INFO("Event: {Object: %s, Type: statusing image}", resolved_name); + + image_info = storage_img_get(resolved_name); + if (image_info == NULL) { +@@ -506,7 +506,7 @@ int oci_status_image(im_status_request *request, im_status_response *response) + response->image_info->image = image_info; + image_info = NULL; + +- WARN("Event: {Object: %s, Type: statused image}", resolved_name); ++ INFO("Event: {Object: %s, Type: statused image}", resolved_name); + + pack_response: + free(resolved_name); +-- +2.25.1 + diff --git a/0008-fix-isula-cpu-rt-CI.patch b/0008-fix-isula-cpu-rt-CI.patch new file mode 100644 index 0000000..3f2566f --- /dev/null +++ b/0008-fix-isula-cpu-rt-CI.patch @@ -0,0 +1,183 @@ +From edef459d5052dc6d7c29e8a7a48ff4bf1b01bd78 Mon Sep 17 00:00:00 2001 +From: songbuhuang <544824346@qq.com> +Date: Tue, 14 Feb 2023 14:08:01 +0800 +Subject: [PATCH 08/53] fix isula cpu-rt CI + +Signed-off-by: songbuhuang <544824346@qq.com> +--- + CI/test_cases/container_cases/cpu_rt.sh | 102 +++++++++++++++++------- + 1 file changed, 73 insertions(+), 29 deletions(-) + +diff --git a/CI/test_cases/container_cases/cpu_rt.sh b/CI/test_cases/container_cases/cpu_rt.sh +index 3d70c840..353c2d71 100755 +--- a/CI/test_cases/container_cases/cpu_rt.sh ++++ b/CI/test_cases/container_cases/cpu_rt.sh +@@ -21,12 +21,12 @@ + declare -r curr_path=$(dirname $(readlink -f "$0")) + source ../helpers.sh + +-function test_cpu_rt_isulad_spec() ++function test_cpurt_isulad_abnormal() + { + local ret=0 + local test="isulad cpu realtime test => (${FUNCNAME[@]})" + +- msg_info "${test} starting..." ++ msg_info "${test} starting..." + + isulad --cpu-rt-period xx --cpu-rt-runtime 950000 /bin/sh 2>&1 | grep 'Invalid value "xx" for flag --cpu-rt-period: Invalid argument' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Invalid argument for cpu-rt-period" && ((ret++)) +@@ -38,54 +38,64 @@ function test_cpu_rt_isulad_spec() + return ${ret} + } + +-function test_cpu_rt_isula_spec() ++function test_isula_update_normal() + { +- local ret=0 +- local image="busybox" +- local test="container cpu realtime test => (${FUNCNAME[@]})" +- +- msg_info "${test} starting..." +- +- #start isulad without cpu_rt +- start_isulad_without_valgrind ++ #start isulad with cpu_rt ++ isulad --cpu-rt-period 1000000 --cpu-rt-runtime 950000 -l DEBUG > /dev/null 2>&1 & ++ wait_isulad_running ++ ++ c_id=`isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 950000 ${image} sh` ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++)) + +- isula pull ${image} +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to pull image: ${image}" && return ${FAILURE} ++ isula update --cpu-rt-period 900000 --cpu-rt-runtime 800000 $c_id ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to update container cpu-rt-runtime" && ((ret++)) + +- isula images | grep busybox +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${image}" && ((ret++)) ++ isula exec -it $c_id sh -c "cat /sys/fs/cgroup/cpu/cpu.rt_runtime_us" | grep "800000" ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container cpu.rt_runtime_us: 800000" && ((ret++)) + +- test_isula_run_spec ++ isula exec -it $c_id sh -c "cat /sys/fs/cgroup/cpu/cpu.rt_period_us" | grep "900000" ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container cpu.rt_period_us: 900000" && ((ret++)) + +- #start isulad without cpu_rt:isulad cpu.rt_period_us default value is the cpu.rt_period_us of the upper-layer directory,cpu.rt_runtime_us is 0. +- isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 10000 $image /bin/sh 2>&1 | grep "failed to write 10000" | grep "cpu.rt_runtime_us: Invalid argument" +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Invalid argument for cpu-rt-runtime" && ((ret++)) ++ isula rm -f $c_id ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to rm container ${c_id}" && ((ret++)) + + stop_isulad_without_valgrind ++ #set cpu-rt to the initial state ++ isulad --cpu-rt-period 1000000 --cpu-rt-runtime 0 -l DEBUG > /dev/null 2>&1 & ++ wait_isulad_running + ++ msg_info "${test} finished with return ${ret}..." ++ return ${ret} ++} ++ ++function test_isula_update_abnormal() ++{ + #start isulad with cpu_rt + isulad --cpu-rt-period 1000000 --cpu-rt-runtime 950000 -l DEBUG > /dev/null 2>&1 & + wait_isulad_running +- +- test_isula_run_spec + + c_id=`isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 950000 ${image} sh` + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++)) + +- isula update --cpu-rt-runtime 90000 $c_id ++ isula update --cpu-rt-period 800000 --cpu-rt-runtime 900000 $c_id | grep "Invalid --cpu-rt-runtime: rt runtime cannot be higher than rt period" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to update container cpu-rt-runtime" && ((ret++)) + +- isula exec -it $c_id sh -c "cat /sys/fs/cgroup/cpu/cpu.rt_runtime_us" | grep "90000" +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container cpu.rt_runtime_us: 90000" && ((ret++)) ++ isula update --cpu-rt-runtime 1000000 $c_id | grep "updating cgroup cpu.rt_runtime_us to 1000000: Invalid argument" ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to update container cpu-rt-runtime" && ((ret++)) + + isula rm -f $c_id + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to rm container ${c_id}" && ((ret++)) + ++ stop_isulad_without_valgrind ++ #set cpu-rt to the initial state ++ isulad --cpu-rt-period 1000000 --cpu-rt-runtime 0 -l DEBUG > /dev/null 2>&1 & ++ wait_isulad_running ++ + msg_info "${test} finished with return ${ret}..." + return ${ret} + } + +-function test_kernel_without_cpu_rt_spec() ++function test_kernel_without_cpurt() + { + local ret=0 + local image="busybox" +@@ -109,8 +119,24 @@ function test_kernel_without_cpu_rt_spec() + return ${ret} + } + +-function test_isula_run_spec() ++function test_isula_run_abnormal() + { ++ local ret=0 ++ local image="busybox" ++ local test="container cpu realtime test => (${FUNCNAME[@]})" ++ ++ msg_info "${test} starting..." ++ ++ #start isulad without cpu_rt ++ isulad --cpu-rt-period 1000000 --cpu-rt-runtime 950000 -l DEBUG > /dev/null 2>&1 & ++ wait_isulad_running ++ ++ isula pull ${image} ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to pull image: ${image}" && return ${FAILURE} ++ ++ isula images | grep busybox ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${image}" && ((ret++)) ++ + isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime -1 $image /bin/sh 2>&1 | grep "failed to write -1" | grep "cpu.rt_runtime_us: Invalid argument" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Invalid argument for cpu-rt-runtime" && ((ret++)) + +@@ -128,15 +154,33 @@ function test_isula_run_spec() + + isula run -itd --cpu-rt-period 100 --cpu-rt-runtime 10000 $image /bin/sh 2>&1 | grep "Invalid --cpu-rt-runtime: rt runtime cannot be higher than rt period" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cpu-rt-runtime cannot be higher than cpu-rt-period" && ((ret++)) ++ ++ isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 960000 $image /bin/sh 2>&1 | grep "failed to write 960000" | grep "cpu.rt_runtime_us: Invalid argument" ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Invalid argument for cpu-rt-runtime" && ((ret++)) ++ ++ stop_isulad_without_valgrind ++} ++ ++function test_isula_run_normal() ++{ ++ isula run -itd -n box --cpu-rt-period 1000000 --cpu-rt-runtime 900000 $image /bin/sh 2>&1 ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container" && ((ret++)) ++ ++ isula rm -f box ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to rm container ${c_id}" && ((ret++)) ++ + } + + declare -i ans=0 + + if [ -f "/sys/fs/cgroup/cpu/cpu.rt_runtime_us" ];then +- test_cpu_rt_isulad_spec || ((ans++)) +- test_cpu_rt_isula_spec || ((ans++)) ++ test_isula_run_abnormal || ((ans++)) ++ test_isula_run_normal || ((ans++)) ++ test_cpurt_isulad_abnormal || ((ans++)) ++ test_isula_update_normal || ((ans++)) ++ test_isula_update_abnormal || ((ans++)) + else +- test_kernel_without_cpu_rt_spec || ((ans++)) ++ test_kernel_without_cpurt || ((ans++)) + fi + + show_result ${ans} "${curr_path}/${0}" +-- +2.25.1 + diff --git a/0009-fix-cpu-rt-CI.patch b/0009-fix-cpu-rt-CI.patch new file mode 100644 index 0000000..8e0fa06 --- /dev/null +++ b/0009-fix-cpu-rt-CI.patch @@ -0,0 +1,105 @@ +From 0504a907def3efb4c0ad7eabd5921c97090430af Mon Sep 17 00:00:00 2001 +From: songbuhuang <544824346@qq.com> +Date: Tue, 14 Feb 2023 15:55:56 +0800 +Subject: [PATCH 09/53] fix cpu-rt CI + +Signed-off-by: songbuhuang <544824346@qq.com> +--- + CI/test_cases/container_cases/cpu_rt.sh | 32 +++++++++++++++++++------ + 1 file changed, 25 insertions(+), 7 deletions(-) + +diff --git a/CI/test_cases/container_cases/cpu_rt.sh b/CI/test_cases/container_cases/cpu_rt.sh +index 353c2d71..42006bc8 100755 +--- a/CI/test_cases/container_cases/cpu_rt.sh ++++ b/CI/test_cases/container_cases/cpu_rt.sh +@@ -24,14 +24,14 @@ source ../helpers.sh + function test_cpurt_isulad_abnormal() + { + local ret=0 +- local test="isulad cpu realtime test => (${FUNCNAME[@]})" ++ local test="isulad cpu realtime abnormal test => (${FUNCNAME[@]})" + + msg_info "${test} starting..." + +- isulad --cpu-rt-period xx --cpu-rt-runtime 950000 /bin/sh 2>&1 | grep 'Invalid value "xx" for flag --cpu-rt-period: Invalid argument' ++ isulad --cpu-rt-period xx --cpu-rt-runtime 950000 2>&1 | grep 'Invalid value "xx" for flag --cpu-rt-period: Invalid argument' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Invalid argument for cpu-rt-period" && ((ret++)) + +- isulad --cpu-rt-period 1000000 --cpu-rt-runtime xx /bin/sh 2>&1 | grep 'Invalid value "xx" for flag --cpu-rt-runtime: Invalid argument' ++ isulad --cpu-rt-period 1000000 --cpu-rt-runtime xx 2>&1 | grep 'Invalid value "xx" for flag --cpu-rt-runtime: Invalid argument' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Invalid argument for cpu-rt-runtime" && ((ret++)) + + msg_info "${test} finished with return ${ret}..." +@@ -40,6 +40,12 @@ function test_cpurt_isulad_abnormal() + + function test_isula_update_normal() + { ++ local ret=0 ++ local image="busybox" ++ local test="isulad update cpu realtime normal test => (${FUNCNAME[@]})" ++ ++ msg_info "${test} starting..." ++ + #start isulad with cpu_rt + isulad --cpu-rt-period 1000000 --cpu-rt-runtime 950000 -l DEBUG > /dev/null 2>&1 & + wait_isulad_running +@@ -70,6 +76,10 @@ function test_isula_update_normal() + + function test_isula_update_abnormal() + { ++ local ret=0 ++ local image="busybox" ++ local test="isulad update cpu realtime abnormal test => (${FUNCNAME[@]})" ++ + #start isulad with cpu_rt + isulad --cpu-rt-period 1000000 --cpu-rt-runtime 950000 -l DEBUG > /dev/null 2>&1 & + wait_isulad_running +@@ -77,10 +87,10 @@ function test_isula_update_abnormal() + c_id=`isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 950000 ${image} sh` + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++)) + +- isula update --cpu-rt-period 800000 --cpu-rt-runtime 900000 $c_id | grep "Invalid --cpu-rt-runtime: rt runtime cannot be higher than rt period" ++ isula update --cpu-rt-period 800000 --cpu-rt-runtime 900000 $c_id 2>&1 | grep "Invalid --cpu-rt-runtime: rt runtime cannot be higher than rt period" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to update container cpu-rt-runtime" && ((ret++)) + +- isula update --cpu-rt-runtime 1000000 $c_id | grep "updating cgroup cpu.rt_runtime_us to 1000000: Invalid argument" ++ isula update --cpu-rt-runtime 1000000 $c_id 2>&1 | grep "updating cgroup cpu.rt_runtime_us to 1000000: Invalid argument" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to update container cpu-rt-runtime" && ((ret++)) + + isula rm -f $c_id +@@ -158,17 +168,23 @@ function test_isula_run_abnormal() + isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 960000 $image /bin/sh 2>&1 | grep "failed to write 960000" | grep "cpu.rt_runtime_us: Invalid argument" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Invalid argument for cpu-rt-runtime" && ((ret++)) + +- stop_isulad_without_valgrind ++ msg_info "${test} finished with return ${ret}..." ++ return ${ret} + } + + function test_isula_run_normal() + { ++ local ret=0 ++ local image="busybox" ++ + isula run -itd -n box --cpu-rt-period 1000000 --cpu-rt-runtime 900000 $image /bin/sh 2>&1 + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container" && ((ret++)) + + isula rm -f box + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to rm container ${c_id}" && ((ret++)) +- ++ ++ msg_info "${test} finished with return ${ret}..." ++ return ${ret} + } + + declare -i ans=0 +@@ -183,4 +199,6 @@ else + test_kernel_without_cpurt || ((ans++)) + fi + ++isula rm -f $(isula ps -aq) ++ + show_result ${ans} "${curr_path}/${0}" +-- +2.25.1 + diff --git a/0010-fix-cpu-rt-CI.patch b/0010-fix-cpu-rt-CI.patch new file mode 100644 index 0000000..84d7604 --- /dev/null +++ b/0010-fix-cpu-rt-CI.patch @@ -0,0 +1,54 @@ +From 21dc648ef93cd0fb858a408bc843d25a5e20e320 Mon Sep 17 00:00:00 2001 +From: songbuhuang <544824346@qq.com> +Date: Wed, 15 Feb 2023 16:09:38 +0800 +Subject: [PATCH 10/53] fix cpu-rt CI + +Signed-off-by: songbuhuang <544824346@qq.com> +--- + CI/test_cases/container_cases/cpu_rt.sh | 12 ++++++------ + 1 file changed, 6 insertions(+), 6 deletions(-) + +diff --git a/CI/test_cases/container_cases/cpu_rt.sh b/CI/test_cases/container_cases/cpu_rt.sh +index 42006bc8..39c0b427 100755 +--- a/CI/test_cases/container_cases/cpu_rt.sh ++++ b/CI/test_cases/container_cases/cpu_rt.sh +@@ -50,14 +50,14 @@ function test_isula_update_normal() + isulad --cpu-rt-period 1000000 --cpu-rt-runtime 950000 -l DEBUG > /dev/null 2>&1 & + wait_isulad_running + +- c_id=`isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 950000 ${image} sh` ++ c_id=`isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 1000 ${image} sh` + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++)) + +- isula update --cpu-rt-period 900000 --cpu-rt-runtime 800000 $c_id ++ isula update --cpu-rt-period 900000 --cpu-rt-runtime 2000 $c_id + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to update container cpu-rt-runtime" && ((ret++)) + +- isula exec -it $c_id sh -c "cat /sys/fs/cgroup/cpu/cpu.rt_runtime_us" | grep "800000" +- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container cpu.rt_runtime_us: 800000" && ((ret++)) ++ isula exec -it $c_id sh -c "cat /sys/fs/cgroup/cpu/cpu.rt_runtime_us" | grep "2000" ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container cpu.rt_runtime_us: 2000" && ((ret++)) + + isula exec -it $c_id sh -c "cat /sys/fs/cgroup/cpu/cpu.rt_period_us" | grep "900000" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container cpu.rt_period_us: 900000" && ((ret++)) +@@ -84,7 +84,7 @@ function test_isula_update_abnormal() + isulad --cpu-rt-period 1000000 --cpu-rt-runtime 950000 -l DEBUG > /dev/null 2>&1 & + wait_isulad_running + +- c_id=`isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 950000 ${image} sh` ++ c_id=`isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 1000 ${image} sh` + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++)) + + isula update --cpu-rt-period 800000 --cpu-rt-runtime 900000 $c_id 2>&1 | grep "Invalid --cpu-rt-runtime: rt runtime cannot be higher than rt period" +@@ -177,7 +177,7 @@ function test_isula_run_normal() + local ret=0 + local image="busybox" + +- isula run -itd -n box --cpu-rt-period 1000000 --cpu-rt-runtime 900000 $image /bin/sh 2>&1 ++ isula run -itd -n box --cpu-rt-period 1000000 --cpu-rt-runtime 1000 $image /bin/sh 2>&1 + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container" && ((ret++)) + + isula rm -f box +-- +2.25.1 + diff --git a/0011-Bugfix-in-config-and-executor.patch b/0011-Bugfix-in-config-and-executor.patch new file mode 100644 index 0000000..e038afd --- /dev/null +++ b/0011-Bugfix-in-config-and-executor.patch @@ -0,0 +1,52 @@ +From 933eceb4545a28dba44c72f183dc7104d0fea714 Mon Sep 17 00:00:00 2001 +From: Xuepeng Xu +Date: Wed, 15 Feb 2023 12:19:40 +0800 +Subject: [PATCH 11/53] Bugfix in config and executor + +Signed-off-by: Xuepeng Xu +--- + src/daemon/config/isulad_config.c | 2 +- + src/daemon/executor/container_cb/execution_create.c | 6 +++--- + 2 files changed, 4 insertions(+), 4 deletions(-) + +diff --git a/src/daemon/config/isulad_config.c b/src/daemon/config/isulad_config.c +index 917e3eaa..38bf4bf9 100644 +--- a/src/daemon/config/isulad_config.c ++++ b/src/daemon/config/isulad_config.c +@@ -314,7 +314,7 @@ char *conf_get_routine_rootdir(const char *runtime) + } + + /* path = conf->rootpath + / + engines + / + runtime + /0 */ +- if (strlen(conf->json_confs->graph) > (SIZE_MAX - strlen(ENGINE_ROOTPATH_NAME)) - 3) { ++ if (strlen(conf->json_confs->graph) > (SIZE_MAX - strlen(ENGINE_ROOTPATH_NAME) - strlen(runtime)) - 3) { + ERROR("Graph path is too long"); + goto out; + } +diff --git a/src/daemon/executor/container_cb/execution_create.c b/src/daemon/executor/container_cb/execution_create.c +index cc9ae716..4cc333fd 100644 +--- a/src/daemon/executor/container_cb/execution_create.c ++++ b/src/daemon/executor/container_cb/execution_create.c +@@ -533,7 +533,7 @@ static char *try_generate_id() + int i = 0; + int max_time = 10; + char *id = NULL; +- char *value = NULL; ++ container_t *value = NULL; + + id = util_smart_calloc_s(sizeof(char), (CONTAINER_ID_MAX_LEN + 1)); + if (id == NULL) { +@@ -547,9 +547,9 @@ static char *try_generate_id() + goto err_out; + } + +- value = container_name_index_get(id); ++ value = containers_store_get(id); + if (value != NULL) { +- free(value); ++ container_unref(value); + value = NULL; + continue; + } else { +-- +2.25.1 + diff --git a/0012-fix-cpu-rt-disable-after-reboot-machine.patch b/0012-fix-cpu-rt-disable-after-reboot-machine.patch new file mode 100644 index 0000000..5ae07d1 --- /dev/null +++ b/0012-fix-cpu-rt-disable-after-reboot-machine.patch @@ -0,0 +1,823 @@ +From 166edf2093b2c35fe4e479ca4b6568be8c98f907 Mon Sep 17 00:00:00 2001 +From: haozi007 +Date: Wed, 15 Feb 2023 17:47:12 +0800 +Subject: [PATCH 12/53] fix cpu-rt disable after reboot machine + +1. ensure parent cgroup cpu-rt of container, should do in start container; +2. current do in create container, will cause failed of start container with cpu-rt after reboot machine + +Signed-off-by: haozi007 +--- + src/daemon/common/sysinfo.c | 46 +++- + src/daemon/common/sysinfo.h | 2 + + src/daemon/executor/container_cb/execution.c | 158 ++++++++++++- + .../executor/container_cb/execution_create.c | 222 +----------------- + src/daemon/modules/api/specs_api.h | 2 +- + src/daemon/modules/spec/specs.c | 84 ++++--- + test/specs/specs/specs_ut.cc | 40 ++-- + 7 files changed, 280 insertions(+), 274 deletions(-) + +diff --git a/src/daemon/common/sysinfo.c b/src/daemon/common/sysinfo.c +index 38416db4..7559d653 100644 +--- a/src/daemon/common/sysinfo.c ++++ b/src/daemon/common/sysinfo.c +@@ -24,8 +24,10 @@ + #include + #include + ++#include ++#include ++ + #include "err_msg.h" +-#include "isula_libutils/log.h" + #include "utils.h" + #include "utils_array.h" + #include "utils_file.h" +@@ -1627,3 +1629,45 @@ free_out: + } + return minfos; + } ++ ++char *sysinfo_cgroup_controller_cpurt_mnt_path() ++{ ++ int nret = 0; ++ __isula_auto_free char *mnt = NULL; ++ __isula_auto_free char *root = NULL; ++ char fpath[PATH_MAX] = { 0 }; ++ sysinfo_t *sysinfo = NULL; ++ ++ sysinfo = get_sys_info(true); ++ if (sysinfo == NULL) { ++ ERROR("Can not get system info"); ++ return NULL; ++ } ++ ++ if (!(sysinfo->cgcpuinfo.cpu_rt_period)) { ++ ERROR("Daemon-scoped cpu-rt-period and cpu-rt-runtime are not supported by kernel"); ++ isulad_set_error_message("Daemon-scoped cpu-rt-period and cpu-rt-runtime are not supported by kernel"); ++ return NULL; ++ } ++ ++ nret = find_cgroup_mountpoint_and_root("cpu", &mnt, &root); ++ if (nret != 0 || mnt == NULL || root == NULL) { ++ ERROR("Can not find cgroup mnt and root path for subsystem 'cpu'"); ++ isulad_set_error_message("Can not find cgroup mnt and root path for subsystem 'cpu'"); ++ return NULL; ++ } ++ ++ // When iSulad is run inside docker, the root is based of the host cgroup. ++ // Replace root to "/" ++ if (strncmp(root, "/docker/", strlen("/docker/")) == 0) { ++ root[1] = '\0'; ++ } ++ ++ nret = snprintf(fpath, sizeof(fpath), "%s/%s", mnt, root); ++ if (nret < 0 || (size_t)nret >= sizeof(fpath)) { ++ ERROR("Failed to print string"); ++ return NULL; ++ } ++ ++ return util_strdup_s(fpath); ++} +\ No newline at end of file +diff --git a/src/daemon/common/sysinfo.h b/src/daemon/common/sysinfo.h +index 8468e00a..bbb3c6b5 100644 +--- a/src/daemon/common/sysinfo.h ++++ b/src/daemon/common/sysinfo.h +@@ -139,6 +139,8 @@ mountinfo_t *find_mount_info(mountinfo_t **minfos, const char *dir); + + void free_mounts_info(mountinfo_t **minfos); + ++char *sysinfo_cgroup_controller_cpurt_mnt_path(); ++ + #ifdef __cplusplus + } + #endif +diff --git a/src/daemon/executor/container_cb/execution.c b/src/daemon/executor/container_cb/execution.c +index 7b18a8e1..ed70fc14 100644 +--- a/src/daemon/executor/container_cb/execution.c ++++ b/src/daemon/executor/container_cb/execution.c +@@ -18,6 +18,12 @@ + #include + #include + #include ++#include ++#include ++#include ++#include ++#include ++ + #include + #include + #include +@@ -34,13 +40,13 @@ + #include + #include + #include +-#include +-#include +-#include +-#include ++#include ++#include + +-#include "isula_libutils/log.h" ++#include "isulad_config.h" ++#include "sysinfo.h" + #include "container_api.h" ++#include "specs_api.h" + #include "execution_extend.h" + #include "execution_information.h" + #include "execution_stream.h" +@@ -302,6 +308,135 @@ static void pack_start_response(container_start_response *response, uint32_t cc, + } + } + ++static int do_init_cpurt_cgroups_path(const char *path, int recursive_depth, const char *mnt_root, ++ int64_t cpu_rt_period, int64_t cpu_rt_runtime); ++ ++/* maybe create cpu realtime file */ ++static int maybe_create_cpu_realtime_file(int64_t value, const char *file, const char *path) ++{ ++ int ret; ++ __isula_auto_close int fd = -1; ++ ssize_t nwrite; ++ char fpath[PATH_MAX] = { 0 }; ++ char buf[ISULAD_NUMSTRLEN64] = { 0 }; ++ ++ if (value == 0) { ++ return 0; ++ } ++ ++ ret = util_mkdir_p(path, CONFIG_DIRECTORY_MODE); ++ if (ret != 0) { ++ ERROR("Failed to mkdir: %s", path); ++ return -1; ++ } ++ ++ ret = snprintf(fpath, sizeof(fpath), "%s/%s", path, file); ++ if (ret < 0 || ret >= sizeof(fpath)) { ++ ERROR("Failed to print string"); ++ return -1; ++ } ++ ret = snprintf(buf, sizeof(buf), "%lld", (long long int)value); ++ if (ret < 0 || (size_t)ret >= sizeof(buf)) { ++ ERROR("Failed to print string"); ++ return -1; ++ } ++ ++ fd = util_open(fpath, O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, 0700); ++ if (fd < 0) { ++ ERROR("Failed to open file: %s: %s", fpath, strerror(errno)); ++ isulad_set_error_message("Failed to open file: %s: %s", fpath, strerror(errno)); ++ return -1; ++ } ++ nwrite = util_write_nointr(fd, buf, strlen(buf)); ++ if (nwrite < 0 || nwrite != strlen(buf)) { ++ ERROR("Failed to write %s to %s: %s", buf, fpath, strerror(errno)); ++ isulad_set_error_message("Failed to write '%s' to '%s': %s", buf, fpath, strerror(errno)); ++ return -1; ++ } ++ ++ return 0; ++} ++ ++static int recursively_create_cgroup(const char *path, const char *mnt_root, int recursive_depth, int64_t cpu_rt_period, ++ int64_t cpu_rt_runtime) ++{ ++ int ret = 0; ++ __isula_auto_free char *dup = NULL; ++ char *dirpath = NULL; ++ char fpath[PATH_MAX] = { 0 }; ++ ++ dup = util_strdup_s(path); ++ dirpath = dirname(dup); ++ ret = do_init_cpurt_cgroups_path(dirpath, (recursive_depth + 1), mnt_root, cpu_rt_period, cpu_rt_runtime); ++ if (ret != 0) { ++ return ret; ++ } ++ ++ int nret = snprintf(fpath, sizeof(fpath), "%s/%s", mnt_root, path); ++ if (nret < 0 || (size_t)nret >= sizeof(fpath)) { ++ ERROR("Failed to print string"); ++ return ret; ++ } ++ ++ ret = maybe_create_cpu_realtime_file(cpu_rt_period, "cpu.rt_period_us", fpath); ++ if (ret != 0) { ++ return ret; ++ } ++ ++ return maybe_create_cpu_realtime_file(cpu_rt_runtime, "cpu.rt_runtime_us", fpath); ++} ++ ++/* init cgroups path */ ++static int do_init_cpurt_cgroups_path(const char *path, int recursive_depth, const char *mnt_root, ++ int64_t cpu_rt_period, int64_t cpu_rt_runtime) ++{ ++ if ((recursive_depth + 1) > MAX_PATH_DEPTH) { ++ ERROR("Reach the max cgroup depth:%s", path); ++ return -1; ++ } ++ ++ if (path == NULL || strcmp(path, "/") == 0 || strcmp(path, ".") == 0) { ++ return 0; ++ } ++ ++ // Recursively create cgroup to ensure that the system and all parent cgroups have values set ++ // for the period and runtime as this limits what the children can be set to. ++ return recursively_create_cgroup(path, mnt_root, recursive_depth, cpu_rt_period, cpu_rt_runtime); ++} ++ ++// TODO: maybe we should adapt to cgroup v2 ++static int cpurt_controller_init(const char *id, const host_config *host_spec) ++{ ++ __isula_auto_free char *mnt_root = NULL; ++ __isula_auto_free char *cgroups_path = NULL; ++ char *dirpath = NULL; ++ int64_t cpu_rt_period = 0; ++ int64_t cpu_rt_runtime = 0; ++ ++ cgroups_path = merge_container_cgroups_path(id, host_spec); ++ if (cgroups_path == NULL || strcmp(cgroups_path, "/") == 0 || strcmp(cgroups_path, ".") == 0) { ++ return 0; ++ } ++ ++ if (conf_get_cgroup_cpu_rt(&cpu_rt_period, &cpu_rt_runtime)) { ++ return -1; ++ } ++ ++ if (cpu_rt_period == 0 && cpu_rt_runtime == 0) { ++ return 0; ++ } ++ ++ mnt_root = sysinfo_cgroup_controller_cpurt_mnt_path(); ++ if (mnt_root == NULL) { ++ ERROR("Failed to get cpu rt controller mnt root path"); ++ return -1; ++ } ++ ++ dirpath = dirname(cgroups_path); ++ ++ return do_init_cpurt_cgroups_path(dirpath, 0, mnt_root, cpu_rt_period, cpu_rt_runtime); ++} ++ + static int container_start_prepare(container_t *cont, const container_start_request *request, int stdinfd, + struct io_write_wrapper *stdout_handler, struct io_write_wrapper *stderr_handler, + char **fifopath, char *fifos[], int *sync_fd, pthread_t *thread_id) +@@ -314,6 +449,19 @@ static int container_start_prepare(container_t *cont, const container_start_requ + return -1; + } + ++ // init cgroup path for cpu_rt_runtime and cpu_rt_period ++ // we should do this in start container, not create container ++ // because, in scenarios: ++ // 1. enable cpu-rt of isulad; ++ // 2. then run container with --cpu-rt-runtime ++ // 3. then reboot machine; ++ // 4. finally, start before container, it will failed... ++ // cause of no one to set value into cgroup/isulad/cpu-rt-runtime and cpu-rt-period. ++ if (cpurt_controller_init(id, cont->hostconfig) != 0) { ++ isulad_set_error_message("Failed to init controller of cpu-rt for container \"%s\".", id); ++ return -1; ++ } ++ + if (prepare_start_io(cont, request, fifopath, fifos, stdinfd, stdout_handler, stderr_handler, sync_fd, thread_id) != + 0) { + return -1; +diff --git a/src/daemon/executor/container_cb/execution_create.c b/src/daemon/executor/container_cb/execution_create.c +index 4cc333fd..4abc89c7 100644 +--- a/src/daemon/executor/container_cb/execution_create.c ++++ b/src/daemon/executor/container_cb/execution_create.c +@@ -19,6 +19,14 @@ + #include + #include + #include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++#include + #include + #include + #include +@@ -26,14 +34,7 @@ + #include + #include + #include +-#include +-#include +-#include +-#include +-#include +-#include + +-#include "isula_libutils/log.h" + #include "isulad_config.h" + #include "specs_api.h" + #include "verify.h" +@@ -58,9 +59,6 @@ + #include "opt_log.h" + #include "runtime_api.h" + +-static int do_init_cpurt_cgroups_path(const char *path, int recursive_depth, const char *mnt_root, +- int64_t cpu_rt_period, int64_t cpu_rt_runtime); +- + static int create_request_check(const container_create_request *request) + { + int ret = 0; +@@ -1323,203 +1321,6 @@ static int save_container_config_before_create(const char *id, const char *runti + return 0; + } + +-/* maybe create cpu realtime file */ +-static int maybe_create_cpu_realtime_file(int64_t value, const char *file, const char *path) +-{ +- int ret; +- int fd = -1; +- ssize_t nwrite; +- char fpath[PATH_MAX] = { 0 }; +- char buf[ISULAD_NUMSTRLEN64] = { 0 }; +- +- if (value == 0) { +- return 0; +- } +- +- ret = util_mkdir_p(path, CONFIG_DIRECTORY_MODE); +- if (ret != 0) { +- ERROR("Failed to mkdir: %s", path); +- return -1; +- } +- +- ret = snprintf(fpath, sizeof(fpath), "%s/%s", path, file); +- if (ret < 0 || ret >= sizeof(fpath)) { +- ERROR("Failed to print string"); +- return -1; +- } +- ret = snprintf(buf, sizeof(buf), "%lld", (long long int)value); +- if (ret < 0 || (size_t)ret >= sizeof(buf)) { +- ERROR("Failed to print string"); +- return -1; +- } +- +- fd = util_open(fpath, O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, 0700); +- if (fd < 0) { +- ERROR("Failed to open file: %s: %s", fpath, strerror(errno)); +- isulad_set_error_message("Failed to open file: %s: %s", fpath, strerror(errno)); +- return -1; +- } +- nwrite = util_write_nointr(fd, buf, strlen(buf)); +- if (nwrite < 0) { +- ERROR("Failed to write %s to %s: %s", buf, fpath, strerror(errno)); +- isulad_set_error_message("Failed to write '%s' to '%s': %s", buf, fpath, strerror(errno)); +- close(fd); +- return -1; +- } +- close(fd); +- +- return 0; +-} +- +-static int recursively_create_cgroup(const char *path, const char *mnt_root, int recursive_depth, int64_t cpu_rt_period, +- int64_t cpu_rt_runtime) +-{ +- int ret = 0; +- char *dup = NULL; +- char *dirpath = NULL; +- char fpath[PATH_MAX] = { 0 }; +- +- dup = util_strdup_s(path); +- dirpath = dirname(dup); +- ret = do_init_cpurt_cgroups_path(dirpath, (recursive_depth + 1), mnt_root, cpu_rt_period, cpu_rt_runtime); +- free(dup); +- if (ret != 0) { +- return ret; +- } +- +- int nret = snprintf(fpath, sizeof(fpath), "%s/%s", mnt_root, path); +- if (nret < 0 || (size_t)nret >= sizeof(fpath)) { +- ERROR("Failed to print string"); +- ret = -1; +- goto out; +- } +- +- ret = maybe_create_cpu_realtime_file(cpu_rt_period, "cpu.rt_period_us", fpath); +- if (ret != 0) { +- goto out; +- } +- +- ret = maybe_create_cpu_realtime_file(cpu_rt_runtime, "cpu.rt_runtime_us", fpath); +- if (ret != 0) { +- goto out; +- } +- +-out: +- return ret; +-} +- +-/* init cgroups path */ +-static int do_init_cpurt_cgroups_path(const char *path, int recursive_depth, const char *mnt_root, +- int64_t cpu_rt_period, int64_t cpu_rt_runtime) +-{ +- if ((recursive_depth + 1) > MAX_PATH_DEPTH) { +- ERROR("Reach the max cgroup depth:%s", path); +- return -1; +- } +- +- if (path == NULL || strcmp(path, "/") == 0 || strcmp(path, ".") == 0) { +- return 0; +- } +- +- // Recursively create cgroup to ensure that the system and all parent cgroups have values set +- // for the period and runtime as this limits what the children can be set to. +- if (recursively_create_cgroup(path, mnt_root, recursive_depth, cpu_rt_period, cpu_rt_runtime)) { +- return -1; +- } +- +- return 0; +-} +- +-static char *get_cpurt_controller_mnt_path() +-{ +- char *res = NULL; +- int nret = 0; +- char *mnt = NULL; +- char *root = NULL; +- char fpath[PATH_MAX] = { 0 }; +- +- nret = find_cgroup_mountpoint_and_root("cpu", &mnt, &root); +- if (nret != 0 || mnt == NULL || root == NULL) { +- ERROR("Can not find cgroup mnt and root path for subsystem 'cpu'"); +- isulad_set_error_message("Can not find cgroup mnt and root path for subsystem 'cpu'"); +- goto out; +- } +- +- // When iSulad is run inside docker, the root is based of the host cgroup. +- // Replace root to "/" +- if (strncmp(root, "/docker/", strlen("/docker/")) == 0) { +- root[1] = '\0'; +- } +- +- nret = snprintf(fpath, sizeof(fpath), "%s/%s", mnt, root); +- if (nret < 0 || (size_t)nret >= sizeof(fpath)) { +- ERROR("Failed to print string"); +- goto out; +- } +- +- res = util_strdup_s(fpath); +- +-out: +- free(mnt); +- free(root); +- return res; +-} +- +-static int cpurt_controller_init(const char *cgroups_path) +-{ +- int ret = 0; +- char *dup = NULL; +- char *dirpath = NULL; +- int64_t cpu_rt_period = 0; +- int64_t cpu_rt_runtime = 0; +- sysinfo_t *sysinfo = NULL; +- char *mnt_root = NULL; +- +- if (cgroups_path == NULL || strcmp(cgroups_path, "/") == 0 || strcmp(cgroups_path, ".") == 0) { +- return 0; +- } +- +- if (conf_get_cgroup_cpu_rt(&cpu_rt_period, &cpu_rt_runtime)) { +- return -1; +- } +- +- if (cpu_rt_period == 0 && cpu_rt_runtime == 0) { +- return 0; +- } +- +- sysinfo = get_sys_info(true); +- if (sysinfo == NULL) { +- ERROR("Can not get system info"); +- ret = -1; +- goto out; +- } +- +- if (!(sysinfo->cgcpuinfo.cpu_rt_period)) { +- ERROR("Daemon-scoped cpu-rt-period and cpu-rt-runtime are not supported by kernel"); +- isulad_set_error_message("Daemon-scoped cpu-rt-period and cpu-rt-runtime are not supported by kernel"); +- ret = -1; +- goto out; +- } +- +- mnt_root = get_cpurt_controller_mnt_path(); +- if (mnt_root == NULL) { +- ERROR("Failed to get cpu rt controller mnt root path"); +- isulad_set_error_message("Failed to get cpu rt controller mnt root path"); +- ret = -1; +- goto out; +- } +- +- dup = util_strdup_s(cgroups_path); +- dirpath = dirname(dup); +- +- ret = do_init_cpurt_cgroups_path(dirpath, 0, mnt_root, cpu_rt_period, cpu_rt_runtime); +- +-out: +- free(mnt_root); +- free(dup); +- return ret; +-} +- + /* + * request -> host_spec + container_spec + * container_spec + image config +@@ -1680,13 +1481,6 @@ int container_create_cb(const container_create_request *request, container_creat + goto umount_channel; + } + +- // init cgroup path for cpu_rt_runtime and cpu_rt_period +- if (cpurt_controller_init(oci_spec->linux->cgroups_path) != 0) { +- ERROR("Unable to init CPU RT controller %s", oci_spec->linux->cgroups_path); +- cc = ISULAD_ERR_EXEC; +- goto umount_channel; +- } +- + if (container_v2_spec_merge_contaner_spec(v2_spec) != 0) { + ERROR("Failed to merge container settings"); + cc = ISULAD_ERR_EXEC; +diff --git a/src/daemon/modules/api/specs_api.h b/src/daemon/modules/api/specs_api.h +index 4c132108..e0a73f55 100644 +--- a/src/daemon/modules/api/specs_api.h ++++ b/src/daemon/modules/api/specs_api.h +@@ -28,7 +28,7 @@ extern "C" { + + int merge_all_specs(host_config *host_spec, const char *real_rootfs, container_config_v2_common_config *v2_spec, + oci_runtime_spec *oci_spec); +-int merge_oci_cgroups_path(const char *id, oci_runtime_spec *oci_spec, const host_config *host_spec); ++char *merge_container_cgroups_path(const char *id, const host_config *host_spec); + int merge_global_config(oci_runtime_spec *oci_spec); + oci_runtime_spec *load_oci_config(const char *rootpath, const char *name); + oci_runtime_spec *default_spec(bool system_container); +diff --git a/src/daemon/modules/spec/specs.c b/src/daemon/modules/spec/specs.c +index 12d9b96d..857fc3dc 100644 +--- a/src/daemon/modules/spec/specs.c ++++ b/src/daemon/modules/spec/specs.c +@@ -26,6 +26,7 @@ + #include + #include + #include ++#include + #include + #include + +@@ -165,36 +166,43 @@ out: + return ret; + } + +-static int make_annotations_cgroup_dir(const container_config *container_spec, const host_config *host_spec) ++static char *do_get_container_cgroup_path(const host_config *host_spec) + { +- int ret = 0; +- char cleaned[PATH_MAX] = { 0 }; +- char *default_cgroup_parent = NULL; + char *path = NULL; + +- default_cgroup_parent = conf_get_isulad_cgroup_parent(); + if (host_spec->cgroup_parent != NULL) { +- path = host_spec->cgroup_parent; +- } else if (default_cgroup_parent != NULL) { +- path = default_cgroup_parent; ++ // first, use user setting ++ path = util_strdup_s(host_spec->cgroup_parent); ++ } else { ++ // second, if user donot set, use setting from daemon config ++ path = conf_get_isulad_cgroup_parent(); + } ++ + if (path == NULL) { +- path = "/isulad"; ++ // third, all faild, just use default '/isulad' ++ path = util_strdup_s("/isulad"); + } ++ ++ return path; ++} ++ ++static int make_annotations_cgroup_dir(const container_config *container_spec, const host_config *host_spec) ++{ ++ char cleaned[PATH_MAX] = { 0 }; ++ __isula_auto_free char *path = NULL; ++ ++ path = do_get_container_cgroup_path(host_spec); + if (util_clean_path(path, cleaned, sizeof(cleaned)) == NULL) { + ERROR("Failed to clean path: %s", path); +- ret = -1; +- goto out; ++ return -1; + } ++ + if (append_json_map_string_string(container_spec->annotations, "cgroup.dir", cleaned)) { + ERROR("Realloc annotations failed"); +- ret = -1; +- goto out; ++ return -1; + } + +-out: +- free(default_cgroup_parent); +- return ret; ++ return 0; + } + + static int make_annotations_oom_score_adj(const container_config *container_spec, const host_config *host_spec) +@@ -2058,42 +2066,40 @@ out: + return ret; + } + +-int merge_oci_cgroups_path(const char *id, oci_runtime_spec *oci_spec, const host_config *host_spec) ++char *merge_container_cgroups_path(const char *id, const host_config *host_spec) + { +- int ret = 0; +- char *default_cgroup_parent = NULL; +- char *path = NULL; ++ __isula_auto_free char *path = NULL; + ++ if (id == NULL || host_spec == NULL) { ++ ERROR("Invalid arguments"); ++ return NULL; ++ } ++ ++ path = do_get_container_cgroup_path(host_spec); ++ ++ return util_path_join(path, id); ++} ++ ++static int merge_oci_cgroups_path(const char *id, oci_runtime_spec *oci_spec, const host_config *host_spec) ++{ + if (id == NULL || oci_spec == NULL || host_spec == NULL) { + ERROR("Invalid arguments"); +- ret = -1; +- goto out; ++ return -1; + } + + if (make_sure_oci_spec_linux(oci_spec) != 0) { + ERROR("Failed to make oci spec linux"); +- ret = -1; +- goto out; ++ return -1; + } + +- default_cgroup_parent = conf_get_isulad_cgroup_parent(); +- path = default_cgroup_parent; +- if (host_spec->cgroup_parent != NULL) { +- path = host_spec->cgroup_parent; +- } ++ free(oci_spec->linux->cgroups_path); ++ oci_spec->linux->cgroups_path = merge_container_cgroups_path(id, host_spec); + +- if (path == NULL) { +- free(oci_spec->linux->cgroups_path); +- oci_spec->linux->cgroups_path = util_path_join("/isulad", id); +- return 0; ++ if (oci_spec->linux->cgroups_path == NULL) { ++ WARN("OCI spec cgroups path is NULL"); + } + +- free(oci_spec->linux->cgroups_path); +- oci_spec->linux->cgroups_path = util_path_join(path, id); +- +-out: +- free(default_cgroup_parent); +- return ret; ++ return 0; + } + + int merge_all_specs(host_config *host_spec, const char *real_rootfs, container_config_v2_common_config *v2_spec, +diff --git a/test/specs/specs/specs_ut.cc b/test/specs/specs/specs_ut.cc +index c4014e2e..96aa1c63 100644 +--- a/test/specs/specs/specs_ut.cc ++++ b/test/specs/specs/specs_ut.cc +@@ -232,15 +232,16 @@ char *invoke_conf_get_isulad_cgroup_parent() + return util_strdup_s("/var/lib/isulad/engines/lcr"); + } + +-TEST_F(SpecsUnitTest, test_merge_oci_cgroups_path_1) ++TEST_F(SpecsUnitTest, test_merge_container_cgroups_path_1) + { +- ASSERT_EQ(merge_oci_cgroups_path(nullptr, nullptr, nullptr), -1); ++ ASSERT_EQ(merge_container_cgroups_path(nullptr, nullptr), nullptr); + } + +-TEST_F(SpecsUnitTest, test_merge_oci_cgroups_path_2) ++TEST_F(SpecsUnitTest, test_merge_container_cgroups_path_2) + { + oci_runtime_spec *oci_spec = nullptr; + host_config *host_spec = nullptr; ++ char *merged_cp = nullptr; + + oci_spec = (oci_runtime_spec *)util_common_calloc_s(sizeof(oci_runtime_spec)); + ASSERT_TRUE(oci_spec != nullptr); +@@ -250,20 +251,23 @@ TEST_F(SpecsUnitTest, test_merge_oci_cgroups_path_2) + + EXPECT_CALL(m_isulad_conf, GetCgroupParent()).WillRepeatedly(Invoke(invoke_conf_get_isulad_cgroup_parent_null)); + +- ASSERT_EQ(merge_oci_cgroups_path("123", oci_spec, host_spec), 0); ++ merged_cp = merge_container_cgroups_path("123", host_spec); ++ ASSERT_NE(merged_cp, nullptr); + +- ASSERT_STREQ(oci_spec->linux->cgroups_path, "/isulad/123"); ++ ASSERT_STREQ(merged_cp, "/isulad/123"); + + free_oci_runtime_spec(oci_spec); + free_host_config(host_spec); ++ free(merged_cp); + + testing::Mock::VerifyAndClearExpectations(&m_isulad_conf); + } + +-TEST_F(SpecsUnitTest, test_merge_oci_cgroups_path_3) ++TEST_F(SpecsUnitTest, test_merge_container_cgroups_path_3) + { + oci_runtime_spec *oci_spec = nullptr; + host_config *host_spec = nullptr; ++ char *merged_cp = nullptr; + + oci_spec = (oci_runtime_spec *)util_common_calloc_s(sizeof(oci_runtime_spec)); + ASSERT_TRUE(oci_spec != nullptr); +@@ -275,20 +279,23 @@ TEST_F(SpecsUnitTest, test_merge_oci_cgroups_path_3) + + EXPECT_CALL(m_isulad_conf, GetCgroupParent()).WillRepeatedly(Invoke(invoke_conf_get_isulad_cgroup_parent_null)); + +- ASSERT_EQ(merge_oci_cgroups_path("123", oci_spec, host_spec), 0); ++ merged_cp = merge_container_cgroups_path("123", host_spec); ++ ASSERT_NE(merged_cp, nullptr); + +- ASSERT_STREQ(oci_spec->linux->cgroups_path, "/test/123"); ++ ASSERT_STREQ(merged_cp, "/test/123"); + + free_oci_runtime_spec(oci_spec); + free_host_config(host_spec); ++ free(merged_cp); + + testing::Mock::VerifyAndClearExpectations(&m_isulad_conf); + } + +-TEST_F(SpecsUnitTest, test_merge_oci_cgroups_path_4) ++TEST_F(SpecsUnitTest, test_merge_container_cgroups_path_4) + { + oci_runtime_spec *oci_spec = nullptr; + host_config *host_spec = nullptr; ++ char *merged_cp = nullptr; + + oci_spec = (oci_runtime_spec *)util_common_calloc_s(sizeof(oci_runtime_spec)); + ASSERT_TRUE(oci_spec != nullptr); +@@ -298,20 +305,23 @@ TEST_F(SpecsUnitTest, test_merge_oci_cgroups_path_4) + + EXPECT_CALL(m_isulad_conf, GetCgroupParent()).WillRepeatedly(Invoke(invoke_conf_get_isulad_cgroup_parent)); + +- ASSERT_EQ(merge_oci_cgroups_path("123", oci_spec, host_spec), 0); ++ merged_cp = merge_container_cgroups_path("123", host_spec); ++ ASSERT_NE(merged_cp, nullptr); + +- ASSERT_STREQ(oci_spec->linux->cgroups_path, "/var/lib/isulad/engines/lcr/123"); ++ ASSERT_STREQ(merged_cp, "/var/lib/isulad/engines/lcr/123"); + + free_oci_runtime_spec(oci_spec); + free_host_config(host_spec); ++ free(merged_cp); + + testing::Mock::VerifyAndClearExpectations(&m_isulad_conf); + } + +-TEST_F(SpecsUnitTest, test_merge_oci_cgroups_path_5) ++TEST_F(SpecsUnitTest, test_merge_container_cgroups_path_5) + { + oci_runtime_spec *oci_spec = nullptr; + host_config *host_spec = nullptr; ++ char *merged_cp = nullptr; + + oci_spec = (oci_runtime_spec *)util_common_calloc_s(sizeof(oci_runtime_spec)); + ASSERT_TRUE(oci_spec != nullptr); +@@ -323,12 +333,14 @@ TEST_F(SpecsUnitTest, test_merge_oci_cgroups_path_5) + + EXPECT_CALL(m_isulad_conf, GetCgroupParent()).WillRepeatedly(Invoke(invoke_conf_get_isulad_cgroup_parent)); + +- ASSERT_EQ(merge_oci_cgroups_path("123", oci_spec, host_spec), 0); ++ merged_cp = merge_container_cgroups_path("123", host_spec); ++ ASSERT_NE(merged_cp, nullptr); + +- ASSERT_STREQ(oci_spec->linux->cgroups_path, "/test/123"); ++ ASSERT_STREQ(merged_cp, "/test/123"); + + free_oci_runtime_spec(oci_spec); + free_host_config(host_spec); ++ free(merged_cp); + + testing::Mock::VerifyAndClearExpectations(&m_isulad_conf); + } +-- +2.25.1 + diff --git a/0013-fix-selinux_label_ut-timeout-and-add-timeout-for-all.patch b/0013-fix-selinux_label_ut-timeout-and-add-timeout-for-all.patch new file mode 100644 index 0000000..38efc68 --- /dev/null +++ b/0013-fix-selinux_label_ut-timeout-and-add-timeout-for-all.patch @@ -0,0 +1,465 @@ +From af8fb9fcf604775f527b58e1b02f220dffd8ff35 Mon Sep 17 00:00:00 2001 +From: zhangxiaoyu +Date: Thu, 16 Feb 2023 15:26:10 +0800 +Subject: [PATCH 13/53] fix selinux_label_ut timeout and add timeout for all ut + +Signed-off-by: zhangxiaoyu +--- + test/cgroup/cpu/CMakeLists.txt | 1 + + test/cmd/isula/extend/pause/CMakeLists.txt | 1 + + test/cmd/isula/extend/resume/CMakeLists.txt | 2 +- + test/cmd/isula/information/info/CMakeLists.txt | 1 + + test/cmd/isula/information/ps/CMakeLists.txt | 1 + + test/cmd/isula/utils/client_show_format/CMakeLists.txt | 1 + + test/cmd/isula/utils/template_string_parse/CMakeLists.txt | 1 + + test/cmd/isulad-shim/CMakeLists.txt | 1 + + test/cutils/mainloop/CMakeLists.txt | 1 + + test/cutils/map/CMakeLists.txt | 1 + + test/cutils/path/CMakeLists.txt | 1 + + test/cutils/util_atomic/CMakeLists.txt | 1 + + test/cutils/utils_aes/CMakeLists.txt | 1 + + test/cutils/utils_array/CMakeLists.txt | 1 + + test/cutils/utils_base64/CMakeLists.txt | 1 + + test/cutils/utils_convert/CMakeLists.txt | 1 + + test/cutils/utils_error/CMakeLists.txt | 1 + + test/cutils/utils_file/CMakeLists.txt | 1 + + test/cutils/utils_filters/CMakeLists.txt | 1 + + test/cutils/utils_fs/CMakeLists.txt | 1 + + test/cutils/utils_mount_spec/CMakeLists.txt | 1 + + test/cutils/utils_namespace/CMakeLists.txt | 1 + + test/cutils/utils_network/CMakeLists.txt | 1 + + test/cutils/utils_pwgr/CMakeLists.txt | 1 + + test/cutils/utils_regex/CMakeLists.txt | 1 + + test/cutils/utils_string/CMakeLists.txt | 1 + + test/cutils/utils_timestamp/CMakeLists.txt | 1 + + test/cutils/utils_utils/CMakeLists.txt | 1 + + test/cutils/utils_verify/CMakeLists.txt | 1 + + test/image/oci/oci_config_merge/CMakeLists.txt | 1 + + test/image/oci/registry/CMakeLists.txt | 1 + + test/image/oci/storage/images/CMakeLists.txt | 1 + + test/image/oci/storage/layers/CMakeLists.txt | 2 ++ + test/image/oci/storage/rootfs/CMakeLists.txt | 1 + + test/network/CMakeLists.txt | 1 + + test/runtime/isula/CMakeLists.txt | 1 + + test/runtime/lcr/CMakeLists.txt | 1 + + .../services/execution/execute/execution_extend/CMakeLists.txt | 1 + + test/services/execution/spec/CMakeLists.txt | 2 ++ + test/services/execution/spec/selinux_label_ut.cc | 2 ++ + test/specs/specs/CMakeLists.txt | 1 + + test/specs/specs_extend/CMakeLists.txt | 1 + + test/volume/CMakeLists.txt | 3 ++- + 43 files changed, 47 insertions(+), 2 deletions(-) + +diff --git a/test/cgroup/cpu/CMakeLists.txt b/test/cgroup/cpu/CMakeLists.txt +index 159b0d85..6a8af719 100644 +--- a/test/cgroup/cpu/CMakeLists.txt ++++ b/test/cgroup/cpu/CMakeLists.txt +@@ -26,3 +26,4 @@ target_include_directories(${EXE} PUBLIC + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cmd/isula/extend/pause/CMakeLists.txt b/test/cmd/isula/extend/pause/CMakeLists.txt +index af48c23f..6eab5172 100644 +--- a/test/cmd/isula/extend/pause/CMakeLists.txt ++++ b/test/cmd/isula/extend/pause/CMakeLists.txt +@@ -47,3 +47,4 @@ target_include_directories(${EXE} PUBLIC + ) + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cmd/isula/extend/resume/CMakeLists.txt b/test/cmd/isula/extend/resume/CMakeLists.txt +index 729c4d3b..0812a66e 100644 +--- a/test/cmd/isula/extend/resume/CMakeLists.txt ++++ b/test/cmd/isula/extend/resume/CMakeLists.txt +@@ -48,4 +48,4 @@ target_include_directories(${EXE} PUBLIC + ) + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) +- ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cmd/isula/information/info/CMakeLists.txt b/test/cmd/isula/information/info/CMakeLists.txt +index 2f134986..d8d8ddb7 100644 +--- a/test/cmd/isula/information/info/CMakeLists.txt ++++ b/test/cmd/isula/information/info/CMakeLists.txt +@@ -49,3 +49,4 @@ target_include_directories(${EXE} PUBLIC + ) + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cmd/isula/information/ps/CMakeLists.txt b/test/cmd/isula/information/ps/CMakeLists.txt +index 81d4202c..436e611b 100644 +--- a/test/cmd/isula/information/ps/CMakeLists.txt ++++ b/test/cmd/isula/information/ps/CMakeLists.txt +@@ -51,3 +51,4 @@ target_include_directories(${EXE} PUBLIC + ) + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cmd/isula/utils/client_show_format/CMakeLists.txt b/test/cmd/isula/utils/client_show_format/CMakeLists.txt +index 60d538ad..e19332cf 100644 +--- a/test/cmd/isula/utils/client_show_format/CMakeLists.txt ++++ b/test/cmd/isula/utils/client_show_format/CMakeLists.txt +@@ -19,3 +19,4 @@ target_include_directories(${EXE} PUBLIC + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cmd/isula/utils/template_string_parse/CMakeLists.txt b/test/cmd/isula/utils/template_string_parse/CMakeLists.txt +index 32256b6b..25efd25c 100644 +--- a/test/cmd/isula/utils/template_string_parse/CMakeLists.txt ++++ b/test/cmd/isula/utils/template_string_parse/CMakeLists.txt +@@ -19,3 +19,4 @@ target_include_directories(${EXE} PUBLIC + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cmd/isulad-shim/CMakeLists.txt b/test/cmd/isulad-shim/CMakeLists.txt +index 1c2de232..dc293f6d 100644 +--- a/test/cmd/isulad-shim/CMakeLists.txt ++++ b/test/cmd/isulad-shim/CMakeLists.txt +@@ -30,3 +30,4 @@ target_include_directories(${EXE} PUBLIC + ) + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/mainloop/CMakeLists.txt b/test/cutils/mainloop/CMakeLists.txt +index 78e3f18d..61ea7ffc 100644 +--- a/test/cutils/mainloop/CMakeLists.txt ++++ b/test/cutils/mainloop/CMakeLists.txt +@@ -25,3 +25,4 @@ target_link_libraries(${EXE} + libutils_ut -lcrypto -lyajl -lz) + + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/map/CMakeLists.txt b/test/cutils/map/CMakeLists.txt +index 4059559f..bd21ee3f 100644 +--- a/test/cutils/map/CMakeLists.txt ++++ b/test/cutils/map/CMakeLists.txt +@@ -14,3 +14,4 @@ target_include_directories(${EXE} PUBLIC + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} libutils_ut -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/path/CMakeLists.txt b/test/cutils/path/CMakeLists.txt +index 745258a1..aa0c6c6e 100644 +--- a/test/cutils/path/CMakeLists.txt ++++ b/test/cutils/path/CMakeLists.txt +@@ -15,3 +15,4 @@ target_include_directories(${EXE} PUBLIC + set_target_properties(${EXE} PROPERTIES LINK_FLAGS "-Wl,--wrap,getcwd -Wl,--wrap,readlink") + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/util_atomic/CMakeLists.txt b/test/cutils/util_atomic/CMakeLists.txt +index 071b2a04..21b05c46 100644 +--- a/test/cutils/util_atomic/CMakeLists.txt ++++ b/test/cutils/util_atomic/CMakeLists.txt +@@ -16,3 +16,4 @@ target_include_directories(${EXE} PUBLIC + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/utils_aes/CMakeLists.txt b/test/cutils/utils_aes/CMakeLists.txt +index f7535bb3..c4d175c8 100644 +--- a/test/cutils/utils_aes/CMakeLists.txt ++++ b/test/cutils/utils_aes/CMakeLists.txt +@@ -14,3 +14,4 @@ target_include_directories(${EXE} PUBLIC + ) + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/utils_array/CMakeLists.txt b/test/cutils/utils_array/CMakeLists.txt +index 71733e31..afe564bf 100644 +--- a/test/cutils/utils_array/CMakeLists.txt ++++ b/test/cutils/utils_array/CMakeLists.txt +@@ -16,3 +16,4 @@ target_include_directories(${EXE} PUBLIC + set_target_properties(${EXE} PROPERTIES LINK_FLAGS "-Wl,--wrap,calloc") + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/utils_base64/CMakeLists.txt b/test/cutils/utils_base64/CMakeLists.txt +index d5b99361..35e6ba04 100644 +--- a/test/cutils/utils_base64/CMakeLists.txt ++++ b/test/cutils/utils_base64/CMakeLists.txt +@@ -15,3 +15,4 @@ target_include_directories(${EXE} PUBLIC + ) + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/utils_convert/CMakeLists.txt b/test/cutils/utils_convert/CMakeLists.txt +index 30068208..fcf70a7c 100644 +--- a/test/cutils/utils_convert/CMakeLists.txt ++++ b/test/cutils/utils_convert/CMakeLists.txt +@@ -15,3 +15,4 @@ target_include_directories(${EXE} PUBLIC + ) + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/utils_error/CMakeLists.txt b/test/cutils/utils_error/CMakeLists.txt +index 28016605..9607aeeb 100644 +--- a/test/cutils/utils_error/CMakeLists.txt ++++ b/test/cutils/utils_error/CMakeLists.txt +@@ -14,3 +14,4 @@ target_include_directories(${EXE} PUBLIC + ) + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/utils_file/CMakeLists.txt b/test/cutils/utils_file/CMakeLists.txt +index 20317e15..01b2ff47 100644 +--- a/test/cutils/utils_file/CMakeLists.txt ++++ b/test/cutils/utils_file/CMakeLists.txt +@@ -16,3 +16,4 @@ target_include_directories(${EXE} PUBLIC + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/utils_filters/CMakeLists.txt b/test/cutils/utils_filters/CMakeLists.txt +index 31d3ac25..561ff4ef 100644 +--- a/test/cutils/utils_filters/CMakeLists.txt ++++ b/test/cutils/utils_filters/CMakeLists.txt +@@ -16,3 +16,4 @@ target_include_directories(${EXE} PUBLIC + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/utils_fs/CMakeLists.txt b/test/cutils/utils_fs/CMakeLists.txt +index 7ff3176a..e909aee6 100644 +--- a/test/cutils/utils_fs/CMakeLists.txt ++++ b/test/cutils/utils_fs/CMakeLists.txt +@@ -14,3 +14,4 @@ target_include_directories(${EXE} PUBLIC + ) + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/utils_mount_spec/CMakeLists.txt b/test/cutils/utils_mount_spec/CMakeLists.txt +index 24fb5add..099805d2 100644 +--- a/test/cutils/utils_mount_spec/CMakeLists.txt ++++ b/test/cutils/utils_mount_spec/CMakeLists.txt +@@ -14,3 +14,4 @@ target_include_directories(${EXE} PUBLIC + ) + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/utils_namespace/CMakeLists.txt b/test/cutils/utils_namespace/CMakeLists.txt +index 8add4a71..fb535e71 100644 +--- a/test/cutils/utils_namespace/CMakeLists.txt ++++ b/test/cutils/utils_namespace/CMakeLists.txt +@@ -14,3 +14,4 @@ target_include_directories(${EXE} PUBLIC + ) + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/utils_network/CMakeLists.txt b/test/cutils/utils_network/CMakeLists.txt +index cf0fc481..22e421ab 100644 +--- a/test/cutils/utils_network/CMakeLists.txt ++++ b/test/cutils/utils_network/CMakeLists.txt +@@ -20,3 +20,4 @@ target_link_libraries(${EXE} + libutils_ut -lcrypto -lyajl -lz) + + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/utils_pwgr/CMakeLists.txt b/test/cutils/utils_pwgr/CMakeLists.txt +index 5938991e..34acb92a 100644 +--- a/test/cutils/utils_pwgr/CMakeLists.txt ++++ b/test/cutils/utils_pwgr/CMakeLists.txt +@@ -15,3 +15,4 @@ target_include_directories(${EXE} PUBLIC + ) + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/utils_regex/CMakeLists.txt b/test/cutils/utils_regex/CMakeLists.txt +index 3f6410b2..45ba2604 100644 +--- a/test/cutils/utils_regex/CMakeLists.txt ++++ b/test/cutils/utils_regex/CMakeLists.txt +@@ -14,3 +14,4 @@ target_include_directories(${EXE} PUBLIC + ) + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/utils_string/CMakeLists.txt b/test/cutils/utils_string/CMakeLists.txt +index 1343f4e6..b9968c43 100644 +--- a/test/cutils/utils_string/CMakeLists.txt ++++ b/test/cutils/utils_string/CMakeLists.txt +@@ -16,3 +16,4 @@ target_include_directories(${EXE} PUBLIC + set_target_properties(${EXE} PROPERTIES LINK_FLAGS "-Wl,--wrap,util_strdup_s -Wl,--wrap,calloc -Wl,--wrap,strcat_s") + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/utils_timestamp/CMakeLists.txt b/test/cutils/utils_timestamp/CMakeLists.txt +index 38aec640..6da3fcc9 100644 +--- a/test/cutils/utils_timestamp/CMakeLists.txt ++++ b/test/cutils/utils_timestamp/CMakeLists.txt +@@ -14,3 +14,4 @@ target_include_directories(${EXE} PUBLIC + ) + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/utils_utils/CMakeLists.txt b/test/cutils/utils_utils/CMakeLists.txt +index 7b3bd546..99a83e7a 100644 +--- a/test/cutils/utils_utils/CMakeLists.txt ++++ b/test/cutils/utils_utils/CMakeLists.txt +@@ -14,3 +14,4 @@ target_include_directories(${EXE} PUBLIC + ) + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/utils_verify/CMakeLists.txt b/test/cutils/utils_verify/CMakeLists.txt +index abf9596f..dd9ef78b 100644 +--- a/test/cutils/utils_verify/CMakeLists.txt ++++ b/test/cutils/utils_verify/CMakeLists.txt +@@ -14,3 +14,4 @@ target_include_directories(${EXE} PUBLIC + ) + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/image/oci/oci_config_merge/CMakeLists.txt b/test/image/oci/oci_config_merge/CMakeLists.txt +index 88047fde..42cd2e78 100644 +--- a/test/image/oci/oci_config_merge/CMakeLists.txt ++++ b/test/image/oci/oci_config_merge/CMakeLists.txt +@@ -73,3 +73,4 @@ target_include_directories(${EXE} PUBLIC + set_target_properties(${EXE} PROPERTIES LINK_FLAGS "-Wl,--wrap,util_common_calloc_s -Wl,--wrap,util_smart_calloc_s -Wl,--wrap,merge_env") + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/image/oci/registry/CMakeLists.txt b/test/image/oci/registry/CMakeLists.txt +index 727a615a..13ed95b2 100644 +--- a/test/image/oci/registry/CMakeLists.txt ++++ b/test/image/oci/registry/CMakeLists.txt +@@ -63,3 +63,4 @@ target_include_directories(${EXE} PUBLIC + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lcrypto -lyajl -lz libhttpclient) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/image/oci/storage/images/CMakeLists.txt b/test/image/oci/storage/images/CMakeLists.txt +index b00c5a0e..3e6b69a4 100644 +--- a/test/image/oci/storage/images/CMakeLists.txt ++++ b/test/image/oci/storage/images/CMakeLists.txt +@@ -45,3 +45,4 @@ target_include_directories(${EXE} PUBLIC + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/image/oci/storage/layers/CMakeLists.txt b/test/image/oci/storage/layers/CMakeLists.txt +index 3fe8ab7c..952e9483 100644 +--- a/test/image/oci/storage/layers/CMakeLists.txt ++++ b/test/image/oci/storage/layers/CMakeLists.txt +@@ -66,6 +66,7 @@ target_link_libraries(${DRIVER_EXE} + -lwebsockets -lcrypto -lyajl -larchive ${SELINUX_LIBRARY} -ldevmapper -lz) + + add_test(NAME ${DRIVER_EXE} COMMAND ${DRIVER_EXE} --gtest_output=xml:${DRIVER_EXE}-Results.xml) ++set_tests_properties(${DRIVER_EXE} PROPERTIES TIMEOUT 120) + + # storage_layers_ut + SET(LAYER_EXE storage_layers_ut) +@@ -141,3 +142,4 @@ target_link_libraries(${LAYER_EXE} + -lwebsockets -lcrypto -lyajl -larchive ${SELINUX_LIBRARY} -ldevmapper -lz) + + add_test(NAME ${LAYER_EXE} COMMAND ${LAYER_EXE} --gtest_output=xml:${LAYER_EXE}-Results.xml) ++set_tests_properties(${LAYER_EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/image/oci/storage/rootfs/CMakeLists.txt b/test/image/oci/storage/rootfs/CMakeLists.txt +index 5bf568f9..4d7d3533 100644 +--- a/test/image/oci/storage/rootfs/CMakeLists.txt ++++ b/test/image/oci/storage/rootfs/CMakeLists.txt +@@ -45,3 +45,4 @@ target_include_directories(${EXE} PUBLIC + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/network/CMakeLists.txt b/test/network/CMakeLists.txt +index e354bebc..be31fd0e 100644 +--- a/test/network/CMakeLists.txt ++++ b/test/network/CMakeLists.txt +@@ -86,3 +86,4 @@ target_include_directories(${EXE} PUBLIC + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/runtime/isula/CMakeLists.txt b/test/runtime/isula/CMakeLists.txt +index 38a454b0..f5821953 100644 +--- a/test/runtime/isula/CMakeLists.txt ++++ b/test/runtime/isula/CMakeLists.txt +@@ -58,3 +58,4 @@ target_include_directories(${EXE} PUBLIC + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lpthread -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/runtime/lcr/CMakeLists.txt b/test/runtime/lcr/CMakeLists.txt +index 6f8f784c..979cbe5a 100644 +--- a/test/runtime/lcr/CMakeLists.txt ++++ b/test/runtime/lcr/CMakeLists.txt +@@ -58,3 +58,4 @@ target_include_directories(${EXE} PUBLIC + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/services/execution/execute/execution_extend/CMakeLists.txt b/test/services/execution/execute/execution_extend/CMakeLists.txt +index aaff39ef..8588fac3 100644 +--- a/test/services/execution/execute/execution_extend/CMakeLists.txt ++++ b/test/services/execution/execute/execution_extend/CMakeLists.txt +@@ -76,3 +76,4 @@ target_include_directories(${EXE} PUBLIC + ) + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/services/execution/spec/CMakeLists.txt b/test/services/execution/spec/CMakeLists.txt +index e1aa680e..d4a9d9b0 100644 +--- a/test/services/execution/spec/CMakeLists.txt ++++ b/test/services/execution/spec/CMakeLists.txt +@@ -74,4 +74,6 @@ target_include_directories(${MOCK_EXE} PUBLIC + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${SELINUX_LIBRARY} ${ISULA_LIBUTILS_LIBRARY} -lcrypto -lyajl -lz) + target_link_libraries(${MOCK_EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${SELINUX_LIBRARY} ${ISULA_LIBUTILS_LIBRARY} -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) + add_test(NAME ${MOCK_EXE} COMMAND ${MOCK_EXE} --gtest_output=xml:${MOCK_EXE}-Results.xml) ++set_tests_properties(${MOCK_EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/services/execution/spec/selinux_label_ut.cc b/test/services/execution/spec/selinux_label_ut.cc +index c0e9ab1c..f9a73cca 100644 +--- a/test/services/execution/spec/selinux_label_ut.cc ++++ b/test/services/execution/spec/selinux_label_ut.cc +@@ -179,11 +179,13 @@ protected: + void SetUp() override + { + CreateTestedObjects(); ++ selinux_state_init(); + } + + void TearDown() override + { + ClearTestedObjects(); ++ selinux_state_free(); + } + + private: +diff --git a/test/specs/specs/CMakeLists.txt b/test/specs/specs/CMakeLists.txt +index b730959c..c4b36c5f 100644 +--- a/test/specs/specs/CMakeLists.txt ++++ b/test/specs/specs/CMakeLists.txt +@@ -83,3 +83,4 @@ target_include_directories(${EXE} PUBLIC + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/specs/specs_extend/CMakeLists.txt b/test/specs/specs_extend/CMakeLists.txt +index 7d05deb4..06f46a37 100644 +--- a/test/specs/specs_extend/CMakeLists.txt ++++ b/test/specs/specs_extend/CMakeLists.txt +@@ -79,3 +79,4 @@ target_include_directories(${EXE} PUBLIC + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/volume/CMakeLists.txt b/test/volume/CMakeLists.txt +index 6eea3f76..e2045b19 100644 +--- a/test/volume/CMakeLists.txt ++++ b/test/volume/CMakeLists.txt +@@ -39,4 +39,5 @@ target_include_directories(${EXE} PUBLIC + ) + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lcrypto -lyajl -lz) +-add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) +\ No newline at end of file ++add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +-- +2.25.1 + diff --git a/0014-add-retry-for-read-write.patch b/0014-add-retry-for-read-write.patch new file mode 100644 index 0000000..c51a570 --- /dev/null +++ b/0014-add-retry-for-read-write.patch @@ -0,0 +1,360 @@ +From 734fca150e1c5da2814a55e0315bde8e828e6e8a Mon Sep 17 00:00:00 2001 +From: zhangxiaoyu +Date: Fri, 17 Feb 2023 16:07:53 +0800 +Subject: [PATCH 14/53] add retry for read/write + +Signed-off-by: zhangxiaoyu +--- + src/cmd/isulad-shim/common.c | 6 +++--- + src/cmd/isulad-shim/process.c | 2 +- + src/cmd/isulad/main.c | 4 ++-- + src/daemon/common/selinux_label.c | 2 +- + src/daemon/entry/connect/grpc/grpc_containers_service.cc | 9 ++++++--- + src/daemon/entry/cri/sysctl_tools.c | 2 +- + src/daemon/executor/container_cb/execution.c | 4 ++-- + .../modules/container/container_gc/containers_gc.c | 3 ++- + src/daemon/modules/events_sender/event_sender.c | 2 +- + src/daemon/modules/image/oci/storage/storage.c | 4 +++- + src/daemon/modules/log/log_gather.c | 6 +++--- + src/daemon/modules/plugin/plugin.c | 2 +- + src/daemon/modules/runtime/isula/isula_rt_ops.c | 4 ++-- + src/daemon/modules/service/io_handler.c | 2 +- + src/daemon/modules/service/service_container.c | 2 +- + src/utils/cutils/utils.c | 2 +- + src/utils/cutils/utils_aes.c | 2 +- + src/utils/cutils/utils_file.c | 2 +- + src/utils/tar/util_archive.c | 4 ++-- + src/utils/tar/util_gzip.c | 2 +- + 20 files changed, 36 insertions(+), 30 deletions(-) + +diff --git a/src/cmd/isulad-shim/common.c b/src/cmd/isulad-shim/common.c +index bb8464bb..0c345187 100644 +--- a/src/cmd/isulad-shim/common.c ++++ b/src/cmd/isulad-shim/common.c +@@ -196,7 +196,7 @@ int generate_random_str(char *id, size_t len) + } + for (i = 0; i < len; i++) { + int nret; +- if (read(fd, &num, sizeof(int)) < 0) { ++ if (read_nointr(fd, &num, sizeof(int)) < 0) { + close(fd); + return SHIM_ERR; + } +@@ -232,8 +232,8 @@ void write_message(int fd, const char *level, const char *fmt, ...) + va_end(arg_list); + + snprintf(msg, MAX_MESSAGE_LEN - 1, "{\"level\": \"%s\", \"msg\": \"%s\"}\n", level, buf); +- nwrite = write(fd, msg, strlen(msg)); +- if (nwrite != strlen(msg)) { ++ nwrite = write_nointr_in_total(fd, msg, strlen(msg)); ++ if (nwrite < 0 || (size_t)nwrite != strlen(msg)) { + return; + } + } +diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c +index 8a0aa142..02ce3c85 100644 +--- a/src/cmd/isulad-shim/process.c ++++ b/src/cmd/isulad-shim/process.c +@@ -283,7 +283,7 @@ static void *do_io_copy(void *data) + break; + } + +- int r_count = read(ioc->fd_from, buf, DEFAULT_IO_COPY_BUF); ++ int r_count = util_read_nointr(ioc->fd_from, buf, DEFAULT_IO_COPY_BUF); + if (r_count == -1) { + if (errno == EAGAIN || errno == EINTR) { + continue; +diff --git a/src/cmd/isulad/main.c b/src/cmd/isulad/main.c +index b17657c5..a75fb189 100644 +--- a/src/cmd/isulad/main.c ++++ b/src/cmd/isulad/main.c +@@ -482,8 +482,8 @@ int check_and_save_pid(const char *fn) + goto out; + } + +- len = (int)write(fd, pidbuf, strlen(pidbuf)); +- if (len < 0) { ++ len = util_write_nointr(fd, pidbuf, strlen(pidbuf)); ++ if (len < 0 || len != strlen(pidbuf)) { + ERROR("Failed to write pid to file:%s: %s", fn, strerror(errno)); + ret = -1; + } +diff --git a/src/daemon/common/selinux_label.c b/src/daemon/common/selinux_label.c +index 24294780..173f3acb 100644 +--- a/src/daemon/common/selinux_label.c ++++ b/src/daemon/common/selinux_label.c +@@ -310,7 +310,7 @@ static int get_random_value(unsigned int range, unsigned int *val) + return -1; + } + +- if (read(fd, &num, sizeof(int)) < 0) { ++ if (util_read_nointr(fd, &num, sizeof(int)) < 0) { + ERROR("Failed to read urandom value\n"); + ret = -1; + goto out; +diff --git a/src/daemon/entry/connect/grpc/grpc_containers_service.cc b/src/daemon/entry/connect/grpc/grpc_containers_service.cc +index c0210ed9..eb79223b 100644 +--- a/src/daemon/entry/connect/grpc/grpc_containers_service.cc ++++ b/src/daemon/entry/connect/grpc/grpc_containers_service.cc +@@ -292,7 +292,8 @@ Status ContainerServiceImpl::RemoteStart(ServerContext *context, + break; + } + const std::string &command = request.stdin(); +- if (write(read_pipe_fd[1], (void *)(command.c_str()), command.length()) < 0) { ++ int nret = util_write_nointr_in_total(read_pipe_fd[1], command.c_str(), command.length()); ++ if (nret < 0 || (size_t)nret != command.length()) { + ERROR("sub write over!"); + break; + } +@@ -407,7 +408,8 @@ public: + } + for (int i = 0; i < request.cmd_size(); i++) { + std::string command = request.cmd(i); +- if (write(m_read_pipe_fd, (void *)(command.c_str()), command.length()) < 0) { ++ int nret = util_write_nointr_in_total(m_read_pipe_fd, command.c_str(), command.length()); ++ if (nret < 0 || (size_t)nret != command.length()) { + ERROR("sub write over!"); + return; + } +@@ -629,7 +631,8 @@ Status ContainerServiceImpl::Attach(ServerContext *context, ServerReaderWriter= PATH_MAX - 1) { + ERROR("Invalid layer id: %s", lid); +@@ -1438,7 +1439,8 @@ static int do_add_checked_layer(const char *lid, int fd, map_t *checked_layers) + (void)memcpy(buf, lid, strlen(lid)); + buf[strlen(lid)] = '\n'; + // save checked layer ids into file +- if (util_write_nointr(fd, buf, strlen(lid) + 1) < 0) { ++ nret = util_write_nointr(fd, buf, strlen(lid) + 1); ++ if (nret < 0 || (size_t)nret != strlen(lid) + 1) { + ERROR("Write checked layer data failed: %s", strerror(errno)); + ret = -1; + goto out; +diff --git a/src/daemon/modules/log/log_gather.c b/src/daemon/modules/log/log_gather.c +index 49facaa2..414c9ad1 100644 +--- a/src/daemon/modules/log/log_gather.c ++++ b/src/daemon/modules/log/log_gather.c +@@ -183,9 +183,9 @@ static int write_into_file(const void *buf, size_t g_log_size) + return -1; + } + } +- ret = (int)write(g_log_fd, buf, g_log_size); +- if (ret <= 0) { +- return ret; ++ ret = util_write_nointr_in_total(g_log_fd, buf, g_log_size); ++ if (ret < 0 || (size_t)ret != g_log_size) { ++ return -1; + } + + write_size += ret; +diff --git a/src/daemon/modules/plugin/plugin.c b/src/daemon/modules/plugin/plugin.c +index 53afeeaf..1c0af368 100644 +--- a/src/daemon/modules/plugin/plugin.c ++++ b/src/daemon/modules/plugin/plugin.c +@@ -618,7 +618,7 @@ static int process_plugin_events(int inotify_fd, const char *plugin_dir) + struct inotify_event *plugin_event = NULL; + char buffer[8192 + 1] = { 0 }; + int action = 0; +- events_length = read(inotify_fd, buffer, 8192); ++ events_length = util_read_nointr(inotify_fd, buffer, 8192); + + if (events_length <= 0) { + ERROR("Failed to wait events"); +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index 76e3bcb7..5463bb1b 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -1363,8 +1363,8 @@ int rt_isula_exec_resize(const char *id, const char *runtime, const rt_exec_resi + goto out; + } + +- count = write(fd, data, RESIZE_DATA_SIZE); +- if (count <= 0) { ++ count = util_write_nointr(fd, data, strlen(data)); ++ if (count < 0 || (size_t)count != strlen(data)) { + ERROR("write exec resize data error"); + ret = -1; + goto out; +diff --git a/src/daemon/modules/service/io_handler.c b/src/daemon/modules/service/io_handler.c +index 893733bc..98c763a4 100644 +--- a/src/daemon/modules/service/io_handler.c ++++ b/src/daemon/modules/service/io_handler.c +@@ -340,7 +340,7 @@ static ssize_t write_to_fd(void *context, const void *data, size_t len) + { + ssize_t ret; + ret = util_write_nointr(*(int *)context, data, len); +- if ((ret <= 0) || (ret != (ssize_t)len)) { ++ if (ret < 0 || (size_t)ret != len) { + ERROR("Failed to write: %s", strerror(errno)); + return -1; + } +diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c +index eeb035a0..cc777411 100644 +--- a/src/daemon/modules/service/service_container.c ++++ b/src/daemon/modules/service/service_container.c +@@ -345,7 +345,7 @@ static int write_env_content(const char *env_path, const char **env, size_t env_ + goto out; + } + nret = util_write_nointr(fd, env_content, strlen(env_content)); +- if (nret < 0 || nret != len - 1) { ++ if (nret < 0 || (size_t)nret != strlen(env_content)) { + SYSERROR("Write env file failed"); + free(env_content); + ret = -1; +diff --git a/src/utils/cutils/utils.c b/src/utils/cutils/utils.c +index de636bcb..f99b28e4 100644 +--- a/src/utils/cutils/utils.c ++++ b/src/utils/cutils/utils.c +@@ -1251,7 +1251,7 @@ int util_generate_random_str(char *id, size_t len) + } + for (i = 0; i < len; i++) { + int nret; +- if (read(fd, &num, sizeof(int)) < 0) { ++ if (util_read_nointr(fd, &num, sizeof(int)) < 0) { + ERROR("Failed to read urandom value"); + close(fd); + return -1; +diff --git a/src/utils/cutils/utils_aes.c b/src/utils/cutils/utils_aes.c +index 1e25ecd3..055a9538 100644 +--- a/src/utils/cutils/utils_aes.c ++++ b/src/utils/cutils/utils_aes.c +@@ -77,7 +77,7 @@ int util_aes_key(const char *key_file, bool create, unsigned char *aeskey) + goto out; + } + +- if (read(fd, aeskey, AES_256_CFB_KEY_LEN) != AES_256_CFB_KEY_LEN) { ++ if (util_read_nointr(fd, aeskey, AES_256_CFB_KEY_LEN) != AES_256_CFB_KEY_LEN) { + ERROR("read key file %s failed: %s", key_file, strerror(errno)); + ret = -1; + goto out; +diff --git a/src/utils/cutils/utils_file.c b/src/utils/cutils/utils_file.c +index cdd712a7..34c5b060 100644 +--- a/src/utils/cutils/utils_file.c ++++ b/src/utils/cutils/utils_file.c +@@ -998,7 +998,7 @@ int util_file2str(const char *filename, char *buf, size_t len) + if (fd == -1) { + return -1; + } +- num_read = (int)read(fd, buf, len - 1); ++ num_read = (int)util_read_nointr(fd, buf, len - 1); + if (num_read <= 0) { + num_read = -1; + } else { +diff --git a/src/utils/tar/util_archive.c b/src/utils/tar/util_archive.c +index 2d56d8a7..7ace2924 100644 +--- a/src/utils/tar/util_archive.c ++++ b/src/utils/tar/util_archive.c +@@ -662,7 +662,7 @@ child_out: + if (ret != 0) { + ERROR("Wait archive_untar_handler failed with error:%s", strerror(errno)); + fcntl(pipe_stderr[0], F_SETFL, O_NONBLOCK); +- if (read(pipe_stderr[0], errbuf, BUFSIZ) < 0) { ++ if (util_read_nointr(pipe_stderr[0], errbuf, BUFSIZ) < 0) { + ERROR("read error message from child failed"); + } + } +@@ -1057,7 +1057,7 @@ child_out: + if (ret != 0) { + ERROR("tar failed"); + fcntl(pipe_for_read[0], F_SETFL, O_NONBLOCK); +- if (read(pipe_for_read[0], errbuf, BUFSIZ) < 0) { ++ if (util_read_nointr(pipe_for_read[0], errbuf, BUFSIZ) < 0) { + ERROR("read error message from child failed"); + } + } +diff --git a/src/utils/tar/util_gzip.c b/src/utils/tar/util_gzip.c +index 5c34d719..2f4750be 100644 +--- a/src/utils/tar/util_gzip.c ++++ b/src/utils/tar/util_gzip.c +@@ -212,7 +212,7 @@ int gzip(const char *filename, size_t len) + return -1; + } + +- size_read = read(pipefd[0], buffer, BUFSIZ); ++ size_read = util_read_nointr(pipefd[0], buffer, BUFSIZ); + close(pipefd[0]); + + if (size_read) { +-- +2.25.1 + diff --git a/0015-support-pull-image-with-digest.patch b/0015-support-pull-image-with-digest.patch new file mode 100644 index 0000000..5650701 --- /dev/null +++ b/0015-support-pull-image-with-digest.patch @@ -0,0 +1,302 @@ +From aaf8dec80eff5390404d7da66dbb229e44c76b12 Mon Sep 17 00:00:00 2001 +From: haozi007 +Date: Thu, 16 Feb 2023 18:22:02 +0800 +Subject: [PATCH 15/53] support pull image with digest + +usage: isula pull busybox@sha256:907ca53d7e2947e849b839b1cd258c98fd3916c60f2e6e70c30edbf741ab6754 + +Signed-off-by: haozi007 +--- + src/daemon/executor/image_cb/image_cb.c | 8 ++++ + src/daemon/modules/image/oci/oci_pull.c | 23 ++++++---- + .../modules/image/oci/registry/registry.c | 2 +- + .../oci/storage/image_store/image_store.c | 7 +++ + src/daemon/modules/image/oci/utils_images.c | 45 +++++++++++++++---- + src/daemon/modules/image/oci/utils_images.h | 2 + + src/utils/cutils/utils_verify.c | 25 ++++++++--- + src/utils/cutils/utils_verify.h | 3 ++ + 8 files changed, 91 insertions(+), 24 deletions(-) + +diff --git a/src/daemon/executor/image_cb/image_cb.c b/src/daemon/executor/image_cb/image_cb.c +index 06de7543..124feb21 100644 +--- a/src/daemon/executor/image_cb/image_cb.c ++++ b/src/daemon/executor/image_cb/image_cb.c +@@ -561,6 +561,14 @@ static int trans_one_image(image_list_images_response *response, size_t image_in + out_image->name = util_strdup_s(im_image->repo_tags[repo_index]); + } + ++ if (out_image->name == NULL && im_image->repo_digests != NULL && im_image->repo_digests_len > 0) { ++ // repo digest must valid, so just get lastest @ ++ char *pod = strrchr(im_image->repo_digests[0], '@'); ++ if (pod != NULL) { ++ out_image->name = util_sub_string(im_image->repo_digests[0], 0, (size_t)(pod - im_image->repo_digests[0])); ++ } ++ } ++ + out_image->target = util_common_calloc_s(sizeof(image_descriptor)); + if (out_image->target == NULL) { + ERROR("Out of memory"); +diff --git a/src/daemon/modules/image/oci/oci_pull.c b/src/daemon/modules/image/oci/oci_pull.c +index 5e774c9e..5b35ca2b 100644 +--- a/src/daemon/modules/image/oci/oci_pull.c ++++ b/src/daemon/modules/image/oci/oci_pull.c +@@ -117,10 +117,19 @@ static int pull_image(const im_pull_request *request, char **name) + options->skip_tls_verify = oci_image_data->insecure_skip_verify_enforce; + insecure_registries = oci_image_data->insecure_registries; + ++ // key of image which save in image-store ++ options->dest_image_name = oci_normalize_image_name(request->image); ++ ++ // add default tag if required ++ with_tag = oci_default_tag(request->image); ++ + host = oci_get_host(request->image); + if (host != NULL) { +- options->image_name = oci_default_tag(request->image); +- options->dest_image_name = oci_normalize_image_name(request->image); ++ // 1. image_name use for split host/tag/name ++ // 2. user for tag of log ++ options->image_name = with_tag; ++ with_tag = NULL; ++ + update_option_insecure_registry(options, insecure_registries, host); + ret = registry_pull(options); + if (ret != 0) { +@@ -141,13 +150,12 @@ static int pull_image(const im_pull_request *request, char **name) + } + host = oci_host_from_mirror(*mirror); + update_option_insecure_registry(options, insecure_registries, host); +- with_tag = oci_default_tag(request->image); ++ // add current mirror to image name ++ free(options->image_name); + options->image_name = oci_add_host(host, with_tag); +- free(with_tag); +- with_tag = NULL; + free(host); + host = NULL; +- options->dest_image_name = oci_normalize_image_name(request->image); ++ + ret = registry_pull(options); + if (ret != 0) { + continue; +@@ -159,10 +167,9 @@ static int pull_image(const im_pull_request *request, char **name) + *name = util_strdup_s(options->dest_image_name); + + out: ++ free(with_tag); + free(host); +- host = NULL; + free_registry_pull_options(options); +- options = NULL; + + return ret; + } +diff --git a/src/daemon/modules/image/oci/registry/registry.c b/src/daemon/modules/image/oci/registry/registry.c +index 143de6e4..62d0c35e 100644 +--- a/src/daemon/modules/image/oci/registry/registry.c ++++ b/src/daemon/modules/image/oci/registry/registry.c +@@ -1861,7 +1861,7 @@ static int prepare_pull_desc(pull_descriptor *desc, registry_pull_options *optio + } + + if (!util_valid_image_name(options->dest_image_name)) { +- ERROR("Invalid dest image name %s", options->image_name); ++ ERROR("Invalid dest image name %s", options->dest_image_name); + isulad_try_set_error_message("Invalid image name"); + return -1; + } +diff --git a/src/daemon/modules/image/oci/storage/image_store/image_store.c b/src/daemon/modules/image/oci/storage/image_store/image_store.c +index 39bda87d..cf1e88ff 100644 +--- a/src/daemon/modules/image/oci/storage/image_store/image_store.c ++++ b/src/daemon/modules/image/oci/storage/image_store/image_store.c +@@ -1979,6 +1979,7 @@ static int resort_image_names(const char **names, size_t names_len, char **first + MAX_IMAGE_NAME_LENGTH - MAX_IMAGE_DIGEST_LENGTH); + } + ++ // TODO: maybe should support other digest + if (prefix != NULL && strcmp(prefix, DIGEST_PREFIX) == 0) { + if (util_array_append(image_digests, names[i]) != 0) { + ERROR("Failed to append image to digest: %s", names[i]); +@@ -2172,6 +2173,7 @@ static int get_image_repo_digests(char ***old_repo_digests, char **image_tags, i + goto out; + } + ++ // get repo digest from images which with tag + if (pack_repo_digest(old_repo_digests, (const char **)image_tags, digest, repo_digests) != 0) { + ERROR("Failed to pack repo digest"); + ret = -1; +@@ -2194,12 +2196,17 @@ static int pack_image_tags_and_repo_digest(image_t *img, imagetool_image *info) + char *image_digest = NULL; + char **repo_digests = NULL; + ++ // get names from image-store names: ++ // 1. image names with tag; ++ // 2. image names with digests; ++ // 3. get first image name, current unused; + if (resort_image_names((const char **)img->simage->names, img->simage->names_len, &name, &tags, &digests) != 0) { + ERROR("Failed to resort image names"); + ret = -1; + goto out; + } + ++ // update repo digests from tags + if (get_image_repo_digests(&digests, tags, img, &image_digest, &repo_digests) != 0) { + ERROR("Failed to get image repo digests"); + ret = -1; +diff --git a/src/daemon/modules/image/oci/utils_images.c b/src/daemon/modules/image/oci/utils_images.c +index 9e7bb16f..ad7fe0f4 100644 +--- a/src/daemon/modules/image/oci/utils_images.c ++++ b/src/daemon/modules/image/oci/utils_images.c +@@ -42,6 +42,26 @@ + // nanos of 2038-01-19T03:14:07, the max valid linux time + #define MAX_NANOS 2147483647000000000 + ++char *oci_image_digest_pos(const char *name) ++{ ++ char *pos = NULL; ++ ++ if (name == NULL) { ++ return NULL; ++ } ++ ++ pos = strrchr(name, '@'); ++ if (pos == NULL) { ++ return NULL; ++ } ++ ++ if (util_reg_match(__DIGESTPattern, pos) != 0) { ++ return NULL; ++ } ++ ++ return pos; ++} ++ + char *get_last_part(char **parts) + { + char *last_part = NULL; +@@ -98,6 +118,7 @@ char *oci_default_tag(const char *name) + } + + last_part = get_last_part(parts); ++ // will pass image name with digest and with tag + if (last_part != NULL && strrchr(last_part, ':') == NULL) { + add_default_tag = DEFAULT_TAG; + } +@@ -181,9 +202,9 @@ char *oci_normalize_image_name(const char *name) + return result; + } + +-int oci_split_image_name(const char *image_name, char **host, char **name, char **tag) ++int oci_split_image_name(const char *image_name, char **host, char **name, char **tag_digest) + { +- char *tag_pos = NULL; ++ char *tag_digest_pos = NULL; + char *name_pos = NULL; + char *tmp_image_name = NULL; + +@@ -193,18 +214,24 @@ int oci_split_image_name(const char *image_name, char **host, char **name, char + } + + tmp_image_name = util_strdup_s(image_name); +- tag_pos = util_tag_pos(tmp_image_name); +- if (tag_pos != NULL) { +- *tag_pos = 0; +- tag_pos++; +- if (tag != NULL) { +- *tag = util_strdup_s(tag_pos); ++ ++ // check digest first ++ tag_digest_pos = oci_image_digest_pos(tmp_image_name); ++ if (tag_digest_pos == NULL) { ++ tag_digest_pos = util_tag_pos(tmp_image_name); ++ } ++ ++ if (tag_digest_pos != NULL) { ++ *tag_digest_pos = '\0'; ++ tag_digest_pos++; ++ if (tag_digest != NULL) { ++ *tag_digest = util_strdup_s(tag_digest_pos); + } + } + + name_pos = strchr(tmp_image_name, '/'); + if (name_pos != NULL) { +- *name_pos = 0; ++ *name_pos = '\0'; + name_pos++; + if (name != NULL) { + *name = util_strdup_s(name_pos); +diff --git a/src/daemon/modules/image/oci/utils_images.h b/src/daemon/modules/image/oci/utils_images.h +index daa8c040..97879e41 100644 +--- a/src/daemon/modules/image/oci/utils_images.h ++++ b/src/daemon/modules/image/oci/utils_images.h +@@ -59,6 +59,8 @@ char *oci_get_isulad_tmpdir(const char *root_dir); + int makesure_isulad_tmpdir_perm_right(const char *root_dir); + char *get_hostname_to_strip(); + ++char *oci_image_digest_pos(const char *name); ++ + #ifdef __cplusplus + } + #endif +diff --git a/src/utils/cutils/utils_verify.c b/src/utils/cutils/utils_verify.c +index d39d8da5..5868e890 100644 +--- a/src/utils/cutils/utils_verify.c ++++ b/src/utils/cutils/utils_verify.c +@@ -359,7 +359,7 @@ cleanup: + bool util_valid_image_name(const char *name) + { + char *copy = NULL; +- char *tag_pos = NULL; ++ char *check_pos = NULL; + bool bret = false; + + if (name == NULL) { +@@ -372,13 +372,26 @@ bool util_valid_image_name(const char *name) + } + + copy = util_strdup_s(name); +- tag_pos = util_tag_pos(copy); +- if (tag_pos != NULL) { +- if (util_reg_match(__TagPattern, tag_pos)) { ++ ++ // 1. first, check digest or not ++ check_pos = strrchr(copy, '@'); ++ if (check_pos != NULL) { ++ // image name with digest ++ if (util_reg_match(__DIGESTPattern, check_pos)) { + goto cleanup; + } +- +- *tag_pos = '\0'; ++ *check_pos = '\0'; ++ } else { ++ // image name without digest ++ // 2. check tag or not ++ check_pos = util_tag_pos(copy); ++ if (check_pos != NULL) { ++ if (util_reg_match(__TagPattern, check_pos)) { ++ goto cleanup; ++ } ++ ++ *check_pos = '\0'; ++ } + } + + if (util_reg_match(__NamePattern, copy)) { +diff --git a/src/utils/cutils/utils_verify.h b/src/utils/cutils/utils_verify.h +index a885250f..ad4466ef 100644 +--- a/src/utils/cutils/utils_verify.h ++++ b/src/utils/cutils/utils_verify.h +@@ -33,6 +33,9 @@ extern "C" { + "^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])" \ + "((\\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9]))+)?(:[0-9]+)?/)?[a-z0-9]" \ + "+((([._]|__|[-]*)[a-z0-9]+)+)?((/[a-z0-9]+((([._]|__|[-]*)[a-z0-9]+)+)?)+)?$" ++ ++#define __DIGESTPattern "@[a-z0-9]+:[a-z0-9]{32,}" ++ + #define VALID_VOLUME_NAME "[a-zA-Z0-9][a-zA-Z0-9_.-]{1,63}" + + extern const char *g_all_caps[]; +-- +2.25.1 + diff --git a/0016-isulad-shim-support-execSync-with-timeout.patch b/0016-isulad-shim-support-execSync-with-timeout.patch new file mode 100644 index 0000000..a4e8c98 --- /dev/null +++ b/0016-isulad-shim-support-execSync-with-timeout.patch @@ -0,0 +1,186 @@ +From 32dbf764fd5b7f6941c49750b49dbba253bd3234 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Mon, 13 Feb 2023 15:36:58 +0800 +Subject: [PATCH 16/53] isulad-shim support execSync with timeout + +Signed-off-by: zhongtao +--- + .../executor/container_cb/execution_stream.c | 2 +- + .../modules/runtime/isula/isula_rt_ops.c | 37 +++++++++++++++--- + src/utils/cutils/utils.c | 39 +++++++++++++++++++ + src/utils/cutils/utils.h | 5 +++ + 4 files changed, 77 insertions(+), 6 deletions(-) + +diff --git a/src/daemon/executor/container_cb/execution_stream.c b/src/daemon/executor/container_cb/execution_stream.c +index fde0335e..1a7353b5 100644 +--- a/src/daemon/executor/container_cb/execution_stream.c ++++ b/src/daemon/executor/container_cb/execution_stream.c +@@ -161,7 +161,7 @@ static int container_exec_cb(const container_exec_request *request, container_ex + + if (exec_container(cont, request, *response, stdinfd, stdout_handler, stderr_handler) != 0) { + ret = -1; +- goto out; ++ goto pack_err_response; + } + + goto out; +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index 5463bb1b..6f2b4f7d 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -205,6 +205,10 @@ static void show_shim_runtime_errlog(const char *workdir) + char buf1[SHIM_LOG_SIZE] = { 0 }; + char buf2[SHIM_LOG_SIZE] = { 0 }; + ++ if (g_isulad_errmsg != NULL) { ++ return; ++ } ++ + get_err_message(buf1, sizeof(buf1), workdir, "shim-log.json"); + get_err_message(buf2, sizeof(buf2), workdir, "log.json"); + ERROR("shim-log: %s", buf1); +@@ -686,8 +690,29 @@ static int status_to_exit_code(int status) + return exit_code; + } + ++static int try_wait_pid(pid_t pid) ++{ ++ if (waitpid(pid, NULL, WNOHANG) == pid) { ++ return 0; ++ } ++ ++ return 1; ++} ++ ++static void kill_and_show_err(pid_t pid) ++{ ++ int nret = 0; ++ kill(pid, SIGKILL); ++ // wait atmost 0.5 seconds ++ DO_RETRY_CALL(5, 100000, nret, try_wait_pid, pid); ++ if (nret != 0) { ++ WARN("Fail to wait isulad-shim"); ++ } ++ isulad_set_error_message("Exec container error;exec timeout"); ++} ++ + static int shim_create(bool fg, const char *id, const char *workdir, const char *bundle, const char *runtime_cmd, +- int *exit_code) ++ int *exit_code, const int64_t timeout) + { + pid_t pid = 0; + int exec_fd[2] = { -1, -1 }; +@@ -778,7 +803,7 @@ realexec: + goto out; + } + +- status = util_wait_for_pid_status(pid); ++ status = util_waitpid_with_timeout(pid, timeout, kill_and_show_err); + if (status < 0) { + ERROR("failed wait shim-parent %d exit %s", pid, strerror(errno)); + ret = -1; +@@ -792,7 +817,9 @@ realexec: + out: + if (ret != 0) { + show_shim_runtime_errlog(workdir); +- kill(pid, SIGKILL); /* can kill other process? */ ++ if (timeout <= 0) { ++ kill(pid, SIGKILL); /* can kill other process? */ ++ } + } + + return ret; +@@ -901,7 +928,7 @@ int rt_isula_create(const char *id, const char *runtime, const rt_create_params_ + } + + get_runtime_cmd(runtime, &cmd); +- ret = shim_create(false, id, workdir, params->bundle, cmd, NULL); ++ ret = shim_create(false, id, workdir, params->bundle, cmd, NULL, -1); + if (ret != 0) { + runtime_call_delete_force(workdir, runtime, id); + ERROR("%s: failed create shim process", id); +@@ -1173,7 +1200,7 @@ int rt_isula_exec(const char *id, const char *runtime, const rt_exec_params_t *p + } + + get_runtime_cmd(runtime, &cmd); +- ret = shim_create(fg_exec(params), id, workdir, bundle, cmd, exit_code); ++ ret = shim_create(fg_exec(params), id, workdir, bundle, cmd, exit_code, params->timeout); + if (ret != 0) { + ERROR("%s: failed create shim process for exec %s", id, exec_id); + goto errlog_out; +diff --git a/src/utils/cutils/utils.c b/src/utils/cutils/utils.c +index f99b28e4..2c4c01e4 100644 +--- a/src/utils/cutils/utils.c ++++ b/src/utils/cutils/utils.c +@@ -311,6 +311,45 @@ rep: + return 0; + } + ++/* ++ * If timeout <= 0, blocking wait pid. ++ * If timeout > 0, non-blocking wait pid with timeout. ++ * When waitpid timeout, calling handle_timeout_callback_t. ++ */ ++int util_waitpid_with_timeout(pid_t pid, const int64_t timeout, handle_timeout_callback_t cb) ++{ ++ int nret = 0; ++ time_t start_time = time(NULL); ++ time_t end_time; ++ double interval; ++ ++ if (timeout <= 0) { ++ return util_wait_for_pid_status(pid); ++ } ++ ++ for (;;) { ++ nret = waitpid(pid, NULL, WNOHANG); ++ if (nret == pid) { ++ break; ++ } ++ if (nret == -1 && errno != EINTR) { ++ return -1; ++ } ++ end_time = time(NULL); ++ interval = difftime(end_time, start_time); ++ if (nret == 0 && interval >= timeout) { ++ INFO("Wait %d timeout", pid); ++ if (cb != NULL) { ++ cb(pid); ++ } ++ return -1; ++ } ++ // sleep some time instead to avoid cpu full running and then retry. ++ sleep(0.1); ++ } ++ return 0; ++} ++ + int util_wait_for_pid_status(pid_t pid) + { + int st; +diff --git a/src/utils/cutils/utils.h b/src/utils/cutils/utils.h +index 6261dc05..01107605 100644 +--- a/src/utils/cutils/utils.h ++++ b/src/utils/cutils/utils.h +@@ -302,6 +302,9 @@ typedef struct _proc_t { + processor; /* current (or most recent?) CPU */ + } proc_t; + ++// handle waitpid timeout. ++typedef void(*handle_timeout_callback_t)(pid_t pid); ++ + struct signame { + int num; + const char *name; +@@ -329,6 +332,8 @@ char *util_strdup_s(const char *src); + + int util_wait_for_pid(pid_t pid); + ++int util_waitpid_with_timeout(pid_t pid, const int64_t timeout, handle_timeout_callback_t cb); ++ + void util_contain_errmsg(const char *errmsg, int *exit_code); + + char *util_short_digest(const char *digest); +-- +2.25.1 + diff --git a/0017-Refine-the-commit-info.patch b/0017-Refine-the-commit-info.patch new file mode 100644 index 0000000..2b46810 --- /dev/null +++ b/0017-Refine-the-commit-info.patch @@ -0,0 +1,30 @@ +From a1c06194fea99d1011551fd84b1fb1f28b974170 Mon Sep 17 00:00:00 2001 +From: sailorvii +Date: Tue, 21 Feb 2023 02:40:50 +0000 +Subject: [PATCH 17/53] Refine the commit info. + +--- + docs/design/detailed/Network/native_network_design_zh.md | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/docs/design/detailed/Network/native_network_design_zh.md b/docs/design/detailed/Network/native_network_design_zh.md +index 0ca0f850..27c10c3e 100644 +--- a/docs/design/detailed/Network/native_network_design_zh.md ++++ b/docs/design/detailed/Network/native_network_design_zh.md +@@ -299,10 +299,10 @@ cache:::unFinish + + ## 4.2 adaptor模块 + +-1. 查看 CRI adapter 模块的设计文档： [CRI_adapter_design](./cni_operator_design_zh.md) 。 ++1. 查看 CRI adapter 模块的设计文档： [CRI_adapter_design](./CRI_adapter_design_zh.md) 。 + + 2. 查看 native network adapter 模块的设计文档： [native_network_adapter_design](./native_network_adapter_design_zh.md) 。 + + ## 4.3 cni-operator模块 + +-- 查看 cni operator 模块的设计文档： [cni_operator_design](./cni_operator_design_zh.md) 。 +\ No newline at end of file ++- 查看 cni operator 模块的设计文档： [cni_operator_design](./cni_operator_design_zh.md) 。 +-- +2.25.1 + diff --git a/0018-Refine-typo-of-word-container.patch b/0018-Refine-typo-of-word-container.patch new file mode 100644 index 0000000..34f8397 --- /dev/null +++ b/0018-Refine-typo-of-word-container.patch @@ -0,0 +1,85 @@ +From 53ec87b8c5224b1069bef50d09403c53fb48640f Mon Sep 17 00:00:00 2001 +From: sailorvii +Date: Tue, 21 Feb 2023 06:50:21 +0000 +Subject: [PATCH 18/53] Refine typo of word "container". + +--- + src/daemon/executor/container_cb/execution_create.c | 2 +- + src/daemon/modules/api/container_api.h | 2 +- + src/daemon/modules/container/container_unix.c | 2 +- + src/daemon/modules/service/service_network.c | 6 +++--- + 4 files changed, 6 insertions(+), 6 deletions(-) + +diff --git a/src/daemon/executor/container_cb/execution_create.c b/src/daemon/executor/container_cb/execution_create.c +index 4abc89c7..e8f74f1b 100644 +--- a/src/daemon/executor/container_cb/execution_create.c ++++ b/src/daemon/executor/container_cb/execution_create.c +@@ -1481,7 +1481,7 @@ int container_create_cb(const container_create_request *request, container_creat + goto umount_channel; + } + +- if (container_v2_spec_merge_contaner_spec(v2_spec) != 0) { ++ if (container_v2_spec_merge_container_spec(v2_spec) != 0) { + ERROR("Failed to merge container settings"); + cc = ISULAD_ERR_EXEC; + goto umount_channel; +diff --git a/src/daemon/modules/api/container_api.h b/src/daemon/modules/api/container_api.h +index 270d6da6..1511db78 100644 +--- a/src/daemon/modules/api/container_api.h ++++ b/src/daemon/modules/api/container_api.h +@@ -175,7 +175,7 @@ void container_unlock(container_t *cont); + + char *container_get_env_nolock(const container_t *cont, const char *key); + +-int container_v2_spec_merge_contaner_spec(container_config_v2_common_config *v2_spec); ++int container_v2_spec_merge_container_spec(container_config_v2_common_config *v2_spec); + + char *container_get_command(const container_t *cont); + +diff --git a/src/daemon/modules/container/container_unix.c b/src/daemon/modules/container/container_unix.c +index adc11be7..1a252b92 100644 +--- a/src/daemon/modules/container/container_unix.c ++++ b/src/daemon/modules/container/container_unix.c +@@ -470,7 +470,7 @@ out: + } + + /* container merge basic v2 spec info */ +-int container_v2_spec_merge_contaner_spec(container_config_v2_common_config *v2_spec) ++int container_v2_spec_merge_container_spec(container_config_v2_common_config *v2_spec) + { + int ret = 0; + int i = 0; +diff --git a/src/daemon/modules/service/service_network.c b/src/daemon/modules/service/service_network.c +index 2e7fa28c..2d5f2f6e 100644 +--- a/src/daemon/modules/service/service_network.c ++++ b/src/daemon/modules/service/service_network.c +@@ -569,7 +569,7 @@ err_out: + return NULL; + } + +-static container_network_settings *dup_contaner_network_settings(const container_network_settings *settings) ++static container_network_settings *dup_container_network_settings(const container_network_settings *settings) + { + char *jstr = NULL; + container_network_settings *res = NULL; +@@ -1278,7 +1278,7 @@ static int update_container_network_settings(container_t *cont, const cni_anno_p + bool to_disk = false; + container_network_settings *backup = NULL; + +- backup = dup_contaner_network_settings(cont->network_settings); ++ backup = dup_container_network_settings(cont->network_settings); + if (backup == NULL) { + ERROR("Failed to dup container network settings"); + return -1; +@@ -1509,7 +1509,7 @@ static int drop_container_network_settings(container_t *cont) + return -1; + } + +- backup = dup_contaner_network_settings(cont->network_settings); ++ backup = dup_container_network_settings(cont->network_settings); + if (backup == NULL) { + ERROR("Failed to dup container network settings"); + return -1; +-- +2.25.1 + diff --git a/0019-cleancode-for-read-write.patch b/0019-cleancode-for-read-write.patch new file mode 100644 index 0000000..3fb9e1a --- /dev/null +++ b/0019-cleancode-for-read-write.patch @@ -0,0 +1,54 @@ +From f9224d47ddc4193678f7ffe501be144fedff0102 Mon Sep 17 00:00:00 2001 +From: zhangxiaoyu +Date: Mon, 20 Feb 2023 17:28:33 +0800 +Subject: [PATCH 19/53] cleancode for read/write + +Signed-off-by: zhangxiaoyu +--- + src/cmd/isulad-shim/process.c | 2 +- + src/cmd/isulad/main.c | 2 +- + src/daemon/entry/connect/grpc/grpc_containers_service.cc | 2 +- + 3 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c +index 02ce3c85..8a0aa142 100644 +--- a/src/cmd/isulad-shim/process.c ++++ b/src/cmd/isulad-shim/process.c +@@ -283,7 +283,7 @@ static void *do_io_copy(void *data) + break; + } + +- int r_count = util_read_nointr(ioc->fd_from, buf, DEFAULT_IO_COPY_BUF); ++ int r_count = read(ioc->fd_from, buf, DEFAULT_IO_COPY_BUF); + if (r_count == -1) { + if (errno == EAGAIN || errno == EINTR) { + continue; +diff --git a/src/cmd/isulad/main.c b/src/cmd/isulad/main.c +index a75fb189..0cdbfb53 100644 +--- a/src/cmd/isulad/main.c ++++ b/src/cmd/isulad/main.c +@@ -483,7 +483,7 @@ int check_and_save_pid(const char *fn) + } + + len = util_write_nointr(fd, pidbuf, strlen(pidbuf)); +- if (len < 0 || len != strlen(pidbuf)) { ++ if (len < 0 || (size_t)len != strlen(pidbuf)) { + ERROR("Failed to write pid to file:%s: %s", fn, strerror(errno)); + ret = -1; + } +diff --git a/src/daemon/entry/connect/grpc/grpc_containers_service.cc b/src/daemon/entry/connect/grpc/grpc_containers_service.cc +index eb79223b..7340c3ed 100644 +--- a/src/daemon/entry/connect/grpc/grpc_containers_service.cc ++++ b/src/daemon/entry/connect/grpc/grpc_containers_service.cc +@@ -409,7 +409,7 @@ public: + for (int i = 0; i < request.cmd_size(); i++) { + std::string command = request.cmd(i); + int nret = util_write_nointr_in_total(m_read_pipe_fd, command.c_str(), command.length()); +- if (nret < 0 || (size_t)nret != command.length()) { ++ if (nret < 0 || (size_t)nret != command.length()) { + ERROR("sub write over!"); + return; + } +-- +2.25.1 + diff --git a/0020-add-crictl-timeout-and-sync-for-CI.patch b/0020-add-crictl-timeout-and-sync-for-CI.patch new file mode 100644 index 0000000..dbab1fd --- /dev/null +++ b/0020-add-crictl-timeout-and-sync-for-CI.patch @@ -0,0 +1,127 @@ +From 7941e0fcd8d7b8edb303a1661233fd9688c46819 Mon Sep 17 00:00:00 2001 +From: zhangxiaoyu +Date: Mon, 20 Feb 2023 15:42:40 +0800 +Subject: [PATCH 20/53] add crictl timeout and sync for CI + +Signed-off-by: zhangxiaoyu +--- + CI/test_cases/container_cases/bigdata_stream.sh | 7 +++++++ + CI/test_cases/container_cases/bigdata_stream_runc.sh | 4 ++++ + CI/test_cases/helpers.sh | 5 ++++- + 3 files changed, 15 insertions(+), 1 deletion(-) + +diff --git a/CI/test_cases/container_cases/bigdata_stream.sh b/CI/test_cases/container_cases/bigdata_stream.sh +index 7e74d700..3bfc2d50 100755 +--- a/CI/test_cases/container_cases/bigdata_stream.sh ++++ b/CI/test_cases/container_cases/bigdata_stream.sh +@@ -124,6 +124,7 @@ function test_concurrent_bigdata_stream() + pids[${#pids[@]}]=$! + done + wait ${pids[*]// /|} ++ sync && sync + + for index in $(seq 1 5); do + ls -l /home/iocopy_stream_data_500M_$index +@@ -151,6 +152,7 @@ function test_concurrent_bigdata_stream_without_pty() + pids[${#pids[@]}]=$! + done + wait ${pids[*]// /|} ++ sync && sync + + for index in $(seq 1 5); do + ls -l /home/iocopy_stream_data_500M_$index +@@ -209,6 +211,7 @@ function test_stream_with_stop_client() + kill -18 $pid + + wait $pid ++ sync && sync + + ls -l /home/iocopy_stream_data_500M + total_size=$(stat -c"%s" /home/iocopy_stream_data_500M) +@@ -254,6 +257,7 @@ function test_stream_with_stop_attach() + kill -18 $pid + + wait $exec_pid ++ sync && sync + + ls -l /home/iocopy_stream_data_500M + total_size=$(stat -c"%s" /home/iocopy_stream_data_500M) +@@ -299,6 +303,7 @@ function test_stream_with_stop_lxc_monitor() + kill -18 $pid + + wait $exec_pid ++ sync && sync + + ls -l /home/iocopy_stream_data_500M + total_size=$(stat -c"%s" /home/iocopy_stream_data_500M) +@@ -347,6 +352,7 @@ function test_stream_with_stop_isulad() + kill -18 $(cat /var/run/isulad.pid) + + wait $pid ++ sync && sync + + ls -l /home/iocopy_stream_data_500M + total_size=$(stat -c"%s" /home/iocopy_stream_data_500M) +@@ -397,6 +403,7 @@ function test_stream_with_runc() + isula exec -it $RUNCID cat test_500M > /home/iocopy_stream_data_500M + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to cat bigdata" && ((ret++)) + ++ sync && sync + total_size=$(stat -c"%s" /home/iocopy_stream_data_500M) + [[ $total_size -ne 524288000 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stream iocopy loss data" && ((ret++)) + +diff --git a/CI/test_cases/container_cases/bigdata_stream_runc.sh b/CI/test_cases/container_cases/bigdata_stream_runc.sh +index 6933bef0..3a384cd8 100755 +--- a/CI/test_cases/container_cases/bigdata_stream_runc.sh ++++ b/CI/test_cases/container_cases/bigdata_stream_runc.sh +@@ -117,6 +117,7 @@ function test_cat_bigdata() + pids[${#pids[@]}]=$! + done + wait ${pids[*]// /|} ++ sync && sync + + for index in $(seq 1 5); do + ls -l /home/iocopy_stream_data_500M_$index +@@ -144,6 +145,7 @@ function test_cat_bigdata_without_pty() + pids[${#pids[@]}]=$! + done + wait ${pids[*]// /|} ++ sync && sync + + for index in $(seq 1 5); do + ls -l /home/iocopy_stream_data_500M_$index +@@ -173,6 +175,7 @@ function test_stream_with_stop_client() + kill -18 $pid + + wait $pid ++ sync && sync + + ls -l /home/iocopy_stream_data_500M + total_size=$(stat -c"%s" /home/iocopy_stream_data_500M) +@@ -217,6 +220,7 @@ function test_stream_with_stop_isulad() + kill -18 $(cat /var/run/isulad.pid) + + wait $pid ++ sync && sync + + ls -l /home/iocopy_stream_data_500M + total_size=$(stat -c"%s" /home/iocopy_stream_data_500M) +diff --git a/CI/test_cases/helpers.sh b/CI/test_cases/helpers.sh +index bba4e7e3..5ea4ff94 100755 +--- a/CI/test_cases/helpers.sh ++++ b/CI/test_cases/helpers.sh +@@ -69,7 +69,10 @@ function testcontainer() { + + function crictl() { + CRICTL=$(which crictl) +- "$CRICTL" -i unix:///var/run/isulad.sock -r unix:///var/run/isulad.sock "$@" ++ # Default timeout is 2s. ++ # In some high IO testcase, isulad handle CRI request time maybe more than 2s. ++ # And the crictl will print error message "context deadline exceeded" ++ "$CRICTL" -i unix:///var/run/isulad.sock -r unix:///var/run/isulad.sock --timeout 5s "$@" + } + + function msg_ok() +-- +2.25.1 + diff --git a/0021-unlock-m_podsLock-if-new-failed.patch b/0021-unlock-m_podsLock-if-new-failed.patch new file mode 100644 index 0000000..c0dc084 --- /dev/null +++ b/0021-unlock-m_podsLock-if-new-failed.patch @@ -0,0 +1,27 @@ +From f6243bb672bca8fd2e32752480aa92dc8f97adc9 Mon Sep 17 00:00:00 2001 +From: zhangxiaoyu +Date: Wed, 22 Feb 2023 10:43:52 +0800 +Subject: [PATCH 21/53] unlock m_podsLock if new failed + +Signed-off-by: zhangxiaoyu +--- + src/daemon/entry/cri/network_plugin.cc | 3 +++ + 1 file changed, 3 insertions(+) + +diff --git a/src/daemon/entry/cri/network_plugin.cc b/src/daemon/entry/cri/network_plugin.cc +index f6a155c3..4a119d6b 100644 +--- a/src/daemon/entry/cri/network_plugin.cc ++++ b/src/daemon/entry/cri/network_plugin.cc +@@ -345,6 +345,9 @@ void PluginManager::Lock(const std::string &fullPodName, Errors &error) + auto tmpLock = std::unique_ptr(new (std::nothrow) PodLock()); + if (tmpLock == nullptr) { + error.SetError("Out of memory"); ++ if (pthread_mutex_unlock(&m_podsLock) != 0) { ++ error.SetError("plugin manager unlock failed"); ++ } + return; + } + lock = tmpLock.get(); +-- +2.25.1 + diff --git a/0022-Update-CRI.patch b/0022-Update-CRI.patch new file mode 100644 index 0000000..713338e --- /dev/null +++ b/0022-Update-CRI.patch @@ -0,0 +1,73 @@ +From 65c3b3c803128f92113f9f21bf41da1ad56017c8 Mon Sep 17 00:00:00 2001 +From: shijiaqi1 +Date: Wed, 8 Feb 2023 13:31:36 +0800 +Subject: [PATCH 22/53] Update-CRI + +--- + .../cri/cri_container_manager_service.cc | 19 +++++++++++++++++++ + src/daemon/entry/cri/cri_helpers.cc | 19 +++++++++++++++++++ + 2 files changed, 38 insertions(+) + +diff --git a/src/daemon/entry/cri/cri_container_manager_service.cc b/src/daemon/entry/cri/cri_container_manager_service.cc +index 710556a3..b02367c8 100644 +--- a/src/daemon/entry/cri/cri_container_manager_service.cc ++++ b/src/daemon/entry/cri/cri_container_manager_service.cc +@@ -1179,6 +1179,25 @@ void ContainerManagerService::UpdateContainerResources(const std::string &contai + if (!resources.cpuset_mems().empty()) { + hostconfig->cpuset_mems = util_strdup_s(resources.cpuset_mems().c_str()); + } ++ if (resources.hugepage_limits_size() != 0) { ++ hostconfig->hugetlbs = (host_config_hugetlbs_element **)util_smart_calloc_s( ++ sizeof(host_config_hugetlbs_element *), resources.hugepage_limits_size()); ++ if (hostconfig->hugetlbs == nullptr) { ++ error.SetError("Out of memory"); ++ return; ++ } ++ for (int i = 0; i < resources.hugepage_limits_size(); i++) { ++ hostconfig->hugetlbs[i] = ++ (host_config_hugetlbs_element *)util_common_calloc_s(sizeof(host_config_hugetlbs_element)); ++ if (hostconfig->hugetlbs[i] == nullptr) { ++ error.SetError("Out of memory"); ++ goto cleanup; ++ } ++ hostconfig->hugetlbs[i]->page_size = util_strdup_s(resources.hugepage_limits(i).page_size().c_str()); ++ hostconfig->hugetlbs[i]->limit = resources.hugepage_limits(i).limit(); ++ hostconfig->hugetlbs_len++; ++ } ++ } + + request->host_config = host_config_generate_json(hostconfig, &ctx, &perror); + if (request->host_config == nullptr) { +diff --git a/src/daemon/entry/cri/cri_helpers.cc b/src/daemon/entry/cri/cri_helpers.cc +index 2f6dcf78..6d59ec11 100644 +--- a/src/daemon/entry/cri/cri_helpers.cc ++++ b/src/daemon/entry/cri/cri_helpers.cc +@@ -461,6 +461,25 @@ void UpdateCreateConfig(container_config *createConfig, host_config *hc, + } + } + hc->unified = unified; ++ if (rOpts.hugepage_limits_size() != 0) { ++ hc->hugetlbs = (host_config_hugetlbs_element **)util_smart_calloc_s(sizeof(host_config_hugetlbs_element *), ++ rOpts.hugepage_limits_size()); ++ if (hc->hugetlbs == nullptr) { ++ error.SetError("Out of memory"); ++ return; ++ } ++ for (int i = 0; i < rOpts.hugepage_limits_size(); i++) { ++ hc->hugetlbs[i] = ++ (host_config_hugetlbs_element *)util_common_calloc_s(sizeof(host_config_hugetlbs_element)); ++ if (hc->hugetlbs[i] == nullptr) { ++ error.SetError("Out of memory"); ++ return; ++ } ++ hc->hugetlbs[i]->page_size = util_strdup_s(rOpts.hugepage_limits(i).page_size().c_str()); ++ hc->hugetlbs[i]->limit = rOpts.hugepage_limits(i).limit(); ++ hc->hugetlbs_len++; ++ } ++ } + } + createConfig->open_stdin = config.stdin(); + createConfig->tty = config.tty(); +-- +2.25.1 + diff --git a/0023-add-cgroup-cpu-ut.patch b/0023-add-cgroup-cpu-ut.patch new file mode 100644 index 0000000..9cafd62 --- /dev/null +++ b/0023-add-cgroup-cpu-ut.patch @@ -0,0 +1,70 @@ +From dd238f80fd754b135c7b0e6a6535d45cf57c0b82 Mon Sep 17 00:00:00 2001 +From: songbuhuang <544824346@qq.com> +Date: Wed, 22 Feb 2023 16:46:45 +0800 +Subject: [PATCH 23/53] add cgroup cpu ut + +Signed-off-by: songbuhuang <544824346@qq.com> +--- + test/cgroup/cpu/CMakeLists.txt | 2 ++ + test/cgroup/cpu/cgroup_cpu_ut.cc | 21 +++++++++++++++++++++ + 2 files changed, 23 insertions(+) + +diff --git a/test/cgroup/cpu/CMakeLists.txt b/test/cgroup/cpu/CMakeLists.txt +index 6a8af719..b619fb59 100644 +--- a/test/cgroup/cpu/CMakeLists.txt ++++ b/test/cgroup/cpu/CMakeLists.txt +@@ -22,8 +22,10 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/config + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../mocks + ) + ++set_target_properties(${EXE} PROPERTIES LINK_FLAGS "-Wl,--wrap,util_common_calloc_s") + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) + set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cgroup/cpu/cgroup_cpu_ut.cc b/test/cgroup/cpu/cgroup_cpu_ut.cc +index 032ad656..7d23d014 100644 +--- a/test/cgroup/cpu/cgroup_cpu_ut.cc ++++ b/test/cgroup/cpu/cgroup_cpu_ut.cc +@@ -16,10 +16,18 @@ + #include + #include + #include ++#include + #include "daemon_arguments.h" + #include "isulad_config.h" ++#include "mock.h" ++#include "sysinfo.h" + #include "utils.h" + ++extern "C" { ++ DECLARE_WRAPPER(util_common_calloc_s, void *, (size_t size)); ++ DEFINE_WRAPPER(util_common_calloc_s, void *, (size_t size), (size)); ++} ++ + struct service_arguments *new_args(int64_t cpu_rt_period, int64_t cpu_rt_runtime) + { + struct service_arguments *args = (struct service_arguments *)util_common_calloc_s(sizeof(struct service_arguments)); +@@ -57,4 +65,17 @@ TEST(CgroupCpuUnitTest, test_conf_get_cgroup_cpu_rt) + ASSERT_EQ(cpu_rt_runtime, 0); + } + ++TEST(CgroupCpuUnitTest, test_find_cgroup_mountpoint_and_root) ++{ ++ char *mnt = NULL; ++ char *root = NULL; ++ ASSERT_EQ(find_cgroup_mountpoint_and_root(nullptr, &mnt, &root), -1); ++} + ++TEST(CgroupCpuUnitTest, test_sysinfo_cgroup_controller_cpurt_mnt_path) ++{ ++ MOCK_SET(util_common_calloc_s, nullptr); ++ ASSERT_EQ(get_sys_info(true), nullptr); ++ ASSERT_EQ(sysinfo_cgroup_controller_cpurt_mnt_path(), nullptr); ++ MOCK_CLEAR(util_common_calloc_s); ++} +-- +2.25.1 + diff --git a/0024-remove-temp-variables.patch b/0024-remove-temp-variables.patch new file mode 100644 index 0000000..3faeba9 --- /dev/null +++ b/0024-remove-temp-variables.patch @@ -0,0 +1,179 @@ +From e20767fc709bb945eb8e076a57f07d5b70b5f3ab Mon Sep 17 00:00:00 2001 +From: "ilya.kuksenok" +Date: Wed, 22 Feb 2023 16:17:34 +0300 +Subject: [PATCH 24/53] remove temp variables + +--- + src/api/services/containers/container.proto | 5 +++++ + .../connect/grpc/grpc_containers_client.cc | 5 +++++ + src/client/connect/protocol_type.h | 5 +++++ + .../connect/rest/rest_containers_client.c | 5 +++++ + .../connect/grpc/container/stats_service.cc | 6 ++++++ + .../entry/cri/cri_container_manager_service.cc | 18 ++++++++++++++++-- + .../executor/container_cb/execution_extend.c | 5 +++++ + src/daemon/modules/api/runtime_api.h | 5 +++++ + .../modules/runtime/engines/lcr/lcr_engine.c | 6 ++++++ + 9 files changed, 58 insertions(+), 2 deletions(-) + +diff --git a/src/api/services/containers/container.proto b/src/api/services/containers/container.proto +index 13d1367b..230d18b3 100644 +--- a/src/api/services/containers/container.proto ++++ b/src/api/services/containers/container.proto +@@ -86,6 +86,11 @@ message Container_info { + uint64 cache = 16; + uint64 cache_total = 17; + uint64 inactive_file_total = 18; ++ uint64 avaliable_bytes = 19; ++ uint64 usage_bytes = 20; ++ uint64 rss_bytes = 21; ++ uint64 page_faults = 22; ++ uint64 major_page_faults = 23; + } + + message Event { +diff --git a/src/client/connect/grpc/grpc_containers_client.cc b/src/client/connect/grpc/grpc_containers_client.cc +index 1528b2ee..20766049 100644 +--- a/src/client/connect/grpc/grpc_containers_client.cc ++++ b/src/client/connect/grpc/grpc_containers_client.cc +@@ -1690,6 +1690,11 @@ public: + response->container_stats[i].blkio_read = gresponse->containers(i).blkio_read(); + response->container_stats[i].blkio_write = gresponse->containers(i).blkio_write(); + response->container_stats[i].mem_used = gresponse->containers(i).mem_used(); ++ response->container_stats[i].avaliable_bytes = gresponse->containers(i).avaliable_bytes(); ++ response->container_stats[i].usage_bytes = gresponse->containers(i).usage_bytes(); ++ response->container_stats[i].rss_bytes = gresponse->containers(i).rss_bytes(); ++ response->container_stats[i].page_faults = gresponse->containers(i).page_faults(); ++ response->container_stats[i].major_page_faults = gresponse->containers(i).major_page_faults(); + response->container_stats[i].mem_limit = gresponse->containers(i).mem_limit(); + response->container_stats[i].kmem_used = gresponse->containers(i).kmem_used(); + response->container_stats[i].kmem_limit = gresponse->containers(i).kmem_limit(); +diff --git a/src/client/connect/protocol_type.h b/src/client/connect/protocol_type.h +index 3831c7c0..a2d474a6 100644 +--- a/src/client/connect/protocol_type.h ++++ b/src/client/connect/protocol_type.h +@@ -165,6 +165,11 @@ struct isula_container_info { + // Memory usage + uint64_t mem_used; + uint64_t mem_limit; ++ uint64_t avaliable_bytes; ++ uint64_t usage_bytes; ++ uint64_t rss_bytes; ++ uint64_t page_faults; ++ uint64_t major_page_faults; + // Kernel Memory usage + uint64_t kmem_used; + uint64_t kmem_limit; +diff --git a/src/client/connect/rest/rest_containers_client.c b/src/client/connect/rest/rest_containers_client.c +index 0a735a64..9eee9d40 100644 +--- a/src/client/connect/rest/rest_containers_client.c ++++ b/src/client/connect/rest/rest_containers_client.c +@@ -2195,6 +2195,11 @@ static int unpack_container_info_for_stats_response(const container_stats_respon + // memory usage + infos[i].mem_used = con_info->mem_used; + infos[i].mem_limit = con_info->mem_limit; ++ infos[i].avaliable_bytes = con_info->avaliable_bytes(); ++ infos[i].usage_bytes = con_info->usage_bytes(); ++ infos[i].rss_bytes = con_info->rss_bytes(); ++ infos[i].page_faults = con_info->page_faults(); ++ infos[i].major_page_faults = con_info->major_page_faults(); + // kernel memory usage + infos[i].kmem_used = con_info->kmem_used; + infos[i].kmem_limit = con_info->kmem_limit; +diff --git a/src/daemon/entry/connect/grpc/container/stats_service.cc b/src/daemon/entry/connect/grpc/container/stats_service.cc +index 6e116b4c..7315fe75 100644 +--- a/src/daemon/entry/connect/grpc/container/stats_service.cc ++++ b/src/daemon/entry/connect/grpc/container/stats_service.cc +@@ -87,6 +87,12 @@ void ContainerStatsService::FillResponseTogRPC(void *containerRes, StatsResponse + stats->set_mem_limit(response->container_stats[i]->mem_limit); + stats->set_kmem_used(response->container_stats[i]->kmem_used); + stats->set_kmem_limit(response->container_stats[i]->kmem_limit); ++ stats->set_avaliable_bytes(response->container_stats[i]->avaliable_bytes); ++ stats->set_usage_bytes(response->container_stats[i]->usage_bytes); ++ stats->set_mem_used(response->container_stats[i]->mem_used); ++ stats->set_rss_bytes(response->container_stats[i]->rss_bytes); ++ stats->set_page_faults(response->container_stats[i]->page_faults); ++ stats->set_major_page_faults(response->container_stats[i]->major_page_faults); + if (response->container_stats[i]->name != nullptr) { + stats->set_name(response->container_stats[i]->name); + } +diff --git a/src/daemon/entry/cri/cri_container_manager_service.cc b/src/daemon/entry/cri/cri_container_manager_service.cc +index b02367c8..57284593 100644 +--- a/src/daemon/entry/cri/cri_container_manager_service.cc ++++ b/src/daemon/entry/cri/cri_container_manager_service.cc +@@ -842,8 +842,22 @@ void ContainerManagerService::ContainerStatsToGRPC( + response->container_stats[i]->cpu_use_nanos); + container->mutable_cpu()->set_timestamp(timestamp); + } +- +- containerstats->push_back(move(container)); ++ if (response->container_stats[i]->avaliable_bytes != 0u) { ++ container->mutable_memory()->mutable_available_bytes()->set_value(response->container_stats[i]->avaliable_bytes); ++ } ++ if (response->container_stats[i]->usage_bytes != 0u) { ++ container->mutable_memory()->mutable_usage_bytes()->set_value(response->container_stats[i]->usage_bytes); ++ } ++ if (response->container_stats[i]->rss_bytes != 0u) { ++ container->mutable_memory()->mutable_rss_bytes()->set_value(response->container_stats[i]->rss_bytes); ++ } ++ if (response->container_stats[i]->page_faults != 0u) { ++ container->mutable_memory()->mutable_page_faults()->set_value(response->container_stats[i]->page_faults); ++ } ++ if (response->container_stats[i]->major_page_faults != 0u) { ++ container->mutable_memory()->mutable_major_page_faults()->set_value(response->container_stats[i]->major_page_faults); ++ } ++ containerstats->push_back(std::move(container)); + } + } + +diff --git a/src/daemon/executor/container_cb/execution_extend.c b/src/daemon/executor/container_cb/execution_extend.c +index b0da705e..67d0845a 100644 +--- a/src/daemon/executor/container_cb/execution_extend.c ++++ b/src/daemon/executor/container_cb/execution_extend.c +@@ -243,6 +243,11 @@ static container_info *get_container_stats(const container_t *cont, + info->blkio_write = einfo->blkio_write; + info->mem_used = einfo->mem_used; + info->mem_limit = einfo->mem_limit; ++ info->avaliable_bytes = einfo->avaliable_bytes; ++ info->usage_bytes = einfo->usage_bytes; ++ info->rss_bytes = einfo->rss_bytes; ++ info->page_faults = einfo->page_faults; ++ info->major_page_faults = einfo->major_page_faults; + info->kmem_used = einfo->kmem_used; + info->kmem_limit = einfo->kmem_limit; + +diff --git a/src/daemon/modules/api/runtime_api.h b/src/daemon/modules/api/runtime_api.h +index de4136a3..b245ebf9 100644 +--- a/src/daemon/modules/api/runtime_api.h ++++ b/src/daemon/modules/api/runtime_api.h +@@ -55,6 +55,11 @@ struct runtime_container_resources_stats_info { + /* Memory usage */ + uint64_t mem_used; + uint64_t mem_limit; ++ uint64_t avaliable_bytes; ++ uint64_t usage_bytes; ++ uint64_t rss_bytes; ++ uint64_t page_faults; ++ uint64_t major_page_faults; + /* Kernel Memory usage */ + uint64_t kmem_used; + uint64_t kmem_limit; +diff --git a/src/daemon/modules/runtime/engines/lcr/lcr_engine.c b/src/daemon/modules/runtime/engines/lcr/lcr_engine.c +index 2ca12545..2f7f73f8 100644 +--- a/src/daemon/modules/runtime/engines/lcr/lcr_engine.c ++++ b/src/daemon/modules/runtime/engines/lcr/lcr_engine.c +@@ -162,6 +162,12 @@ static void copy_container_resources_stats(const struct lcr_container_state *lcs + rs_stats->mem_limit = lcs->mem_limit; + rs_stats->kmem_used = lcs->kmem_used; + rs_stats->kmem_limit = lcs->kmem_limit; ++ rs_stats->avaliable_bytes = lcs->avaliable_bytes; ++ rs_stats->usage_bytes = lcs->usage_bytes; ++ rs_stats->mem_used = lcs->mem_used; ++ rs_stats->rss_bytes = lcs->rss_bytes; ++ rs_stats->page_faults = lcs->page_faults; ++ rs_stats->major_page_faults = lcs->major_page_faults; + rs_stats->cache = lcs->cache; + rs_stats->cache_total = lcs->cache_total; + rs_stats->inactive_file_total = lcs->inactive_file_total; +-- +2.25.1 + diff --git a/0025-fix-read-member-error-from-struct.patch b/0025-fix-read-member-error-from-struct.patch new file mode 100644 index 0000000..041cd12 --- /dev/null +++ b/0025-fix-read-member-error-from-struct.patch @@ -0,0 +1,34 @@ +From 3e71fe5910a1b0c9dbb899e06614103a2cebf0fc Mon Sep 17 00:00:00 2001 +From: haozi007 +Date: Fri, 24 Feb 2023 10:25:46 +0800 +Subject: [PATCH 25/53] fix read member error from struct + +Signed-off-by: haozi007 +--- + src/client/connect/rest/rest_containers_client.c | 10 +++++----- + 1 file changed, 5 insertions(+), 5 deletions(-) + +diff --git a/src/client/connect/rest/rest_containers_client.c b/src/client/connect/rest/rest_containers_client.c +index 9eee9d40..0ae9a4f6 100644 +--- a/src/client/connect/rest/rest_containers_client.c ++++ b/src/client/connect/rest/rest_containers_client.c +@@ -2195,11 +2195,11 @@ static int unpack_container_info_for_stats_response(const container_stats_respon + // memory usage + infos[i].mem_used = con_info->mem_used; + infos[i].mem_limit = con_info->mem_limit; +- infos[i].avaliable_bytes = con_info->avaliable_bytes(); +- infos[i].usage_bytes = con_info->usage_bytes(); +- infos[i].rss_bytes = con_info->rss_bytes(); +- infos[i].page_faults = con_info->page_faults(); +- infos[i].major_page_faults = con_info->major_page_faults(); ++ infos[i].avaliable_bytes = con_info->avaliable_bytes; ++ infos[i].usage_bytes = con_info->usage_bytes; ++ infos[i].rss_bytes = con_info->rss_bytes; ++ infos[i].page_faults = con_info->page_faults; ++ infos[i].major_page_faults = con_info->major_page_faults; + // kernel memory usage + infos[i].kmem_used = con_info->kmem_used; + infos[i].kmem_limit = con_info->kmem_limit; +-- +2.25.1 + diff --git a/0026-Fix-PR-runc.patch b/0026-Fix-PR-runc.patch new file mode 100644 index 0000000..0467a7c --- /dev/null +++ b/0026-Fix-PR-runc.patch @@ -0,0 +1,108 @@ +From 39d7fd140e8b590a925e5cdf8ace20b0161328c8 Mon Sep 17 00:00:00 2001 +From: sailorvii +Date: Mon, 27 Feb 2023 02:33:46 +0000 +Subject: [PATCH 26/53] =?UTF-8?q?Fix=20PR=20runc=E8=BF=90=E8=A1=8C?= + =?UTF-8?q?=E5=A4=B1=E8=B4=A5.?= +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +The environment NOTIFY_SOCKET is used when runc start but not runc create. +As the source code: https://github.com/opencontainers/runc/blob/main/start.go#L35. + +So move the related code to the right location. +--- + .../modules/runtime/isula/isula_rt_ops.c | 27 +++++++++---------- + 1 file changed, 13 insertions(+), 14 deletions(-) + +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index 6f2b4f7d..60742d42 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -760,12 +760,6 @@ static int shim_create(bool fg, const char *id, const char *workdir, const char + goto realexec; + } + +- // clear NOTIFY_SOCKET from the env to adapt runc create +- if (unsetenv("NOTIFY_SOCKET") != 0) { +- (void)dprintf(exec_fd[1], "%s: unset env NOTIFY_SOCKET failed %s", id, strerror(errno)); +- exit(EXIT_FAILURE); +- } +- + pid = fork(); + if (pid < 0) { + (void)dprintf(exec_fd[1], "%s: fork shim-process failed %s", id, strerror(errno)); +@@ -945,7 +939,7 @@ int rt_isula_start(const char *id, const char *runtime, const rt_start_params_t + char shim_pid_file_name[PATH_MAX] = { 0 }; + pid_t pid = 0; + pid_t shim_pid = -1; +- int ret = 0; ++ int ret = -1; + int splice_ret = 0; + proc_t *proc = NULL; + proc_t *p_proc = NULL; +@@ -967,28 +961,24 @@ int rt_isula_start(const char *id, const char *runtime, const rt_start_params_t + + pid = get_container_process_pid(workdir); + if (pid < 0) { +- ret = -1; + ERROR("%s: failed wait init pid", id); + goto out; + } + + file_read_int(shim_pid_file_name, &shim_pid); + if (shim_pid < 0) { +- ret = -1; + ERROR("%s: failed to read isulad shim pid", id); + goto out; + } + + proc = util_get_process_proc_info(pid); + if (proc == NULL) { +- ret = -1; + ERROR("%s: failed to read pidinfo", id); + goto out; + } + + p_proc = util_get_process_proc_info(shim_pid); + if (p_proc == NULL) { +- ret = -1; + ERROR("%s: failed to read isulad shim pidinfo", id); + goto out; + } +@@ -998,20 +988,29 @@ int rt_isula_start(const char *id, const char *runtime, const rt_start_params_t + pid_info->ppid = shim_pid; + pid_info->pstart_time = p_proc->start_time; + ++ // clear NOTIFY_SOCKET from the env to adapt runc start ++ if (unsetenv("NOTIFY_SOCKET") != 0) { ++ ERROR("%s: unset env NOTIFY_SOCKET failed %s", id); ++ } ++ + if (runtime_call_simple(workdir, runtime, "start", NULL, 0, id, NULL) != 0) { + ERROR("call runtime start id failed"); +- ret = -1; + goto out; + } + ++ ret = 0; + out: + if (ret != 0) { + show_shim_runtime_errlog(workdir); + shim_kill_force(workdir); + } + +- free(proc); +- free(p_proc); ++ if (proc != NULL) { ++ free(proc); ++ } ++ if (p_proc != NULL) { ++ free(p_proc); ++ } + + return ret; + } +-- +2.25.1 + diff --git a/0027-allow-the-paused-container-to-be-stopped.patch b/0027-allow-the-paused-container-to-be-stopped.patch new file mode 100644 index 0000000..8fe9aa1 --- /dev/null +++ b/0027-allow-the-paused-container-to-be-stopped.patch @@ -0,0 +1,68 @@ +From 70173556a3825870b44cc344e83cdf0fd6d577c5 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Thu, 23 Feb 2023 16:08:48 +0800 +Subject: [PATCH 27/53] allow the paused container to be stopped + +Signed-off-by: zhongtao +--- + src/daemon/entry/cri/cri_container_manager_service.cc | 6 +++--- + src/daemon/entry/cri/cri_helpers.cc | 2 +- + src/daemon/modules/service/service_container.c | 7 +++---- + 3 files changed, 7 insertions(+), 8 deletions(-) + +diff --git a/src/daemon/entry/cri/cri_container_manager_service.cc b/src/daemon/entry/cri/cri_container_manager_service.cc +index 57284593..1592c0a6 100644 +--- a/src/daemon/entry/cri/cri_container_manager_service.cc ++++ b/src/daemon/entry/cri/cri_container_manager_service.cc +@@ -1195,14 +1195,14 @@ void ContainerManagerService::UpdateContainerResources(const std::string &contai + } + if (resources.hugepage_limits_size() != 0) { + hostconfig->hugetlbs = (host_config_hugetlbs_element **)util_smart_calloc_s( +- sizeof(host_config_hugetlbs_element *), resources.hugepage_limits_size()); ++ sizeof(host_config_hugetlbs_element *), resources.hugepage_limits_size()); + if (hostconfig->hugetlbs == nullptr) { + error.SetError("Out of memory"); + return; + } +- for (int i = 0; i < resources.hugepage_limits_size(); i++) { ++ for (int i = 0; i < resources.hugepage_limits_size(); i++) { + hostconfig->hugetlbs[i] = +- (host_config_hugetlbs_element *)util_common_calloc_s(sizeof(host_config_hugetlbs_element)); ++ (host_config_hugetlbs_element *)util_common_calloc_s(sizeof(host_config_hugetlbs_element)); + if (hostconfig->hugetlbs[i] == nullptr) { + error.SetError("Out of memory"); + goto cleanup; +diff --git a/src/daemon/entry/cri/cri_helpers.cc b/src/daemon/entry/cri/cri_helpers.cc +index 6d59ec11..c24c8b73 100644 +--- a/src/daemon/entry/cri/cri_helpers.cc ++++ b/src/daemon/entry/cri/cri_helpers.cc +@@ -470,7 +470,7 @@ void UpdateCreateConfig(container_config *createConfig, host_config *hc, + } + for (int i = 0; i < rOpts.hugepage_limits_size(); i++) { + hc->hugetlbs[i] = +- (host_config_hugetlbs_element *)util_common_calloc_s(sizeof(host_config_hugetlbs_element)); ++ (host_config_hugetlbs_element *)util_common_calloc_s(sizeof(host_config_hugetlbs_element)); + if (hc->hugetlbs[i] == nullptr) { + error.SetError("Out of memory"); + return; +diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c +index cc777411..d69ee757 100644 +--- a/src/daemon/modules/service/service_container.c ++++ b/src/daemon/modules/service/service_container.c +@@ -1495,10 +1495,9 @@ int stop_container(container_t *cont, int timeout, bool force, bool restart) + + container_lock(cont); + +- if (container_is_paused(cont->state)) { +- ERROR("Container %s is paused. Unpause the container before stopping or killing", id); +- isulad_set_error_message("Container %s is paused. Unpause the container before stopping or killing", id); +- ret = -1; ++ if (!container_is_running(cont->state)) { ++ INFO("Container %s is already stopped", id); ++ ret = 0; + goto out; + } + +-- +2.25.1 + diff --git a/0028-Refine.patch b/0028-Refine.patch new file mode 100644 index 0000000..b8fb739 --- /dev/null +++ b/0028-Refine.patch @@ -0,0 +1,29 @@ +From 376677f2f38b3c27b14b7a21aa021ea683a2f0e1 Mon Sep 17 00:00:00 2001 +From: sailorvii +Date: Tue, 28 Feb 2023 06:09:48 +0000 +Subject: [PATCH 28/53] Refine. + +--- + src/daemon/modules/runtime/isula/isula_rt_ops.c | 6 ++++++ + 1 file changed, 6 insertions(+) + +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index 60742d42..5d622515 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -760,6 +760,12 @@ static int shim_create(bool fg, const char *id, const char *workdir, const char + goto realexec; + } + ++ // clear NOTIFY_SOCKET from the env to adapt runc create ++ if (unsetenv("NOTIFY_SOCKET") != 0) { ++ (void)dprintf(exec_fd[1], "%s: unset env NOTIFY_SOCKET failed %s", id, strerror(errno)); ++ exit(EXIT_FAILURE); ++ } ++ + pid = fork(); + if (pid < 0) { + (void)dprintf(exec_fd[1], "%s: fork shim-process failed %s", id, strerror(errno)); +-- +2.25.1 + diff --git a/0029-support-isula-update-when-runtime-is-runc.patch b/0029-support-isula-update-when-runtime-is-runc.patch new file mode 100644 index 0000000..aad54f0 --- /dev/null +++ b/0029-support-isula-update-when-runtime-is-runc.patch @@ -0,0 +1,291 @@ +From 7f00006ea65378e7b27049ff3f0eb3fa70e69b09 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Thu, 16 Feb 2023 20:20:54 +0800 +Subject: [PATCH 29/53] support isula update when runtime is runc + +Signed-off-by: zhongtao +--- + .../executor/container_cb/execution_extend.c | 1 + + src/daemon/modules/api/runtime_api.h | 1 + + .../modules/runtime/isula/isula_rt_ops.c | 180 ++++++++++++++++-- + 3 files changed, 161 insertions(+), 21 deletions(-) + +diff --git a/src/daemon/executor/container_cb/execution_extend.c b/src/daemon/executor/container_cb/execution_extend.c +index 67d0845a..2f565d78 100644 +--- a/src/daemon/executor/container_cb/execution_extend.c ++++ b/src/daemon/executor/container_cb/execution_extend.c +@@ -1134,6 +1134,7 @@ static int do_update_resources(const container_update_request *request, containe + if (container_is_running(cont->state)) { + params.rootpath = cont->root_path; + params.hostconfig = hostconfig; ++ params.state = cont->state_path; + if (runtime_update(id, cont->runtime, ¶ms)) { + ERROR("Update container %s failed", id); + ret = -1; +diff --git a/src/daemon/modules/api/runtime_api.h b/src/daemon/modules/api/runtime_api.h +index b245ebf9..199c9f4b 100644 +--- a/src/daemon/modules/api/runtime_api.h ++++ b/src/daemon/modules/api/runtime_api.h +@@ -169,6 +169,7 @@ typedef struct _rt_attach_params_t { + typedef struct _rt_update_params_t { + const char *rootpath; + const host_config *hostconfig; ++ const char *state; + } rt_update_params_t; + + typedef struct _rt_listpids_params_t { +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index 6f2b4f7d..41791388 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -21,6 +21,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -39,6 +40,7 @@ + #include "constants.h" + #include "isula_libutils/shim_client_process_state.h" + #include "isula_libutils/shim_client_runtime_stats.h" ++#include "isula_libutils/shim_client_cgroup_resources.h" + #include "isula_libutils/oci_runtime_state.h" + #include "isulad_config.h" + #include "utils_string.h" +@@ -54,6 +56,9 @@ + #define RESIZE_DATA_SIZE 100 + #define PID_WAIT_TIME 120 + ++// file name formats of cgroup resources json ++#define RESOURCE_FNAME_FORMATS "%s/resources.json" ++ + // handle string from stderr output. + typedef int(*handle_output_callback_t)(const char *output); + +@@ -229,34 +234,27 @@ bool rt_isula_detect(const char *runtime) + static int create_process_json_file(const char *workdir, const shim_client_process_state *p) + { + struct parser_context ctx = { OPT_GEN_SIMPLIFY, 0 }; +- parser_error perr = NULL; +- char *data = NULL; ++ __isula_auto_free parser_error perr = NULL; ++ __isula_auto_free char *data = NULL; + char fname[PATH_MAX] = { 0 }; +- int retcode = 0; + + if (snprintf(fname, sizeof(fname), "%s/process.json", workdir) < 0) { +- ERROR("failed make process.json full path"); ++ ERROR("Failed make process.json full path"); + return -1; + } + + data = shim_client_process_state_generate_json(p, &ctx, &perr); + if (data == NULL) { +- retcode = -1; +- ERROR("failed generate json for process.json error=%s", perr); +- goto out; ++ ERROR("Failed generate json for process.json error=%s", perr); ++ return -1; + } + + if (util_write_file(fname, data, strlen(data), DEFAULT_SECURE_FILE_MODE) != 0) { +- retcode = -1; +- ERROR("failed write process.json"); +- goto out; ++ ERROR("Failed write process.json"); ++ return -1; + } + +-out: +- UTIL_FREE_AND_SET_NULL(perr); +- UTIL_FREE_AND_SET_NULL(data); +- +- return retcode; ++ return 0; + } + + static void get_runtime_cmd(const char *runtime, const char **cmd) +@@ -733,18 +731,18 @@ static int shim_create(bool fg, const char *id, const char *workdir, const char + runtime_exec_param_dump(params); + + if (snprintf(fpid, sizeof(fpid), "%s/shim-pid", workdir) < 0) { +- ERROR("failed make shim-pid full path"); ++ ERROR("Failed make shim-pid full path"); + return -1; + } + + if (pipe2(exec_fd, O_CLOEXEC) != 0) { +- ERROR("failed to create pipe for shim create"); ++ ERROR("Failed to create pipe for shim create"); + return -1; + } + + pid = fork(); + if (pid < 0) { +- ERROR("failed fork for shim parent %s", strerror(errno)); ++ ERROR("Failed fork for shim parent %s", strerror(errno)); + close(exec_fd[0]); + close(exec_fd[1]); + return -1; +@@ -1264,13 +1262,153 @@ int rt_isula_attach(const char *id, const char *runtime, const rt_attach_params_ + return -1; + } + +-int rt_isula_update(const char *id, const char *runtime, const rt_update_params_t *params) ++static int to_engine_resources(const host_config *hostconfig, shim_client_cgroup_resources *cr) ++{ ++ uint64_t period = 0; ++ int64_t quota = 0; ++ ++ if (hostconfig == NULL || cr == NULL) { ++ return -1; ++ } ++ ++ cr->block_io = util_common_calloc_s(sizeof(shim_client_cgroup_resources_block_io)); ++ if (cr->block_io == NULL) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ ++ cr->cpu = util_common_calloc_s(sizeof(shim_client_cgroup_resources_cpu)); ++ if (cr->cpu == NULL) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ ++ cr->memory = util_common_calloc_s(sizeof(shim_client_cgroup_resources_memory)); ++ if (cr->memory == NULL) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ ++ cr->block_io->weight = hostconfig->blkio_weight; ++ cr->cpu->shares = (uint64_t)hostconfig->cpu_shares; ++ cr->cpu->period = (uint64_t)hostconfig->cpu_period; ++ cr->cpu->quota = hostconfig->cpu_quota; ++ cr->cpu->cpus = util_strdup_s(hostconfig->cpuset_cpus); ++ cr->cpu->mems = util_strdup_s(hostconfig->cpuset_mems); ++ cr->memory->limit = (uint64_t)hostconfig->memory; ++ cr->memory->swap = (uint64_t)hostconfig->memory_swap; ++ cr->memory->reservation = (uint64_t)hostconfig->memory_reservation; ++ cr->memory->kernel = (uint64_t)hostconfig->kernel_memory; ++ cr->cpu->realtime_period = hostconfig->cpu_realtime_period; ++ cr->cpu->realtime_runtime = hostconfig->cpu_realtime_runtime; ++ ++ // when --cpus=n is set, nano_cpus = n * 1e9. ++ if (hostconfig->nano_cpus > 0) { ++ // in the case, period will be set to the default value of 100000(0.1s). ++ period = (uint64_t)(100 * Time_Milli / Time_Micro); ++ // set quota = period * n, in order to let container process fully occupy n cpus. ++ if ((hostconfig->nano_cpus / 1e9) > (INT64_MAX / (int64_t)period)) { ++ ERROR("Overflow of quota"); ++ return -1; ++ } ++ quota = hostconfig->nano_cpus / 1e9 * (int64_t)period; ++ cr->cpu->period = period; ++ cr->cpu->quota = quota; ++ } ++ ++ return 0; ++} ++ ++static int create_resources_json_file(const char *workdir, const shim_client_cgroup_resources *cr, char *fname, ++ size_t fname_size) + { +- ERROR("isula update not support on isulad-shim"); +- isulad_set_error_message("isula update not support on isulad-shim"); ++ struct parser_context ctx = { OPT_GEN_SIMPLIFY, 0 }; ++ __isula_auto_free parser_error perr = NULL; ++ __isula_auto_free char *data = NULL; ++ ++ if (snprintf(fname, fname_size, RESOURCE_FNAME_FORMATS, workdir) < 0) { ++ ERROR("Failed make resources.json full path"); ++ return -1; ++ } ++ ++ data = shim_client_cgroup_resources_generate_json(cr, &ctx, &perr); ++ if (data == NULL) { ++ return -1; ++ } ++ ++ if (util_write_file(fname, data, strlen(data), DEFAULT_SECURE_FILE_MODE) != 0) { ++ return -1; ++ } ++ ++ return 0; ++} ++ ++// show std error msg, always return -1. ++static int show_stderr(const char *err) ++{ ++ isulad_set_error_message(err); + return -1; + } + ++int rt_isula_update(const char *id, const char *runtime, const rt_update_params_t *params) ++{ ++ int ret = 0; ++ char workdir[PATH_MAX] = { 0 }; ++ char resources_fname[PATH_MAX] = { 0 }; ++ const char *opts[2] = { 0 }; ++ shim_client_cgroup_resources *cr = NULL; ++ ++ if (id == NULL || runtime == NULL || params == NULL) { ++ ERROR("Nullptr arguments not allowed"); ++ return -1; ++ } ++ ++ ret = snprintf(workdir, sizeof(workdir), "%s/%s/update", params->state, id); ++ if (ret < 0) { ++ ERROR("Failed join update full path"); ++ return ret; ++ } ++ ++ ret = util_mkdir_p(workdir, DEFAULT_SECURE_DIRECTORY_MODE); ++ if (ret < 0) { ++ ERROR("Failed mkdir update workdir %s", workdir); ++ return ret; ++ } ++ ++ cr = util_common_calloc_s(sizeof(shim_client_cgroup_resources)); ++ if (cr == NULL) { ++ ERROR("Out of memory"); ++ goto del_out; ++ } ++ ++ ret = to_engine_resources(params->hostconfig, cr); ++ if (ret < 0) { ++ ERROR("Failed to get resources for update"); ++ goto del_out; ++ } ++ ++ ret = create_resources_json_file(workdir, cr, resources_fname, sizeof(resources_fname)); ++ if (ret != 0) { ++ ERROR("%s: failed create update json file", id); ++ goto del_out; ++ } ++ ++ opts[0] = "--resources"; ++ opts[1] = resources_fname; ++ ++ if (runtime_call_simple(workdir, runtime, "update", opts, 2, id, show_stderr) != 0) { ++ ERROR("Call runtime update id failed"); ++ ret = -1; ++ } ++ ++del_out: ++ if (util_recursive_rmdir(workdir, 0)) { ++ ERROR("Rmdir %s failed", workdir); ++ } ++ free_shim_client_cgroup_resources(cr); ++ return ret; ++} ++ + int rt_isula_pause(const char *id, const char *runtime, const rt_pause_params_t *params) + { + char workdir[PATH_MAX] = { 0 }; +-- +2.25.1 + diff --git a/0030-Refine-as-others-feedback.patch b/0030-Refine-as-others-feedback.patch new file mode 100644 index 0000000..39691ca --- /dev/null +++ b/0030-Refine-as-others-feedback.patch @@ -0,0 +1,41 @@ +From 4900996ca52e46795eb25b05ce50519c7c71ae38 Mon Sep 17 00:00:00 2001 +From: sailorvii +Date: Thu, 2 Mar 2023 09:04:57 +0000 +Subject: [PATCH 30/53] Refine as others' feedback. + +--- + src/daemon/modules/runtime/isula/isula_rt_ops.c | 11 ++++++----- + 1 file changed, 6 insertions(+), 5 deletions(-) + +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index 5d622515..7ec3fc49 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -473,6 +473,12 @@ static void runtime_exec_func(void *arg) + _exit(EXIT_FAILURE); + } + ++ // clear NOTIFY_SOCKET from the env to adapt runc start ++ if (strcmp(rei->subcmd, "start") == 0 && unsetenv("NOTIFY_SOCKET") != 0) { ++ dprintf(STDERR_FILENO, "unset env NOTIFY_SOCKET failed %s", strerror(errno)); ++ _exit(EXIT_FAILURE); ++ } ++ + execvp(rei->cmd, rei->params); + dprintf(STDERR_FILENO, "exec %s %s %s failed", rei->cmd, rei->subcmd, rei->id); + _exit(EXIT_FAILURE); +@@ -994,11 +1000,6 @@ int rt_isula_start(const char *id, const char *runtime, const rt_start_params_t + pid_info->ppid = shim_pid; + pid_info->pstart_time = p_proc->start_time; + +- // clear NOTIFY_SOCKET from the env to adapt runc start +- if (unsetenv("NOTIFY_SOCKET") != 0) { +- ERROR("%s: unset env NOTIFY_SOCKET failed %s", id); +- } +- + if (runtime_call_simple(workdir, runtime, "start", NULL, 0, id, NULL) != 0) { + ERROR("call runtime start id failed"); + goto out; +-- +2.25.1 + diff --git a/0031-fix-CRI-SetupPod-and-TearDownPod-deadlock.patch b/0031-fix-CRI-SetupPod-and-TearDownPod-deadlock.patch new file mode 100644 index 0000000..c5bd888 --- /dev/null +++ b/0031-fix-CRI-SetupPod-and-TearDownPod-deadlock.patch @@ -0,0 +1,134 @@ +From a87e8aeea252e5aec9e1dea0daf99562eb86b092 Mon Sep 17 00:00:00 2001 +From: zhangxiaoyu +Date: Thu, 2 Mar 2023 20:05:49 +0800 +Subject: [PATCH 31/53] fix CRI SetupPod and TearDownPod deadlock + +Signed-off-by: zhangxiaoyu +--- + src/daemon/entry/cri/cni_network_plugin.cc | 14 +++++++---- + .../cri/cri_pod_sandbox_manager_service.cc | 2 ++ + src/daemon/entry/cri/network_plugin.cc | 25 +++++++++++++------ + 3 files changed, 29 insertions(+), 12 deletions(-) + +diff --git a/src/daemon/entry/cri/cni_network_plugin.cc b/src/daemon/entry/cri/cni_network_plugin.cc +index 1bce13f6..0fe095b6 100644 +--- a/src/daemon/entry/cri/cni_network_plugin.cc ++++ b/src/daemon/entry/cri/cni_network_plugin.cc +@@ -120,8 +120,10 @@ auto CniNetworkPlugin::Name() const -> const std::string & + + void CniNetworkPlugin::CheckInitialized(Errors &err) + { +- RLockNetworkMap(err); +- if (err.NotEmpty()) { ++ Errors tmpErr; ++ RLockNetworkMap(tmpErr); ++ if (tmpErr.NotEmpty()) { ++ err.AppendError(tmpErr.GetCMessage()); + return; + } + +@@ -129,9 +131,10 @@ void CniNetworkPlugin::CheckInitialized(Errors &err) + err.SetError("cni config uninitialized"); + } + +- UnlockNetworkMap(err); +- if (err.NotEmpty()) { +- WARN("Unable to update cni config: %s", err.GetCMessage()); ++ UnlockNetworkMap(tmpErr); ++ if (tmpErr.NotEmpty()) { ++ WARN("Unable to update cni config: %s", tmpErr.GetCMessage()); ++ err.AppendError(tmpErr.GetCMessage()); + } + } + +@@ -623,6 +626,7 @@ void CniNetworkPlugin::SetUpPod(const std::string &ns, const std::string &name, + return; + } + ++ err.Clear(); + RLockNetworkMap(err); + if (err.NotEmpty()) { + ERROR("%s", err.GetCMessage()); +diff --git a/src/daemon/entry/cri/cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/cri_pod_sandbox_manager_service.cc +index 3d183861..ee523b9c 100644 +--- a/src/daemon/entry/cri/cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/cri_pod_sandbox_manager_service.cc +@@ -705,6 +705,7 @@ auto PodSandboxManagerService::GetRealSandboxIDToStop(const std::string &podSand + if (status->linux().namespaces().has_options()) { + hostNetwork = (status->linux().namespaces().options().network() == runtime::v1alpha2::NamespaceMode::NODE); + } ++ // if metadata is invalid, don't return -1 and continue stopping pod + if (status->has_metadata()) { + name = status->metadata().name(); + ns = status->metadata().namespace_(); +@@ -831,6 +832,7 @@ auto PodSandboxManagerService::ClearCniNetwork(const std::string &realSandboxID, + } + + stdAnnos.insert(std::pair(CRIHelpers::Constants::POD_SANDBOX_KEY, netnsPath)); ++ pluginErr.Clear(); + m_pluginManager->TearDownPod(ns, name, Network::DEFAULT_NETWORK_INTERFACE_NAME, realSandboxID, stdAnnos, + pluginErr); + if (pluginErr.NotEmpty()) { +diff --git a/src/daemon/entry/cri/network_plugin.cc b/src/daemon/entry/cri/network_plugin.cc +index 4a119d6b..e55db4ea 100644 +--- a/src/daemon/entry/cri/network_plugin.cc ++++ b/src/daemon/entry/cri/network_plugin.cc +@@ -425,20 +425,26 @@ void PluginManager::SetUpPod(const std::string &ns, const std::string &name, con + return; + } + ++ Errors tmpErr; + std::string fullName = name + "_" + ns; +- Lock(fullName, error); +- if (error.NotEmpty()) { ++ Lock(fullName, tmpErr); ++ if (tmpErr.NotEmpty()) { ++ error.AppendError(tmpErr.GetCMessage()); + return; + } + INFO("Calling network plugin %s to set up pod %s", m_plugin->Name().c_str(), fullName.c_str()); + +- Errors tmpErr; + m_plugin->SetUpPod(ns, name, interfaceName, podSandboxID, annotations, options, network_settings_json, tmpErr); + if (tmpErr.NotEmpty()) { + error.Errorf("NetworkPlugin %s failed to set up pod %s network: %s", m_plugin->Name().c_str(), fullName.c_str(), + tmpErr.GetCMessage()); + } +- Unlock(fullName, error); ++ ++ tmpErr.Clear(); ++ Unlock(fullName, tmpErr); ++ if (tmpErr.NotEmpty()) { ++ error.AppendError(tmpErr.GetCMessage()); ++ } + } + + void PluginManager::TearDownPod(const std::string &ns, const std::string &name, const std::string &interfaceName, +@@ -447,8 +453,9 @@ void PluginManager::TearDownPod(const std::string &ns, const std::string &name, + { + Errors tmpErr; + std::string fullName = name + "_" + ns; +- Lock(fullName, error); +- if (error.NotEmpty()) { ++ Lock(fullName, tmpErr); ++ if (tmpErr.NotEmpty()) { ++ error.AppendError(tmpErr.GetCMessage()); + return; + } + if (m_plugin == nullptr) { +@@ -462,7 +469,11 @@ void PluginManager::TearDownPod(const std::string &ns, const std::string &name, + fullName.c_str(), tmpErr.GetCMessage()); + } + unlock: +- Unlock(fullName, error); ++ tmpErr.Clear(); ++ Unlock(fullName, tmpErr); ++ if (tmpErr.NotEmpty()) { ++ error.AppendError(tmpErr.GetCMessage()); ++ } + } + + void NoopNetworkPlugin::Init(const std::string &hairpinMode, const std::string &nonMasqueradeCIDR, int mtu, +-- +2.25.1 + diff --git a/0032-remote-layer-store-demo.patch b/0032-remote-layer-store-demo.patch new file mode 100644 index 0000000..a20a04d --- /dev/null +++ b/0032-remote-layer-store-demo.patch @@ -0,0 +1,2296 @@ +From 9072007e7f61f7658baf7c4101126040dc341d0b Mon Sep 17 00:00:00 2001 +From: "Neil.wrz" +Date: Mon, 9 Jan 2023 17:12:53 -0800 +Subject: [PATCH 32/53] remote layer store demo + +Signed-off-by: Neil +--- + cmake/options.cmake | 6 + + src/daemon/config/isulad_config.c | 1 + + src/daemon/modules/image/oci/oci_image.c | 6 +- + .../modules/image/oci/storage/CMakeLists.txt | 6 + + .../storage/image_store/image_remote_impl.c | 173 +++++++++ + .../oci/storage/image_store/image_store.c | 76 +++- + .../oci/storage/image_store/image_store.h | 11 + + .../storage/layer_store/graphdriver/driver.c | 4 +- + .../storage/layer_store/graphdriver/driver.h | 3 + + .../graphdriver/overlay2/driver_overlay2.c | 158 +++++++- + .../graphdriver/overlay2/driver_overlay2.h | 8 + + .../overlay2/overlay_remote_impl.c | 282 ++++++++++++++ + .../storage/layer_store/layer_remote_impl.c | 219 +++++++++++ + .../oci/storage/layer_store/layer_store.c | 199 +++++++++- + .../oci/storage/layer_store/layer_store.h | 11 + + .../remote_layer_support/CMakeLists.txt | 12 + + .../remote_layer_support/remote_support.c | 122 ++++++ + .../remote_layer_support/remote_support.h | 58 +++ + .../ro_symlink_maintain.c | 347 ++++++++++++++++++ + .../ro_symlink_maintain.h | 52 +++ + .../modules/image/oci/storage/storage.c | 11 +- + .../modules/image/oci/storage/storage.h | 3 + + 22 files changed, 1750 insertions(+), 18 deletions(-) + create mode 100644 src/daemon/modules/image/oci/storage/image_store/image_remote_impl.c + create mode 100644 src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/overlay_remote_impl.c + create mode 100644 src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c + create mode 100644 src/daemon/modules/image/oci/storage/remote_layer_support/CMakeLists.txt + create mode 100644 src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c + create mode 100644 src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h + create mode 100644 src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c + create mode 100644 src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.h + +diff --git a/cmake/options.cmake b/cmake/options.cmake +index a357a504..44ccf672 100644 +--- a/cmake/options.cmake ++++ b/cmake/options.cmake +@@ -148,6 +148,12 @@ if (DISABLE_CLEANUP STREQUAL "ON") + message("${Green}-- Disable cleanup module") + endif() + ++option(ENABLE_REMOTE_LAYER_STORE "enable remote layer store" OFF) ++if (ENABLE_REMOTE_LAYER_STORE STREQUAL "ON") ++ add_definitions(-DENABLE_REMOTE_LAYER_STORE) ++ message("${Green}-- Enable remote layer store") ++endif() ++ + option(MUSL "available for musl" OFF) + if (MUSL) + add_definitions(-D__MUSL__) +diff --git a/src/daemon/config/isulad_config.c b/src/daemon/config/isulad_config.c +index 38bf4bf9..b7bfe2df 100644 +--- a/src/daemon/config/isulad_config.c ++++ b/src/daemon/config/isulad_config.c +@@ -1555,6 +1555,7 @@ static int merge_authorization_conf_into_global(struct service_arguments *args, + static int merge_storage_conf_into_global(struct service_arguments *args, isulad_daemon_configs *tmp_json_confs) + { + override_string_value(&args->json_confs->storage_driver, &tmp_json_confs->storage_driver); ++ args->json_confs->storage_enable_remote_layer = tmp_json_confs->storage_enable_remote_layer; + + if (string_array_append(tmp_json_confs->storage_opts, tmp_json_confs->storage_opts_len, + &(args->json_confs->storage_opts_len), &(args->json_confs->storage_opts)) != 0) { +diff --git a/src/daemon/modules/image/oci/oci_image.c b/src/daemon/modules/image/oci/oci_image.c +index 50d13cec..fa92a861 100644 +--- a/src/daemon/modules/image/oci/oci_image.c ++++ b/src/daemon/modules/image/oci/oci_image.c +@@ -218,6 +218,10 @@ static int storage_module_init_helper(const isulad_daemon_configs *args) + goto out; + } + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ storage_opts->enable_remote_layer = args->storage_enable_remote_layer; ++#endif ++ + if (util_dup_array_of_strings((const char **)args->storage_opts, args->storage_opts_len, &storage_opts->driver_opts, + &storage_opts->driver_opts_len) != 0) { + ERROR("Failed to get storage storage opts"); +@@ -763,4 +767,4 @@ int oci_search(const im_search_request *request, imagetool_search_result **resul + + return ret; + } +-#endif +\ No newline at end of file ++#endif +diff --git a/src/daemon/modules/image/oci/storage/CMakeLists.txt b/src/daemon/modules/image/oci/storage/CMakeLists.txt +index 06c2f378..24b71450 100644 +--- a/src/daemon/modules/image/oci/storage/CMakeLists.txt ++++ b/src/daemon/modules/image/oci/storage/CMakeLists.txt +@@ -3,12 +3,17 @@ aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_storage_srcs) + add_subdirectory(image_store) + add_subdirectory(layer_store) + add_subdirectory(rootfs_store) ++IF (ENABLE_REMOTE_LAYER_STORE) ++add_subdirectory(remote_layer_support) ++ENDIF() ++ + + set(STORAGE_SRCS + ${local_storage_srcs} + ${IMAGE_STORE_SRCS} + ${LAYER_STORE_SRCS} + ${ROOTFS_STORE_SRCS} ++ ${REMOTE_LAYER_SUPPORT_SRCS} + PARENT_SCOPE + ) + +@@ -17,5 +22,6 @@ set(STORAGE_INCS + ${IMAGE_STORE_INCS} + ${LAYER_STORE_INCS} + ${ROOTFS_STORE_INCS} ++ ${REMOTE_LAYER_SUPPORT_INCS} + PARENT_SCOPE + ) +diff --git a/src/daemon/modules/image/oci/storage/image_store/image_remote_impl.c b/src/daemon/modules/image/oci/storage/image_store/image_remote_impl.c +new file mode 100644 +index 00000000..20da8116 +--- /dev/null ++++ b/src/daemon/modules/image/oci/storage/image_store/image_remote_impl.c +@@ -0,0 +1,173 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2020-2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: wangrunze ++ * Create: 2023-03-03 ++ * Description: provide remote image store functions ++ ******************************************************************************/ ++#define _GNU_SOURCE ++#include "image_store.h" ++ ++#include ++#include ++ ++#include "remote_support.h" ++#include "ro_symlink_maintain.h" ++#include "map.h" ++#include "utils_file.h" ++#include "utils.h" ++#include "layer_store.h" ++#include "utils_array.h" ++ ++struct remote_image_data { ++ const char *image_home; ++}; ++ ++static map_t *image_byid_old = NULL; ++static map_t *image_byid_new = NULL; ++ ++static void *remote_support_create(const char *remote_home, const char *remote_ro) ++{ ++ struct remote_image_data *data = util_common_calloc_s(sizeof(struct remote_image_data)); ++ if (data == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ data->image_home = remote_home; ++ image_byid_old = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); ++ image_byid_new = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); ++ return data; ++} ++ ++static void remote_support_destroy(void *data) ++{ ++ if (data == NULL) { ++ return; ++ } ++ ++ map_free(image_byid_old); ++ map_free(image_byid_new); ++ ++ free(data); ++ return; ++} ++ ++static int remote_support_scan(void *data) ++{ ++ int ret = 0; ++ int nret; ++ char **image_dirs = NULL; ++ size_t image_dirs_num = 0; ++ size_t i; ++ char *id_patten = "^[a-f0-9]{64}$"; ++ char image_path[PATH_MAX] = { 0x00 }; ++ bool exist = true; ++ struct remote_image_data *img_data = (struct remote_image_data *)data; ++ ++ ret = util_list_all_subdir(img_data->image_home, &image_dirs); ++ if (ret != 0) { ++ ERROR("Failed to get images directory"); ++ goto out; ++ } ++ image_dirs_num = util_array_len((const char **)image_dirs); ++ ++ for (i = 0; i < image_dirs_num; i++) { ++ bool valid_v1_image = false; ++ ++ if (util_reg_match(id_patten, image_dirs[i]) != 0) { ++ DEBUG("Image's json is placed inside image's data directory, so skip any other file or directory: %s", ++ image_dirs[i]); ++ continue; ++ } ++ ++ nret = snprintf(image_path, sizeof(image_path), "%s/%s", img_data->image_home, image_dirs[i]); ++ if (nret < 0 || (size_t)nret >= sizeof(image_path)) { ++ ERROR("Failed to get image path"); ++ continue; ++ } ++ ++ if (validate_manifest_schema_version_1(image_path, &valid_v1_image) != 0) { ++ ERROR("Failed to validate manifest schema version 1 format"); ++ continue; ++ } ++ ++ if (!valid_v1_image) { ++ map_insert(image_byid_new, util_strdup_s(image_dirs[i]), (void *)&exist); ++ } ++ } ++ ++out: ++ util_free_array(image_dirs); ++ return ret; ++} ++ ++static int remote_support_add(void *data) ++{ ++ char **array_added = NULL; ++ char **array_deleted = NULL; ++ char *top_layer = NULL; ++ map_t *tmp_map = NULL; ++ int i = 0; ++ int ret = 0; ++ ++ if (data == NULL) { ++ return -1; ++ } ++ ++ array_added = added_layers(image_byid_old, image_byid_new); ++ array_deleted = deleted_layers(image_byid_old, image_byid_new); ++ ++ for (i = 0; i < util_array_len((const char **)array_added); i++) { ++ top_layer = get_top_layer_from_json(array_added[i]); ++ if (top_layer != NULL && !layer_remote_layer_valid(top_layer)) { ++ ERROR("ERROR not find valid under layer, remoet image:%s not added", array_added[i]); ++ map_remove(image_byid_new, (void *)array_added[i]); ++ continue; ++ } ++ ++ if (append_image_by_directory_with_lock(array_added[i]) != 0) { ++ ERROR("Failed to load image into memrory: %s", array_added[i]); ++ ret = -1; ++ } ++ } ++ ++ for (i = 0; i < util_array_len((const char **)array_deleted); i++) { ++ if (remove_image_from_memory_with_lock(array_deleted[i]) != 0) { ++ ERROR("Failed to remove remote memory store"); ++ ret = -1; ++ } ++ } ++ ++ tmp_map = image_byid_old; ++ image_byid_old = image_byid_new; ++ image_byid_new = tmp_map; ++ empty_map(image_byid_new); ++ ++ util_free_array(array_added); ++ util_free_array(array_deleted); ++ free(top_layer); ++ ++ return ret; ++} ++ ++remote_support *image_store_impl_remote_support(void) ++{ ++ remote_support *rs = util_common_calloc_s(sizeof(remote_support)); ++ if (rs == NULL) { ++ return NULL; ++ } ++ ++ rs->create = remote_support_create; ++ rs->destroy = remote_support_destroy; ++ rs->scan_remote_dir = remote_support_scan; ++ rs->load_item = remote_support_add; ++ ++ return rs; ++} +diff --git a/src/daemon/modules/image/oci/storage/image_store/image_store.c b/src/daemon/modules/image/oci/storage/image_store/image_store.c +index cf1e88ff..caff3705 100644 +--- a/src/daemon/modules/image/oci/storage/image_store/image_store.c ++++ b/src/daemon/modules/image/oci/storage/image_store/image_store.c +@@ -49,6 +49,9 @@ + #include "image_type.h" + #include "linked_list.h" + #include "utils_verify.h" ++#ifdef ENABLE_REMOTE_LAYER_STORE ++#include "ro_symlink_maintain.h" ++#endif + + // the name of the big data item whose contents we consider useful for computing a "digest" of the + // image, by which we can locate the image later. +@@ -3096,7 +3099,7 @@ out: + return ret; + } + +-static int validate_manifest_schema_version_1(const char *path, bool *valid) ++int validate_manifest_schema_version_1(const char *path, bool *valid) + { + int ret = 0; + int nret; +@@ -3640,6 +3643,10 @@ int image_store_init(struct storage_module_init_options *opts) + goto out; + } + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ remote_image_init(g_image_store->dir); ++#endif ++ + out: + if (ret != 0) { + free_image_store(g_image_store); +@@ -3648,3 +3655,70 @@ out: + free(root_dir); + return ret; + } ++ ++#ifdef ENABLE_REMOTE_LAYER_STORE ++int append_image_by_directory_with_lock(const char *id) ++{ ++ int ret = 0; ++ int nret = 0; ++ char image_path[PATH_MAX] = { 0x00 }; ++ ++ if (!image_store_lock(EXCLUSIVE)) { ++ ERROR("Failed to lock remote image store when handle: %s", id); ++ return -1; ++ } ++ ++ nret = snprintf(image_path, sizeof(image_path), "%s/%s", g_image_store->dir, id); ++ if (nret < 0 || (size_t)nret >= sizeof(image_path)) { ++ ERROR("Failed to get image path"); ++ return -1; ++ } ++ ++ ret = append_image_by_directory(image_path); ++ image_store_unlock(); ++ ++ return ret; ++} ++ ++int remove_image_from_memory_with_lock(const char *id) ++{ ++ int ret = 0; ++ ++ if (!image_store_lock(EXCLUSIVE)) { ++ ERROR("Failed to lock remote image store when handle: %s", id); ++ return -1; ++ } ++ ++ ret = remove_image_from_memory(id); ++ image_store_unlock(); ++ ++ return ret; ++} ++ ++char *get_top_layer_from_json(const char *img_id) ++{ ++ ++ char *ret = NULL; ++ int nret = 0; ++ char image_path[PATH_MAX] = { 0x00 }; ++ storage_image *im = NULL; ++ parser_error err = NULL; ++ ++ nret = snprintf(image_path, sizeof(image_path), "%s/%s/%s", g_image_store->dir, img_id, IMAGE_JSON); ++ if (nret < 0 || (size_t)nret >= sizeof(image_path)) { ++ ERROR("Failed to get image path"); ++ return NULL; ++ } ++ ++ im = storage_image_parse_file(image_path, NULL, &err); ++ if (im == NULL) { ++ ERROR("Failed to parse images path: %s", err); ++ return NULL; ++ } ++ ++ ret = util_strdup_s(im->layer); ++ free_storage_image(im); ++ ++ return ret; ++} ++#endif +diff --git a/src/daemon/modules/image/oci/storage/image_store/image_store.h b/src/daemon/modules/image/oci/storage/image_store/image_store.h +index edd4fa2c..c3cb50e3 100644 +--- a/src/daemon/modules/image/oci/storage/image_store/image_store.h ++++ b/src/daemon/modules/image/oci/storage/image_store/image_store.h +@@ -28,6 +28,9 @@ + #include "isula_libutils/imagetool_image.h" + #include "isula_libutils/imagetool_images_list.h" + #include "isula_libutils/imagetool_image_summary.h" ++#ifdef ENABLE_REMOTE_LAYER_STORE ++#include "remote_support.h" ++#endif + + struct storage_module_init_options; + +@@ -108,6 +111,14 @@ void image_store_free(); + + imagetool_image_summary *image_store_get_image_summary(const char *id); + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++remote_support *image_store_impl_remote_support(); ++int validate_manifest_schema_version_1(const char *path, bool *valid); ++int append_image_by_directory_with_lock(const char *image_dir); ++int remove_image_from_memory_with_lock(const char *id); ++char *get_top_layer_from_json(const char *img_id); /* return top layer id */ ++#endif ++ + #ifdef __cplusplus + } + #endif +diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.c b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.c +index d3b5209a..b83c63b1 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.c ++++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.c +@@ -141,7 +141,9 @@ int graphdriver_init(const struct storage_module_init_options *opts) + ret = -1; + goto out; + } +- ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ g_drivers[i].enable_remote_layer = opts->enable_remote_layer; ++#endif + if (g_drivers[i].ops->init(&g_drivers[i], driver_home, (const char **)opts->driver_opts, + opts->driver_opts_len) != 0) { + ret = -1; +diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.h b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.h +index 7faf70c8..acd847cc 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.h ++++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.h +@@ -90,6 +90,9 @@ struct graphdriver { + bool support_dtype; + + bool support_quota; ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ bool enable_remote_layer; ++#endif + struct pquota_control *quota_ctrl; + + // options for overlay2 +diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.c b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.c +index 330c230a..eedbeef2 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.c ++++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.c +@@ -12,6 +12,7 @@ + * Create: 2020-04-02 + * Description: provide overlay2 function definition + ******************************************************************************/ ++#define _GNU_SOURCE + #include "driver_overlay2.h" + + #include +@@ -44,6 +45,9 @@ + #include "utils_timestamp.h" + #include "selinux_label.h" + #include "err_msg.h" ++#ifdef ENABLE_REMOTE_LAYER_STORE ++#include "ro_symlink_maintain.h" ++#endif + + struct io_read_wrapper; + +@@ -343,6 +347,13 @@ int overlay2_init(struct graphdriver *driver, const char *driver_home, const cha + return -1; + } + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ if (driver->enable_remote_layer && remote_overlay_init(driver_home) != 0) { ++ ERROR("Failed to init overlay remote"); ++ return -1; ++ } ++#endif ++ + driver->home = util_strdup_s(driver_home); + + root_dir = util_path_dir(driver_home); +@@ -423,7 +434,7 @@ static int mk_diff_directory(const char *layer_dir) + int ret = 0; + char *diff_dir = NULL; + #ifdef ENABLE_USERNS_REMAP +- char* userns_remap = conf_get_isulad_userns_remap(); ++ char *userns_remap = conf_get_isulad_userns_remap(); + #endif + + diff_dir = util_path_join(layer_dir, OVERLAY_LAYER_DIFF); +@@ -538,7 +549,7 @@ static int mk_work_directory(const char *layer_dir) + int ret = 0; + char *work_dir = NULL; + #ifdef ENABLE_USERNS_REMAP +- char* userns_remap = conf_get_isulad_userns_remap(); ++ char *userns_remap = conf_get_isulad_userns_remap(); + #endif + + work_dir = util_path_join(layer_dir, OVERLAY_LAYER_WORK); +@@ -575,7 +586,7 @@ static int mk_merged_directory(const char *layer_dir) + int ret = 0; + char *merged_dir = NULL; + #ifdef ENABLE_USERNS_REMAP +- char* userns_remap = conf_get_isulad_userns_remap(); ++ char *userns_remap = conf_get_isulad_userns_remap(); + #endif + + merged_dir = util_path_join(layer_dir, OVERLAY_LAYER_MERGED); +@@ -852,13 +863,115 @@ out: + return ret; + } + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++static int do_create_remote_ro(const char *id, const char *parent, const struct graphdriver *driver, ++ const struct driver_create_opts *create_opts) ++{ ++ int ret = 0; ++ int get_err = 0; ++ char *ro_symlink = NULL; ++ char *ro_home = NULL; ++ char *layer_dir = NULL; ++#ifdef ENABLE_USERNS_REMAP ++ char *userns_remap = conf_get_isulad_userns_remap(); ++#endif ++ ++ ro_home = util_path_join(driver->home, OVERLAY_RO_DIR); ++ if (ro_home == NULL) { ++ ERROR("Failed to join ro_home"); ++ ret = -1; ++ goto out; ++ } ++ ++ layer_dir = util_path_join(ro_home, id); ++ if (layer_dir == NULL) { ++ ERROR("Failed to join layer_dir"); ++ ret = -1; ++ goto out; ++ } ++ ++ ro_symlink = util_path_join(driver->home, id); ++ if (ro_symlink == NULL) { ++ ERROR("Failed to join ro_symlink"); ++ ret = -1; ++ goto out; ++ } ++ ++ if (layer_dir == NULL) { ++ ERROR("Failed to join layer dir:%s", id); ++ ret = -1; ++ goto out; ++ } ++ ++ if (check_parent_valid(parent, driver) != 0) { ++ ret = -1; ++ goto out; ++ } ++ ++ if (util_mkdir_p(layer_dir, 0700) != 0) { ++ ERROR("Unable to create layer directory %s.", layer_dir); ++ ret = -1; ++ goto out; ++ } ++ ++ // mk symbol link ++ if (symlink(layer_dir, ro_symlink) != 0) { ++ SYSERROR("Unable to create symbol link to layer directory %s", layer_dir); ++ ret = -1; ++ goto err_out; ++ } ++ ++#ifdef ENABLE_USERNS_REMAP ++ if (set_file_owner_for_userns_remap(layer_dir, userns_remap) != 0) { ++ ERROR("Unable to change directory %s owner for user remap.", layer_dir); ++ ret = -1; ++ goto out; ++ } ++#endif ++ ++ if (create_opts->storage_opt != NULL && create_opts->storage_opt->len != 0) { ++ if (set_layer_quota(layer_dir, create_opts->storage_opt, driver) != 0) { ++ ERROR("Unable to set layer quota %s", layer_dir); ++ ret = -1; ++ goto err_out; ++ } ++ } ++ ++ if (mk_sub_directories(id, parent, layer_dir, driver->home) != 0) { ++ ret = -1; ++ goto err_out; ++ } ++ ++ goto out; ++ ++err_out: ++ if (util_recursive_rmdir(layer_dir, 0)) { ++ ERROR("Failed to delete layer path: %s", layer_dir); ++ } ++ ++ // to remove a file ++ if (util_fileself_exists(ro_symlink) && !util_force_remove_file(ro_symlink, &get_err)) { ++ ERROR("Failed to remove symbol link %s", ro_symlink); ++ } ++ ++out: ++ free(layer_dir); ++ free(ro_home); ++ free(ro_symlink); ++#ifdef ENABLE_USERNS_REMAP ++ free(userns_remap); ++#endif ++ return ret; ++} ++#endif ++ + static int do_create(const char *id, const char *parent, const struct graphdriver *driver, + const struct driver_create_opts *create_opts) + { + int ret = 0; + char *layer_dir = NULL; + #ifdef ENABLE_USERNS_REMAP +- char* userns_remap = conf_get_isulad_userns_remap(); ++ char *userns_remap = conf_get_isulad_userns_remap(); + #endif + + layer_dir = util_path_join(driver->home, id); +@@ -1002,7 +1115,15 @@ int overlay2_create_ro(const char *id, const char *parent, const struct graphdri + return -1; + } + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ if (driver->enable_remote_layer) { ++ return do_create_remote_ro(id, parent, driver, create_opts); ++ } else { ++ return do_create(id, parent, driver, create_opts); ++ } ++#else + return do_create(id, parent, driver, create_opts); ++#endif + } + + static char *read_layer_link_file(const char *layer_dir) +@@ -1047,6 +1168,9 @@ int overlay2_rm_layer(const char *id, const struct graphdriver *driver) + char *link_id = NULL; + char link_path[PATH_MAX] = { 0 }; + char clean_path[PATH_MAX] = { 0 }; ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ struct stat stat_buf; ++#endif + + if (id == NULL || driver == NULL) { + ERROR("Invalid input arguments"); +@@ -1079,11 +1203,34 @@ int overlay2_rm_layer(const char *id, const struct graphdriver *driver) + } + } + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ if (lstat(layer_dir, &stat_buf) < 0) { ++ SYSERROR("Failed to lstat path: %s", layer_dir); ++ ret = -1; ++ goto out; ++ } ++ ++ if (driver->enable_remote_layer && S_ISLNK(stat_buf.st_mode)) { ++ // jusdge if the dir is symlink? ++ if (remote_overlay_remove_ro_dir(id) != 0) { ++ ERROR("Failed to delete symlink to layer dir: %s", layer_dir); ++ ret = -1; ++ goto out; ++ } ++ } else { ++ if (util_recursive_rmdir(layer_dir, 0) != 0) { ++ SYSERROR("Failed to remove layer directory %s", layer_dir); ++ ret = -1; ++ goto out; ++ } ++ } ++#else + if (util_recursive_rmdir(layer_dir, 0) != 0) { + SYSERROR("Failed to remove layer directory %s", layer_dir); + ret = -1; + goto out; + } ++#endif + + out: + free(layer_dir); +@@ -1747,7 +1894,7 @@ int overlay2_apply_diff(const char *id, const struct graphdriver *driver, const + int ret = 0; + #ifdef ENABLE_USERNS_REMAP + unsigned int size = 0; +- char* userns_remap = conf_get_isulad_userns_remap(); ++ char *userns_remap = conf_get_isulad_userns_remap(); + #endif + char *layer_dir = NULL; + char *layer_diff = NULL; +@@ -2166,3 +2313,4 @@ out: + free(layer_diff); + return ret; + } ++ +diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.h b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.h +index e14271b1..5c1d93fb 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.h ++++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.h +@@ -22,6 +22,9 @@ + #include + + #include "driver.h" ++#ifdef ENABLE_REMOTE_LAYER_STORE ++#include "remote_support.h" ++#endif + + struct driver_create_opts; + struct driver_mount_opts; +@@ -68,6 +71,11 @@ int overlay2_repair_lowers(const char *id, const char *parent, const struct grap + + int overlay2_get_layer_fs_info(const char *id, const struct graphdriver *driver, imagetool_fs_info *fs_info); + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++remote_support *overlay_driver_impl_remote_support(void); ++bool overlay_remote_layer_valid(const char *layer_id); ++#endif ++ + #ifdef __cplusplus + } + #endif +diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/overlay_remote_impl.c b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/overlay_remote_impl.c +new file mode 100644 +index 00000000..a674a00f +--- /dev/null ++++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/overlay_remote_impl.c +@@ -0,0 +1,282 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: wangrunze ++ * Create: 2023-02-27 ++ * Description: provide remote implementation for driver overlay ++ ******************************************************************************/ ++#define _GNU_SOURCE ++#include "driver_overlay2.h" ++ ++#include ++ ++#include "map.h" ++#include "remote_support.h" ++#include "ro_symlink_maintain.h" ++#include "isula_libutils/log.h" ++#include "utils.h" ++#include "utils_array.h" ++#include "utils_file.h" ++#include "path.h" ++ ++#define OVERLAY_LINK_DIR "l" ++#define OVERLAY_LAYER_LINK "link" ++ ++struct remote_overlay_data { ++ const char *overlay_home; ++ const char *overlay_ro; ++}; ++ ++static map_t *overlay_byid_old = NULL; ++static map_t *overlay_byid_new = NULL; ++ ++static void *remote_support_create(const char *remote_home, const char *remote_ro) ++{ ++ struct remote_overlay_data *data = util_common_calloc_s(sizeof(struct remote_overlay_data)); ++ if (data == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ data->overlay_home = remote_home; ++ data->overlay_ro = remote_ro; ++ overlay_byid_old = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); ++ overlay_byid_new = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); ++ ++ return data; ++} ++ ++static void remote_support_destroy(void *data) ++{ ++ if (data == NULL) { ++ return; ++ } ++ ++ map_free(overlay_byid_old); ++ map_free(overlay_byid_new); ++ free(data); ++} ++ ++static bool overlay_walk_dir_cb(const char *path_name, const struct dirent *sub_dir, void *context) ++{ ++ bool exist = true; ++ if (!map_insert(overlay_byid_new, util_strdup_s(sub_dir->d_name), (void *)&exist)) { ++ ERROR("can't insert remote layer into map"); ++ return false; ++ } ++ ++ return true; ++} ++ ++static int remote_support_scan(void *data) ++{ ++ struct remote_overlay_data *remote_data = data; ++ return util_scan_subdirs(remote_data->overlay_ro, overlay_walk_dir_cb, data); ++} ++ ++static int do_diff_symlink(const char *id, char *link_id, const char *driver_home) ++{ ++ int ret = 0; ++ int nret = 0; ++ char target_path[PATH_MAX] = { 0 }; ++ char link_path[PATH_MAX] = { 0 }; ++ char clean_path[PATH_MAX] = { 0 }; ++ ++ nret = snprintf(target_path, PATH_MAX, "../%s/diff", id); ++ if (nret < 0 || nret >= PATH_MAX) { ++ ERROR("Failed to get target path %s", id); ++ ret = -1; ++ goto out; ++ } ++ ++ nret = snprintf(link_path, PATH_MAX, "%s/%s/%s", driver_home, OVERLAY_LINK_DIR, link_id); ++ if (nret < 0 || nret >= PATH_MAX) { ++ ERROR("Failed to get link path %s", link_id); ++ ret = -1; ++ goto out; ++ } ++ ++ if (util_clean_path(link_path, clean_path, sizeof(clean_path)) == NULL) { ++ ERROR("failed to get clean path %s", link_path); ++ ret = -1; ++ goto out; ++ } ++ ++ if (util_fileself_exists(clean_path) && util_path_remove(clean_path) != 0) { ++ ERROR("failed to remove old symbol link"); ++ ret = -1; ++ goto out; ++ } ++ ++ nret = symlink(target_path, clean_path); ++ if (nret < 0) { ++ SYSERROR("Failed to create symlink from \"%s\" to \"%s\"", clean_path, target_path); ++ ret = -1; ++ goto out; ++ } ++ ++out: ++ return ret; ++} ++ ++static int remove_one_remote_overlay_layer(struct remote_overlay_data *data, const char *overlay_id) ++{ ++ char *ro_symlink = NULL; ++ char clean_path[PATH_MAX] = { 0 }; ++ int nret = 0; ++ int ret = 0; ++ ++ nret = asprintf(&ro_symlink, "%s/%s", data->overlay_home, overlay_id); ++ if (nret < 0 || nret > PATH_MAX) { ++ SYSERROR("Create layer symbol link path failed"); ++ ret = -1; ++ goto out; ++ } ++ ++ if (util_clean_path(ro_symlink, clean_path, sizeof(clean_path)) == NULL) { ++ ERROR("Failed to clean path: %s", ro_symlink); ++ ret = -1; ++ goto out; ++ } ++ ++ if (util_path_remove(clean_path) != 0) { ++ SYSERROR("Failed to remove link path %s", clean_path); ++ } ++ ++out: ++ free(ro_symlink); ++ return ret; ++} ++ ++static int add_one_remote_overlay_layer(struct remote_overlay_data *data, const char *overlay_id) ++{ ++ char *ro_symlink = NULL; ++ char *layer_dir = NULL; ++ char *link_file = NULL; ++ char *diff_symlink = NULL; ++ int ret = 0; ++ ++ ro_symlink = util_path_join(data->overlay_home, overlay_id); ++ if (ro_symlink == NULL) { ++ ERROR("Failed to join ro symlink path: %s", overlay_id); ++ ret = -1; ++ goto free_out; ++ } ++ ++ layer_dir = util_path_join(data->overlay_ro, overlay_id); ++ if (layer_dir == NULL) { ++ ERROR("Failed to join ro layer dir: %s", overlay_id); ++ ret = -1; ++ goto free_out; ++ } ++ ++ // add RO symbol link first ++ if (!util_fileself_exists(ro_symlink) && symlink(layer_dir, ro_symlink) != 0) { ++ SYSERROR("Unable to create symbol link to layer directory: %s", layer_dir); ++ ret = -1; ++ goto free_out; ++ } ++ ++ // maintain link ++ // try read link file in layer_dir ++ // mk symlink between ro_symlink ++ link_file = util_path_join(layer_dir, OVERLAY_LAYER_LINK); ++ if (link_file == NULL) { ++ ERROR("Failed to get layer link file %s", layer_dir); ++ ret = -1; ++ goto free_out; ++ } ++ ++ if (!util_fileself_exists(link_file)) { ++ ERROR("link file for layer %s not exist", layer_dir); ++ ret = -1; ++ goto free_out; ++ } ++ ++ diff_symlink = util_read_content_from_file(link_file); ++ if (link_file == NULL) { ++ ERROR("Failed to read content from link file of layer %s", layer_dir); ++ ret = -1; ++ goto free_out; ++ } ++ ++ if (do_diff_symlink(overlay_id, diff_symlink, data->overlay_home) != 0) { ++ ERROR("Failed to add diff link for layer %s", overlay_id); ++ ret = -1; ++ } ++ ++free_out: ++ free(ro_symlink); ++ free(layer_dir); ++ free(link_file); ++ free(diff_symlink); ++ ++ return ret; ++} ++ ++static int remote_support_add(void *data) ++{ ++ int ret = 0; ++ char **array_added = NULL; ++ char **array_deleted = NULL; ++ map_t *tmp_map = NULL; ++ int i = 0; ++ ++ if (data == NULL) { ++ return -1; ++ } ++ ++ array_added = added_layers(overlay_byid_old, overlay_byid_new); ++ array_deleted = deleted_layers(overlay_byid_old, overlay_byid_new); ++ ++ for (i = 0; i < util_array_len((const char **)array_added); i++) { ++ if (add_one_remote_overlay_layer(data, array_added[i]) != 0) { ++ ERROR("Failed to add remote overlay layer: %s", array_added[i]); ++ ret = -1; ++ } ++ } ++ ++ for (i = 0; i < util_array_len((const char **)array_deleted); i++) { ++ if (remove_one_remote_overlay_layer(data, array_deleted[i]) != 0) { ++ ERROR("Failed to delete remote overlay layer: %s", array_deleted[i]); ++ ret = -1; ++ } ++ } ++ ++ tmp_map = overlay_byid_old; ++ overlay_byid_old = overlay_byid_new; ++ overlay_byid_new = tmp_map; ++ empty_map(overlay_byid_new); ++ ++ util_free_array(array_added); ++ util_free_array(array_deleted); ++ ++ return ret; ++} ++ ++remote_support *overlay_driver_impl_remote_support(void) ++{ ++ remote_support *rs = util_common_calloc_s(sizeof(remote_support)); ++ if (rs == NULL) { ++ ERROR("Failed to calloc overlay supporter"); ++ return NULL; ++ } ++ ++ rs->create = remote_support_create; ++ rs->destroy = remote_support_destroy; ++ rs->scan_remote_dir = remote_support_scan; ++ rs->load_item = remote_support_add; ++ ++ return rs; ++} ++ ++bool overlay_remote_layer_valid(const char *layer_id) ++{ ++ return map_search(overlay_byid_old, (void *)layer_id) != NULL; ++} +diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c b/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c +new file mode 100644 +index 00000000..d03fc20b +--- /dev/null ++++ b/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c +@@ -0,0 +1,219 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: wangrunze ++ * Create: 2023-02-27 ++ * Description: remote layer store implementation ++ ******************************************************************************/ ++#define _GNU_SOURCE ++#include "layer_store.h" ++ ++#include ++#include ++#include ++ ++#include "map.h" ++#include "utils.h" ++#include "remote_support.h" ++#include "ro_symlink_maintain.h" ++#include "path.h" ++#include "driver_overlay2.h" ++ ++struct remote_layer_data { ++ const char *layer_home; ++ const char *layer_ro; ++}; ++ ++static map_t *layer_byid_old = NULL; ++static map_t *layer_byid_new = NULL; ++ ++static void *remote_support_create(const char *layer_home, const char *layer_ro) ++{ ++ struct remote_layer_data *data = util_common_calloc_s(sizeof(struct remote_layer_data)); ++ if (data == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ data->layer_home = util_strdup_s(layer_home); ++ data->layer_ro = util_strdup_s(layer_ro); ++ layer_byid_old = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); ++ layer_byid_new = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); ++ ++ return data; ++}; ++ ++static void remote_support_destroy(void *data) ++{ ++ if (data == NULL) { ++ return; ++ } ++ ++ map_free(layer_byid_old); ++ map_free(layer_byid_new); ++ free(data); ++} ++ ++static bool layer_walk_dir_cb(const char *path_name, const struct dirent *sub_dir, void *context) ++{ ++ bool exist = true; ++ ++ if (!map_insert(layer_byid_new, util_strdup_s(sub_dir->d_name), (void *)&exist)) { ++ ERROR("can't insert remote layer into map"); ++ return false; ++ } ++ ++ return true; ++} ++ ++static int remote_support_scan(void *data) ++{ ++ struct remote_layer_data *remote_data = data; ++ return util_scan_subdirs(remote_data->layer_ro, layer_walk_dir_cb, data); ++} ++ ++static int remove_one_remote_layer(struct remote_layer_data *data, char *layer_id) ++{ ++ char *ro_symlink = NULL; ++ char clean_path[PATH_MAX] = { 0 }; ++ int nret = 0; ++ int ret = 0; ++ ++ nret = asprintf(&ro_symlink, "%s/%s", data->layer_home, layer_id); ++ if (nret < 0 || nret > PATH_MAX) { ++ SYSERROR("Create layer symbol link path failed"); ++ ret = -1; ++ goto out; ++ } ++ ++ if (util_clean_path(ro_symlink, clean_path, sizeof(clean_path)) == NULL) { ++ ERROR("Failed to clean path: %s", ro_symlink); ++ ret = -1; ++ goto out; ++ } ++ ++ if (util_path_remove(clean_path) != 0) { ++ SYSERROR("Failed to remove link path %s", clean_path); ++ } ++ ++ if (remove_memory_stores_with_lock(layer_id) != 0) { ++ ERROR("Failed to remove remote layer store memory"); ++ ret = -1; ++ } ++ ++out: ++ free(ro_symlink); ++ return ret; ++ ++} ++ ++static int add_one_remote_layer(struct remote_layer_data *data, char *layer_id) ++{ ++ char *ro_symlink = NULL; ++ char *layer_dir = NULL; ++ int ret = 0; ++ ++ ro_symlink = util_path_join(data->layer_home, layer_id); ++ layer_dir = util_path_join(data->layer_ro, layer_id); ++ ++ if (ro_symlink == NULL) { ++ ERROR("Failed to join ro symlink path: %s", layer_id); ++ ret = -1; ++ goto free_out; ++ } ++ ++ if (layer_dir == NULL) { ++ ERROR("Failed to join ro layer dir: %s", layer_id); ++ ret = -1; ++ goto free_out; ++ } ++ // add symbol link first ++ if (!util_fileself_exists(ro_symlink) && symlink(layer_dir, ro_symlink) != 0) { ++ SYSERROR("Unable to create symbol link to layer directory: %s", layer_dir); ++ ret = -1; ++ goto free_out; ++ } ++ // insert layer into memory ++ if (load_one_layer(layer_id) != 0) { ++ ERROR("Failed to load new layer: %s into memory", layer_id); ++ ret = -1; ++ } ++ ++free_out: ++ free(ro_symlink); ++ free(layer_dir); ++ ++ return ret; ++} ++ ++static int remote_support_add(void *data) ++{ ++ int ret = 0; ++ char **array_added = NULL; ++ char **array_deleted = NULL; ++ map_t *tmp_map = NULL; ++ int i = 0; ++ ++ if (data == NULL) { ++ return -1; ++ } ++ ++ array_added = added_layers(layer_byid_old, layer_byid_new); ++ array_deleted = deleted_layers(layer_byid_old, layer_byid_new); ++ ++ for (i = 0; i < util_array_len((const char **)array_added); i++) { ++ if (!overlay_remote_layer_valid(array_added[i]) != 0) { ++ map_remove(layer_byid_new, (void *)array_added[i]); ++ ERROR("remote overlay layer current not valid: %s", array_added[i]); ++ continue; ++ } ++ ++ if (add_one_remote_layer(data, array_added[i]) != 0) { ++ ERROR("Failed to add remote overlay layer: %s", array_added[i]); ++ ret = -1; ++ } ++ } ++ ++ for (i = 0; i < util_array_len((const char **)array_deleted); i++) { ++ if (remove_one_remote_layer(data, array_deleted[i]) != 0) { ++ ERROR("Failed to delete remote overlay layer: %s", array_deleted[i]); ++ ret = -1; ++ } ++ } ++ ++ tmp_map = layer_byid_old; ++ layer_byid_old = layer_byid_new; ++ layer_byid_new = tmp_map; ++ empty_map(layer_byid_new); ++ ++ util_free_array(array_added); ++ util_free_array(array_deleted); ++ ++ return ret; ++} ++ ++remote_support *layer_store_impl_remote_support() ++{ ++ remote_support *rs = util_common_calloc_s(sizeof(remote_support)); ++ if (rs == NULL) { ++ return NULL; ++ } ++ ++ rs->create = remote_support_create; ++ rs->destroy = remote_support_destroy; ++ rs->scan_remote_dir = remote_support_scan; ++ rs->load_item = remote_support_add; ++ ++ return rs; ++} ++ ++bool layer_remote_layer_valid(const char *layer_id) ++{ ++ return map_search(layer_byid_old, (void *)layer_id) != NULL; ++} +diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c +index 89b4f58c..c00c3356 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c ++++ b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c +@@ -30,7 +30,6 @@ + #include + #include + #include +- + #include + #include + +@@ -48,6 +47,10 @@ + #include "http.h" + #include "utils_base64.h" + #include "constants.h" ++#include "path.h" ++#ifdef ENABLE_REMOTE_LAYER_STORE ++#include "ro_symlink_maintain.h" ++#endif + + #define PAYLOAD_CRC_LEN 12 + +@@ -74,6 +77,9 @@ typedef struct { + static layer_store_metadata g_metadata; + static char *g_root_dir; + static char *g_run_dir; ++#ifdef ENABLE_REMOTE_LAYER_STORE ++static bool g_enable_remote_layer; ++#endif + + static inline char *tar_split_path(const char *id); + static inline char *mountpoint_json_path(const char *id); +@@ -127,7 +133,7 @@ void layer_store_cleanup() + map_free(g_metadata.by_uncompress_digest); + g_metadata.by_uncompress_digest = NULL; + +- linked_list_for_each_safe(item, &(g_metadata.layers_list), next) { ++ linked_list_for_each_safe (item, &(g_metadata.layers_list), next) { + linked_list_del(item); + layer_ref_dec((layer_t *)item->elem); + free(item); +@@ -158,7 +164,7 @@ static void free_digest_layer_t(digest_layer_t *ptr) + return; + } + +- linked_list_for_each_safe(item, &(ptr->layer_list), next) { ++ linked_list_for_each_safe (item, &(ptr->layer_list), next) { + linked_list_del(item); + free(item->elem); + item->elem = NULL; +@@ -272,6 +278,10 @@ static bool init_from_conf(const struct storage_module_init_options *conf) + g_root_dir = tmp_path; + tmp_path = NULL; + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ g_enable_remote_layer = conf->enable_remote_layer; ++#endif ++ + return true; + free_out: + free(g_run_dir); +@@ -609,7 +619,7 @@ static int delete_digest_from_map(map_t *by_digest, const char *digest, const ch + return 0; + } + +- linked_list_for_each_safe(item, &(old_list->layer_list), next) { ++ linked_list_for_each_safe (item, &(old_list->layer_list), next) { + char *t_id = (char *)item->elem; + if (strcmp(t_id, id) == 0) { + linked_list_del(item); +@@ -722,7 +732,7 @@ static int remove_memory_stores(const char *id) + } + } + +- linked_list_for_each_safe(item, &(g_metadata.layers_list), next) { ++ linked_list_for_each_safe (item, &(g_metadata.layers_list), next) { + layer_t *tl = (layer_t *)item->elem; + if (strcmp(tl->slayer->id, id) != 0) { + continue; +@@ -1115,10 +1125,25 @@ static int new_layer_by_opts(const char *id, const struct layer_opts *opts) + ret = -1; + goto out; + } ++ ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ if (g_enable_remote_layer && !opts->writable) { ++ if (remote_layer_build_ro_dir(id) != 0) { ++ ret = -1; ++ goto out; ++ } ++ } else { ++ if (!build_layer_dir(id)) { ++ ret = -1; ++ goto out; ++ } ++ } ++#else + if (!build_layer_dir(id)) { + ret = -1; + goto out; + } ++#endif + + ret = update_layer_datas(id, opts, l); + if (ret != 0) { +@@ -1300,7 +1325,15 @@ clear_memory: + driver_remove: + if (ret != 0) { + (void)graphdriver_rm_layer(lid); ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ if (g_enable_remote_layer) { ++ (void)remote_layer_remove_ro_dir(lid); ++ } else { ++ (void)layer_store_remove_layer(lid); ++ } ++#else + (void)layer_store_remove_layer(lid); ++#endif + } + free_out: + layer_store_unlock(); +@@ -1375,7 +1408,15 @@ static int do_delete_layer(const char *id) + goto free_out; + } + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ if (l->slayer->writable) { ++ ret = layer_store_remove_layer(l->slayer->id); ++ } else { ++ ret = remote_layer_remove_ro_dir(l->slayer->id); ++ } ++#else + ret = layer_store_remove_layer(l->slayer->id); ++#endif + + free_out: + free(tspath); +@@ -1457,7 +1498,7 @@ int layer_store_list(struct layer_list *resp) + goto unlock; + } + +- linked_list_for_each_safe(item, &(g_metadata.layers_list), next) { ++ linked_list_for_each_safe (item, &(g_metadata.layers_list), next) { + layer_t *l = (layer_t *)item->elem; + resp->layers[i] = util_common_calloc_s(sizeof(struct layer)); + if (resp->layers[i] == NULL) { +@@ -1500,7 +1541,7 @@ static int layers_by_digest_map(map_t *m, const char *digest, struct layer_list + goto free_out; + } + +- linked_list_for_each_safe(item, &(id_list->layer_list), next) { ++ linked_list_for_each_safe (item, &(id_list->layer_list), next) { + layer_t *l = NULL; + resp->layers[i] = util_common_calloc_s(sizeof(struct layer)); + if (resp->layers[i] == NULL) { +@@ -1744,6 +1785,114 @@ out: + return ret; + } + ++static layer_t *load_one_layer_from_json(const char *id) ++{ ++ int nret = 0; ++ char *mount_point_path = NULL; ++ char tmpdir[PATH_MAX] = { 0 }; ++ char *rpath = NULL; ++ layer_t *l = NULL; ++ bool layer_valid = false; ++ ++ nret = snprintf(tmpdir, PATH_MAX, "%s/%s", g_root_dir, id); ++ if (nret < 0 || nret >= PATH_MAX) { ++ ERROR("Sprintf: %s failed", id); ++ goto free_out; ++ } ++ ++ mount_point_path = mountpoint_json_path(id); ++ if (mount_point_path == NULL) { ++ ERROR("Out of Memory"); ++ goto free_out; ++ } ++ ++ rpath = layer_json_path(id); ++ if (rpath == NULL) { ++ ERROR("%s is invalid layer", id); ++ goto free_out; ++ } ++ ++ l = load_layer(rpath, mount_point_path); ++ if (l == NULL) { ++ ERROR("load layer: %s failed, remove it", id); ++ goto free_out; ++ } ++ ++ if (do_validate_image_layer(tmpdir, l) != 0) { ++ ERROR("%s is invalid image layer", id); ++ goto free_out; ++ } ++ ++ if (do_validate_rootfs_layer(l) != 0) { ++ ERROR("%s is invalid rootfs layer", id); ++ goto free_out; ++ } ++ ++ layer_valid = true; ++ ++free_out: ++ free(rpath); ++ free(mount_point_path); ++ if (!layer_valid) { ++ free_layer_t(l); ++ l = NULL; ++ } ++ // always return true; ++ // if load layer failed, just remove it ++ return l; ++} ++ ++int load_one_layer(const char *id) ++{ ++ int ret = 0; ++ layer_t *tl = NULL; ++ int i = 0; ++ ++ if (!layer_store_lock(true)) { ++ return -1; ++ } ++ ++ tl = load_one_layer_from_json(id); ++ if (tl == NULL) { ++ ret = -1; ++ goto unlock_out; ++ } ++ ++ if (!map_insert(g_metadata.by_id, (void *)tl->slayer->id, (void *)tl)) { ++ ERROR("Insert id: %s for layer failed", tl->slayer->id); ++ ret = -1; ++ goto unlock_out; ++ } ++ ++ for (; i < tl->slayer->names_len; i++) { ++ // this should be done by master isulad ++ // if (remove_name(tl->slayer->names[i])) { ++ // should_save = true; ++ // } ++ if (!map_insert(g_metadata.by_name, (void *)tl->slayer->names[i], (void *)tl)) { ++ ret = -1; ++ ERROR("Insert name: %s for layer failed", tl->slayer->names[i]); ++ goto unlock_out; ++ } ++ } ++ ret = insert_digest_into_map(g_metadata.by_compress_digest, tl->slayer->compressed_diff_digest, tl->slayer->id); ++ if (ret != 0) { ++ ERROR("update layer: %s compress failed", tl->slayer->id); ++ goto unlock_out; ++ } ++ ++ ret = insert_digest_into_map(g_metadata.by_uncompress_digest, tl->slayer->diff_digest, tl->slayer->id); ++ if (ret != 0) { ++ ERROR("update layer: %s uncompress failed", tl->slayer->id); ++ goto unlock_out; ++ } ++ ++ ret = 0; ++unlock_out: ++ layer_store_unlock(); ++ return ret; ++} ++ + static bool load_layer_json_cb(const char *path_name, const struct dirent *sub_dir, void *context) + { + #define LAYER_NAME_LEN 64 +@@ -1760,6 +1909,14 @@ static bool load_layer_json_cb(const char *path_name, const struct dirent *sub_d + goto free_out; + } + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ // skip RO dir ++ // otherwise, RO dir will be treat as invalid layer dir ++ if (strcmp(sub_dir->d_name, REMOTE_RO_LAYER_DIR) == 0) { ++ goto free_out; ++ } ++#endif ++ + if (!util_dir_exists(tmpdir)) { + // ignore non-dir + DEBUG("%s is not directory", sub_dir->d_name); +@@ -1846,7 +2003,7 @@ static int load_layers_from_json_files() + goto unlock_out; + } + +- linked_list_for_each_safe(item, &(g_metadata.layers_list), next) { ++ linked_list_for_each_safe (item, &(g_metadata.layers_list), next) { + layer_t *tl = (layer_t *)item->elem; + size_t i = 0; + +@@ -1953,6 +2110,13 @@ int layer_store_init(const struct storage_module_init_options *conf) + goto free_out; + } + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ if (g_enable_remote_layer && remote_layer_init(g_root_dir) != 0) { ++ ERROR("Failed to init layer remote"); ++ goto free_out; ++ } ++#endif ++ + if (load_layers_from_json_files() != 0) { + goto free_out; + } +@@ -2125,7 +2289,7 @@ static tar_split *new_tar_split(layer_t *l, const char *tspath) + int ret = 0; + int nret = 0; + tar_split *ts = NULL; +- char path[PATH_MAX] = {0}; ++ char path[PATH_MAX] = { 0 }; + + ts = util_common_calloc_s(sizeof(tar_split)); + if (ts == NULL) { +@@ -2308,3 +2472,20 @@ container_inspect_graph_driver *layer_store_get_metadata_by_layer_id(const char + { + return graphdriver_get_metadata(id); + } ++ ++#ifdef ENABLE_REMOTE_LAYER_STORE ++int remove_memory_stores_with_lock(const char *id) ++{ ++ int ret = 0; ++ ++ if (!layer_store_lock(true)) { ++ ERROR("Failed to lock layer store when handle: %s", id); ++ return -1; ++ } ++ ++ ret = remove_memory_stores(id); ++ layer_store_unlock(); ++ ++ return ret; ++} ++#endif +diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_store.h b/src/daemon/modules/image/oci/storage/layer_store/layer_store.h +index 94d4bf04..44bd297e 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/layer_store.h ++++ b/src/daemon/modules/image/oci/storage/layer_store/layer_store.h +@@ -23,6 +23,10 @@ + + #include "storage.h" + #include "io_wrapper.h" ++#include "map.h" ++#ifdef ENABLE_REMOTE_LAYER_STORE ++#include "remote_support.h" ++#endif + + struct io_read_wrapper; + struct layer_list; +@@ -79,6 +83,13 @@ int layer_store_check(const char *id); + + container_inspect_graph_driver *layer_store_get_metadata_by_layer_id(const char *id); + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++remote_support *layer_store_impl_remote_support(); ++bool layer_remote_layer_valid(const char *layer_id); ++int load_one_layer(const char *id); ++int remove_memory_stores_with_lock(const char *id); ++#endif ++ + #ifdef __cplusplus + } + #endif +diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/CMakeLists.txt b/src/daemon/modules/image/oci/storage/remote_layer_support/CMakeLists.txt +new file mode 100644 +index 00000000..06c78678 +--- /dev/null ++++ b/src/daemon/modules/image/oci/storage/remote_layer_support/CMakeLists.txt +@@ -0,0 +1,12 @@ ++# get current directory sources files ++aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_remote_layer_support_srcs) ++ ++set(REMOTE_LAYER_SUPPORT_SRCS ++ ${local_remote_layer_support_srcs} ++ PARENT_SCOPE ++ ) ++ ++set(REMOTE_LAYER_SUPPORT_INCS ++ ${CMAKE_CURRENT_SOURCE_DIR} ++ PARENT_SCOPE ++) +\ No newline at end of file +diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c +new file mode 100644 +index 00000000..9dc096f7 +--- /dev/null ++++ b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c +@@ -0,0 +1,122 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: wangrunze ++ * Create: 2023-03-03 ++ * Description: provide image store functions ++ ******************************************************************************/ ++ ++#include "remote_support.h" ++ ++#include "layer_store.h" ++#include "image_store.h" ++#include "isula_libutils/log.h" ++#include "driver_overlay2.h" ++#include "utils.h" ++ ++remote_supporter *create_layer_supporter(const char *remote_home, const char *remote_ro) ++{ ++ remote_support *handlers = layer_store_impl_remote_support(); ++ if (handlers == NULL || handlers->create == NULL) { ++ return NULL; ++ } ++ ++ remote_supporter *supporter = (remote_supporter *)util_common_calloc_s(sizeof(remote_supporter)); ++ if (supporter == NULL) { ++ goto err_out; ++ } ++ ++ supporter->handlers = handlers; ++ supporter->data = handlers->create(remote_home, remote_ro); ++ ++ return supporter; ++ ++err_out: ++ free(handlers); ++ free(supporter); ++ return NULL; ++} ++ ++remote_supporter *create_image_supporter(const char *remote_home, const char *remote_ro) ++{ ++ remote_support *handlers = image_store_impl_remote_support(); ++ if (handlers == NULL || handlers->create == NULL) { ++ return NULL; ++ } ++ ++ remote_supporter *supporter = (remote_supporter *)util_common_calloc_s(sizeof(remote_supporter)); ++ if (supporter == NULL) { ++ goto err_out; ++ } ++ ++ supporter->handlers = handlers; ++ supporter->data = handlers->create(remote_home, remote_ro); ++ ++ return supporter; ++ ++err_out: ++ free(handlers); ++ free(supporter); ++ return NULL; ++} ++ ++remote_supporter *create_overlay_supporter(const char *remote_home, const char *remote_ro) ++{ ++ remote_support *handlers = overlay_driver_impl_remote_support(); ++ if (handlers == NULL || handlers->create == NULL) { ++ return NULL; ++ } ++ ++ remote_supporter *supporter = (remote_supporter *)util_common_calloc_s(sizeof(remote_supporter)); ++ if (supporter == NULL) { ++ goto err_out; ++ } ++ ++ supporter->handlers = handlers; ++ supporter->data = handlers->create(remote_home, remote_ro); ++ ++ return supporter; ++ ++err_out: ++ free(handlers); ++ free(supporter); ++ return NULL; ++ ++} ++ ++void destroy_suppoter(remote_supporter *supporter) ++{ ++ if (supporter->handlers->destroy == NULL) { ++ ERROR("destroy_supporter operation not supported"); ++ return; ++ } ++ ++ supporter->handlers->destroy(supporter->data); ++ free(supporter->handlers); ++ free(supporter); ++} ++ ++int scan_remote_dir(remote_supporter *supporter) ++{ ++ if (supporter->handlers->scan_remote_dir == NULL) { ++ ERROR("scan_remote_dir operation not supported"); ++ return -1; ++ } ++ return supporter->handlers->scan_remote_dir(supporter->data); ++} ++ ++int load_item(remote_supporter *supporter) ++{ ++ if (supporter->handlers->scan_remote_dir == NULL) { ++ ERROR("load_item operation not supported"); ++ return -1; ++ } ++ return supporter->handlers->load_item(supporter->data); ++} +diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h +new file mode 100644 +index 00000000..d1f7af35 +--- /dev/null ++++ b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h +@@ -0,0 +1,58 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2020-2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: wangrunze ++ * Create: 2023-03-03 ++ * Description: provide remote support functions ++ ******************************************************************************/ ++ ++#ifndef DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_REMOTE_SUPPORT_H ++#define DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_REMOTE_SUPPORT_H ++ ++#include "linked_list.h" ++#define REMOTE_RO_LAYER_DIR "RO" ++#define OVERLAY_RO_DIR "RO" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++typedef struct { ++ void *(*create)(const char *remote_home, const char *remote_ro); ++ void (*destroy)(void *data); ++ // populate the list contains all dirs ++ int (*scan_remote_dir)(void *data); ++ // consume the list contains all dirs ++ int (*load_item)(void *data); ++} remote_support; ++ ++typedef struct { ++ void *data; ++ remote_support *handlers; ++} remote_supporter; ++ ++// RemoteSupport *impl_remote_support(); ++remote_supporter *create_image_supporter(const char *remote_home, const char *remote_ro); ++ ++remote_supporter *create_layer_supporter(const char *remote_home, const char *remote_ro); ++ ++remote_supporter *create_overlay_supporter(const char *remote_home, const char *remote_ro); ++ ++void destroy_suppoter(remote_supporter *supporter); ++ ++int scan_remote_dir(remote_supporter *supporter); ++ ++int load_item(remote_supporter *supporter); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif +diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c b/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c +new file mode 100644 +index 00000000..7df7a221 +--- /dev/null ++++ b/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c +@@ -0,0 +1,347 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: wangrunze ++ * Create: 2023-01-12 ++ * Description: provide remote symlink maintain functions ++ ******************************************************************************/ ++#define _GNU_SOURCE ++#include "ro_symlink_maintain.h" ++ ++#include ++#include ++#include ++#include ++#include "map.h" ++#include "path.h" ++#include "linked_list.h" ++#include "layer_store.h" ++#include "layer.h" ++#include "isula_libutils/log.h" ++#include "image_store.h" ++#include "remote_support.h" ++#include "utils.h" ++#include "utils_file.h" ++#include "stdlib.h" ++ ++#define REMOTE_RO_LAYER_DIR "RO" ++ ++// overlay-layers and overlay-layers/RO ++static char *image_home; ++ ++static char *layer_ro_dir; ++static char *layer_home; ++ ++// overlay and overlay/RO ++static char *overlay_ro_dir; ++static char *overlay_home; ++ ++struct supporters { ++ remote_supporter *image_supporter; ++ remote_supporter *layer_supporter; ++ remote_supporter *overlay_supporter; ++}; ++ ++static struct supporters supporters; ++ ++int remote_image_init(const char *root_dir) ++{ ++ if (root_dir == NULL) { ++ goto out; ++ } ++ ++ image_home = util_strdup_s(root_dir); ++ if (image_home == NULL) { ++ ERROR("Failed create path for remote image home"); ++ goto out; ++ } ++ return 0; ++ ++out: ++ remote_maintain_cleanup(); ++ return -1; ++} ++ ++int remote_layer_init(const char *root_dir) ++{ ++ if (root_dir == NULL) { ++ goto out; ++ } ++ ++ layer_home = util_strdup_s(root_dir); ++ layer_ro_dir = util_path_join(root_dir, REMOTE_RO_LAYER_DIR); ++ if (layer_ro_dir == NULL) { ++ ERROR("Failed join path when init remote layer maintainer"); ++ goto out; ++ } ++ if (!util_file_exists(layer_ro_dir) && util_mkdir_p(layer_ro_dir, 0700) != 0) { ++ ERROR("Failed to create RO dir under overlay"); ++ goto out; ++ } ++ ++ return 0; ++ ++out: ++ remote_maintain_cleanup(); ++ return -1; ++} ++ ++int remote_overlay_init(const char *driver_home) ++{ ++ if (driver_home == NULL) { ++ goto out; ++ } ++ ++ overlay_home = util_strdup_s(driver_home); ++ overlay_ro_dir = util_path_join(driver_home, REMOTE_RO_LAYER_DIR); ++ if (overlay_ro_dir == NULL) { ++ ERROR("Failed to join path when init remote maintainer"); ++ goto out; ++ } ++ // build RO dir if not exist ++ if (!util_file_exists(overlay_ro_dir) && util_mkdir_p(overlay_ro_dir, 0700) != 0) { ++ ERROR("Failed to create RO dir under overlay"); ++ goto out; ++ } ++ ++ return 0; ++ ++out: ++ remote_maintain_cleanup(); ++ return -1; ++} ++ ++void remote_maintain_cleanup(void) ++{ ++ free(image_home); ++ image_home = NULL; ++ ++ free(layer_home); ++ layer_home = NULL; ++ free(layer_ro_dir); ++ layer_ro_dir = NULL; ++ free(overlay_home); ++ ++ overlay_home = NULL; ++ free(overlay_ro_dir); ++ overlay_ro_dir = NULL; ++} ++ ++// to maintain the symbol links, add new symbol link and delete invalid symbol link ++// arg is const char *driver_home ++// scanning driver->home/RO/ directory, build symlink in driver->home ++static void *remote_refresh_ro_symbol_link(void *arg) ++{ ++ struct supporters *supporters = (struct supporters *)arg; ++ prctl(PR_SET_NAME, "RoLayerRefresh"); ++ ++ while (true) { ++ util_usleep_nointerupt(5 * 1000 * 1000); ++ DEBUG("remote refresh start\n"); ++ scan_remote_dir(supporters->overlay_supporter); ++ load_item(supporters->overlay_supporter); ++ scan_remote_dir(supporters->layer_supporter); ++ load_item(supporters->layer_supporter); ++ scan_remote_dir(supporters->image_supporter); ++ load_item(supporters->image_supporter); ++ DEBUG("remote refresh end\n"); ++ } ++ return NULL; ++} ++ ++int start_refresh_thread(void) ++{ ++ int res = 0; ++ pthread_t a_thread; ++ ++ supporters.image_supporter = create_image_supporter(image_home, NULL); ++ if (supporters.image_supporter == NULL) { ++ goto free_out; ++ } ++ ++ supporters.layer_supporter = create_layer_supporter(layer_home, layer_ro_dir); ++ if (supporters.layer_supporter == NULL) { ++ goto free_out; ++ } ++ ++ supporters.overlay_supporter = create_overlay_supporter(overlay_home, overlay_ro_dir); ++ if (supporters.overlay_supporter == NULL) { ++ goto free_out; ++ } ++ ++ res = pthread_create(&a_thread, NULL, remote_refresh_ro_symbol_link, (void *)&supporters); ++ if (res != 0) { ++ CRIT("Thread creation failed"); ++ return -1; ++ } ++ ++ if (pthread_detach(a_thread) != 0) { ++ SYSERROR("Failed to detach 0x%lx", a_thread); ++ return -1; ++ } ++ ++ return 0; ++ ++free_out: ++ destroy_suppoter(supporters.image_supporter); ++ destroy_suppoter(supporters.layer_supporter); ++ destroy_suppoter(supporters.overlay_supporter); ++ ++ return -1; ++} ++ ++static int do_build_ro_dir(const char *home, const char *id) ++{ ++ char *ro_symlink = NULL; ++ char *ro_layer_dir = NULL; ++ int nret = 0; ++ // bool ret = true; ++ int ret = 0; ++ ++ nret = asprintf(&ro_symlink, "%s/%s", home, id); ++ if (nret < 0 || nret > PATH_MAX) { ++ SYSERROR("Failed create ro layer dir sym link path"); ++ return -1; ++ } ++ ++ nret = asprintf(&ro_layer_dir, "%s/%s/%s", home, REMOTE_RO_LAYER_DIR, id); ++ if (nret < 0 || nret > PATH_MAX) { ++ SYSERROR("Failed to create ro layer dir path"); ++ return -1; ++ } ++ ++ if (util_mkdir_p(ro_layer_dir, IMAGE_STORE_PATH_MODE) != 0) { ++ ret = -1; ++ ERROR("Failed to create layer direcotry %s", ro_layer_dir); ++ goto out; ++ } ++ ++ if (symlink(ro_layer_dir, ro_symlink) != 0) { ++ ret = -1; ++ SYSERROR("Failed to create symlink to layer dir %s", ro_layer_dir); ++ goto err_out; ++ } ++ ++ goto out; ++ ++err_out: ++ if (util_recursive_rmdir(ro_layer_dir, 0)) { ++ ERROR("Failed to delete layer path: %s", ro_layer_dir); ++ } ++ ++out: ++ free(ro_layer_dir); ++ free(ro_symlink); ++ return ret; ++} ++ ++int remote_overlay_build_ro_dir(const char *id) ++{ ++ return do_build_ro_dir(overlay_home, id); ++} ++ ++int remote_layer_build_ro_dir(const char *id) ++{ ++ return do_build_ro_dir(layer_home, id); ++} ++ ++int do_remove_ro_dir(const char *home, const char *id) ++{ ++ char *ro_layer_dir = NULL; ++ char *ro_symlink = NULL; ++ char clean_path[PATH_MAX] = { 0 }; ++ int ret = 0; ++ int nret = 0; ++ ++ if (id == NULL) { ++ return 0; ++ } ++ ++ nret = asprintf(&ro_symlink, "%s/%s", home, id); ++ if (nret < 0 || nret > PATH_MAX) { ++ SYSERROR("Create layer sym link path failed"); ++ return -1; ++ } ++ ++ if (util_clean_path(ro_symlink, clean_path, sizeof(clean_path)) == NULL) { ++ ERROR("Failed to clean path: %s", ro_symlink); ++ ret = -1; ++ goto out; ++ } ++ ++ if (util_path_remove(clean_path) != 0) { ++ SYSERROR("Failed to remove link path %s", clean_path); ++ } ++ ++ nret = asprintf(&ro_layer_dir, "%s/%s/%s", home, REMOTE_RO_LAYER_DIR, id); ++ if (nret < 0 || nret > PATH_MAX) { ++ SYSERROR("Create layer json path failed"); ++ ret = -1; ++ goto out; ++ } ++ ++ ret = util_recursive_rmdir(ro_layer_dir, 0); ++ ++out: ++ free(ro_layer_dir); ++ free(ro_symlink); ++ return ret; ++} ++ ++int remote_layer_remove_ro_dir(const char *id) ++{ ++ return do_remove_ro_dir(layer_home, id); ++} ++ ++int remote_overlay_remove_ro_dir(const char *id) ++{ ++ return do_remove_ro_dir(overlay_home, id); ++} ++ ++static char **map_diff(map_t *map_a, map_t *map_b) ++{ ++ char **array = NULL; ++ map_itor *itor = map_itor_new(map_a); ++ bool *found = NULL; ++ ++ // iter new_map, every item not in old, append them to new_layers ++ for (; map_itor_valid(itor); map_itor_next(itor)) { ++ char *id = map_itor_key(itor); ++ found = map_search(map_b, id); ++ if (found == NULL) { ++ util_array_append(&array, util_strdup_s(id)); ++ } ++ } ++ ++ map_itor_free(itor); ++ ++ return array; ++} ++ ++char **deleted_layers(map_t *old, map_t *new) ++{ ++ return map_diff(old, new); ++} ++ ++char **added_layers(map_t *old, map_t *new) ++{ ++ return map_diff(new, old); ++} ++ ++int empty_map(map_t *mp) ++{ ++ if (mp == NULL) { ++ return -1; ++ } ++ ++ map_clear(mp); ++ mp->store->root = mp->store->nil; ++ return 0; ++} +diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.h b/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.h +new file mode 100644 +index 00000000..25712d40 +--- /dev/null ++++ b/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.h +@@ -0,0 +1,52 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: wangrunze ++ * Create: 2023-01-12 ++ * Description: provide remote symlink maintain functions ++ ******************************************************************************/ ++#ifndef DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_RO_SYMLINK_MAINTAIN_H ++#define DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_RO_SYMLINK_MAINTAIN_H ++ ++#include "map.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++int remote_image_init(const char *root_dir); ++ ++int remote_layer_init(const char *root_dir); ++ ++int remote_overlay_init(const char *driver_home); ++ ++void remote_maintain_cleanup(void); ++ ++int start_refresh_thread(void); ++ ++int remote_layer_build_ro_dir(const char *id); ++ ++int remote_overlay_build_ro_dir(const char *id); ++ ++int remote_layer_remove_ro_dir(const char *id); ++ ++int remote_overlay_remove_ro_dir(const char *id); ++ ++char **deleted_layers(map_t *old, map_t *new); ++ ++char **added_layers(map_t *old, map_t *new); ++ ++int empty_map(map_t *mp); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif +diff --git a/src/daemon/modules/image/oci/storage/storage.c b/src/daemon/modules/image/oci/storage/storage.c +index 2f4bdf5f..31812a22 100644 +--- a/src/daemon/modules/image/oci/storage/storage.c ++++ b/src/daemon/modules/image/oci/storage/storage.c +@@ -42,6 +42,9 @@ + #include "utils_string.h" + #include "utils_verify.h" + #include "sha256.h" ++#ifdef ENABLE_REMOTE_LAYER_STORE ++#include "ro_symlink_maintain.h" ++#endif + + static pthread_rwlock_t g_storage_rwlock; + static char *g_storage_run_root; +@@ -1870,6 +1873,12 @@ int storage_module_init(struct storage_module_init_options *opts) + goto out; + } + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ if (opts->enable_remote_layer && start_refresh_thread() != 0) { ++ ERROR("Failed to start remote refresh thread"); ++ } ++#endif ++ + if (restore_images_size() != 0) { + ERROR("Failed to recal image size"); + ret = -1; +@@ -1906,4 +1915,4 @@ out: + char *storage_rootfs_get_dir() + { + return rootfs_store_get_data_dir(); +-} +\ No newline at end of file ++} +diff --git a/src/daemon/modules/image/oci/storage/storage.h b/src/daemon/modules/image/oci/storage/storage.h +index 3ec47959..7404ee54 100644 +--- a/src/daemon/modules/image/oci/storage/storage.h ++++ b/src/daemon/modules/image/oci/storage/storage.h +@@ -70,6 +70,9 @@ struct storage_module_init_options { + char **driver_opts; + size_t driver_opts_len; + bool integration_check; ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ bool enable_remote_layer; ++#endif + }; + + struct storage_img_create_options { +-- +2.25.1 + diff --git a/0033-add-ci-for-remote-ro.patch b/0033-add-ci-for-remote-ro.patch new file mode 100644 index 0000000..88c9279 --- /dev/null +++ b/0033-add-ci-for-remote-ro.patch @@ -0,0 +1,119 @@ +From 6311ccd6e367f965da4dc1b4c9efb4bf43275f64 Mon Sep 17 00:00:00 2001 +From: Neil +Date: Sun, 5 Mar 2023 12:23:29 +0000 +Subject: [PATCH 33/53] add ci for remote ro + +Signed-off-by: Neil +--- + CI/make-and-install.sh | 4 +- + .../container_cases/test_data/daemon.json | 1 + + CI/test_cases/image_cases/ro_separate.sh | 69 +++++++++++++++++++ + 3 files changed, 72 insertions(+), 2 deletions(-) + create mode 100644 CI/test_cases/image_cases/ro_separate.sh + +diff --git a/CI/make-and-install.sh b/CI/make-and-install.sh +index 3dbff480..fa9c2250 100755 +--- a/CI/make-and-install.sh ++++ b/CI/make-and-install.sh +@@ -103,9 +103,9 @@ rm -rf build + mkdir build + cd build + if [[ ${enable_gcov} -ne 0 ]]; then +- cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_UT=ON -DENABLE_METRICS=ON .. ++ cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_UT=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON .. + else +- cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DENABLE_EMBEDDED=ON -DENABLE_METRICS=ON .. ++ cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DENABLE_EMBEDDED=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON .. + fi + make -j $(nproc) + make install +diff --git a/CI/test_cases/container_cases/test_data/daemon.json b/CI/test_cases/container_cases/test_data/daemon.json +index aa88c9da..2664c6b2 100644 +--- a/CI/test_cases/container_cases/test_data/daemon.json ++++ b/CI/test_cases/container_cases/test_data/daemon.json +@@ -19,6 +19,7 @@ + "hook-spec": "/etc/default/isulad/hooks/default.json", + "start-timeout": "2m", + "storage-driver": "overlay2", ++ "storage-enable-remote-layer": false, + "storage-opts": [ + "overlay2.override_kernel_check=true" + ], +diff --git a/CI/test_cases/image_cases/ro_separate.sh b/CI/test_cases/image_cases/ro_separate.sh +new file mode 100644 +index 00000000..47e04abb +--- /dev/null ++++ b/CI/test_cases/image_cases/ro_separate.sh +@@ -0,0 +1,69 @@ ++#!/bin/bash ++# ++# attributes: isulad basic image ++# concurrent: NA ++# spend time: 22 ++ ++####################################################################### ++##- Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. ++# - iSulad licensed under the Mulan PSL v2. ++# - You can use this software according to the terms and conditions of the Mulan PSL v2. ++# - You may obtain a copy of Mulan PSL v2 at: ++# - http://license.coscl.org.cn/MulanPSL2 ++# - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++# - IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++# - PURPOSE. ++# - See the Mulan PSL v2 for more details. ++##- @Description:CI ++##- @Author: wangrunze ++##- @Create: 2023-03-03 ++####################################################################### ++ ++declare -r curr_path=$(dirname $(readlink -f "$0")) ++source ../helpers.sh ++single_image="${curr_path}/busybox.tar" ++ ++function test_separate_ro() ++{ ++ local ret=0 ++ local test="isula separate ro test => (${FUNCNAME[@]})" ++ ++ msg_info "${test} starting..." ++ ++ sed -i 's/"storage-enable-remote-layer": false/"storage-enable-remote-layer": true/' /etc/isulad/daemon.json ++ start_isulad_with_valgrind ++ wait_isulad_running ++ ++ isula rmi busybox ++ ++ isula pull busybox ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - pull image failed" && ((ret++)) ++ ++ isula run -tid --name test_separate busybox /bin/sh ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - run container failed" && ((ret++)) ++ ++ isula stop test_separate ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stop container failed" && ((ret++)) ++ ++ isula rmi busybox ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - remove image failed" && ((ret++)) ++ ++ isula load -i $single_image ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - load image failed: ${rootfs_tar}" && ((ret++)) ++ ++ check_valgrind_log ++ [[ $? -ne 0 ]] && msg_err "separate ro test - memory leak, please check...." && ((ret++)) ++ ++ sed -i 's/"storage-enable-remote-layer": true/"storage-enable-remote-layer": false/' /etc/isulad/daemon.json ++ start_isulad_with_valgrind ++ wait_isulad_running ++ ++ msg_info "${test} finished with return ${ret}..." ++ return ${ret} ++} ++ ++declare -i ans=0 ++ ++test_separate_ro || ((ans++)) ++ ++show_result ${ans} "${curr_path}/${0}" +-- +2.25.1 + diff --git a/0034-change-sleep-to-usleep-to-avoid-lossing-of-accuracy.patch b/0034-change-sleep-to-usleep-to-avoid-lossing-of-accuracy.patch new file mode 100644 index 0000000..8092e85 --- /dev/null +++ b/0034-change-sleep-to-usleep-to-avoid-lossing-of-accuracy.patch @@ -0,0 +1,26 @@ +From 9663e62598570d16c0e8a70be4341ff72663b8df Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Mon, 6 Mar 2023 09:51:17 +0800 +Subject: [PATCH 34/53] change sleep() to usleep() to avoid lossing of accuracy + +Signed-off-by: zhongtao +--- + src/utils/cutils/utils.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/utils/cutils/utils.c b/src/utils/cutils/utils.c +index 2c4c01e4..983d81d8 100644 +--- a/src/utils/cutils/utils.c ++++ b/src/utils/cutils/utils.c +@@ -345,7 +345,7 @@ int util_waitpid_with_timeout(pid_t pid, const int64_t timeout, handle_timeout_c + return -1; + } + // sleep some time instead to avoid cpu full running and then retry. +- sleep(0.1); ++ usleep(100); + } + return 0; + } +-- +2.25.1 + diff --git a/0035-fix-compile-error-when-not-enable-remote-ro.patch b/0035-fix-compile-error-when-not-enable-remote-ro.patch new file mode 100644 index 0000000..091c939 --- /dev/null +++ b/0035-fix-compile-error-when-not-enable-remote-ro.patch @@ -0,0 +1,62 @@ +From 6a0b11ae6584ee2eefff9bd20c96bc60582ccb6b Mon Sep 17 00:00:00 2001 +From: "Neil.wrz" +Date: Sun, 5 Mar 2023 18:55:40 -0800 +Subject: [PATCH 35/53] fix compile error when not enable remote ro + +Signed-off-by: Neil.wrz +--- + .../modules/image/oci/storage/image_store/CMakeLists.txt | 3 +++ + .../modules/image/oci/storage/layer_store/CMakeLists.txt | 4 ++++ + .../storage/layer_store/graphdriver/overlay2/CMakeLists.txt | 3 +++ + 3 files changed, 10 insertions(+) + +diff --git a/src/daemon/modules/image/oci/storage/image_store/CMakeLists.txt b/src/daemon/modules/image/oci/storage/image_store/CMakeLists.txt +index ecf21caa..7d4fb77c 100644 +--- a/src/daemon/modules/image/oci/storage/image_store/CMakeLists.txt ++++ b/src/daemon/modules/image/oci/storage/image_store/CMakeLists.txt +@@ -1,5 +1,8 @@ + # get current directory sources files + aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_image_store_srcs) ++IF (NOT ENABLE_REMOTE_LAYER_STORE) ++list(REMOVE_ITEM local_image_store_srcs "${CMAKE_CURRENT_SOURCE_DIR}/image_remote_impl.c") ++ENDIF() + + set(IMAGE_STORE_SRCS + ${local_image_store_srcs} +diff --git a/src/daemon/modules/image/oci/storage/layer_store/CMakeLists.txt b/src/daemon/modules/image/oci/storage/layer_store/CMakeLists.txt +index f964f709..e04b4ad7 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/CMakeLists.txt ++++ b/src/daemon/modules/image/oci/storage/layer_store/CMakeLists.txt +@@ -1,5 +1,8 @@ + # get current directory sources files + aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_layer_store_srcs) ++IF (NOT ENABLE_REMOTE_LAYER_STORE) ++list(REMOVE_ITEM local_layer_store_srcs "${CMAKE_CURRENT_SOURCE_DIR}/layer_remote_impl.c") ++ENDIF() + add_subdirectory(graphdriver) + + set(LAYER_STORE_SRCS +@@ -7,6 +10,7 @@ set(LAYER_STORE_SRCS + ${GRAPHDRIVER_SRCS} + PARENT_SCOPE + ) ++ + set(LAYER_STORE_INCS + ${CMAKE_CURRENT_SOURCE_DIR} + ${GRAPHDRIVER_INCS} +diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/CMakeLists.txt b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/CMakeLists.txt +index ceed16b7..dd4e82aa 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/CMakeLists.txt ++++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/CMakeLists.txt +@@ -1,5 +1,8 @@ + # get current directory sources files + aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_overlay2_srcs) ++IF (NOT ENABLE_REMOTE_LAYER_STORE) ++list(REMOVE_ITEM local_overlay2_srcs "${CMAKE_CURRENT_SOURCE_DIR}/overlay_remote_impl.c") ++ENDIF() + + set(OVERLAY2_SRCS + ${local_overlay2_srcs} +-- +2.25.1 + diff --git a/0036-adapt-to-repo-of-openeuler-url-changed.patch b/0036-adapt-to-repo-of-openeuler-url-changed.patch new file mode 100644 index 0000000..dd476a9 --- /dev/null +++ b/0036-adapt-to-repo-of-openeuler-url-changed.patch @@ -0,0 +1,33 @@ +From 42a46f64e533ea0a89f95bef80a421fe06a5bfa2 Mon Sep 17 00:00:00 2001 +From: haozi007 +Date: Mon, 6 Mar 2023 14:38:58 +0800 +Subject: [PATCH 36/53] adapt to repo of openeuler url changed + +Signed-off-by: haozi007 +--- + CI/pr-gateway.sh | 8 ++++++++ + 1 file changed, 8 insertions(+) + +diff --git a/CI/pr-gateway.sh b/CI/pr-gateway.sh +index c715fc39..7c696de3 100755 +--- a/CI/pr-gateway.sh ++++ b/CI/pr-gateway.sh +@@ -18,7 +18,15 @@ if [ $# -eq 1 ]; then + tbranch=$1 + fi + ++sed -i "s#http://repo.openeuler.org#https://repo.huaweicloud.com/openeuler#g" /etc/yum.repos.d/openEuler.repo ++ ++dnf update -y ++ + dnf install -y gtest-devel gmock-devel diffutils cmake gcc-c++ yajl-devel patch make libtool libevent-devel libevhtp-devel grpc grpc-plugins grpc-devel protobuf-devel libcurl libcurl-devel sqlite-devel libarchive-devel device-mapper-devel http-parser-devel libseccomp-devel libcap-devel libselinux-devel libwebsockets libwebsockets-devel systemd-devel git chrpath ++if [ $? -ne 0 ]; then ++ echo "install dependences failed" ++ exit 1 ++fi + + # dnf install -y cargo rust rust-packaging + +-- +2.25.1 + diff --git a/0037-change-goto-branch.patch b/0037-change-goto-branch.patch new file mode 100644 index 0000000..a2352f4 --- /dev/null +++ b/0037-change-goto-branch.patch @@ -0,0 +1,27 @@ +From 66532035178723175ad9522201440372316dc914 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Mon, 6 Mar 2023 17:54:55 +0800 +Subject: [PATCH 37/53] change goto branch + +Signed-off-by: zhongtao +--- + src/daemon/executor/container_cb/execution_stream.c | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/src/daemon/executor/container_cb/execution_stream.c b/src/daemon/executor/container_cb/execution_stream.c +index 1a7353b5..7e928cf7 100644 +--- a/src/daemon/executor/container_cb/execution_stream.c ++++ b/src/daemon/executor/container_cb/execution_stream.c +@@ -161,7 +161,8 @@ static int container_exec_cb(const container_exec_request *request, container_ex + + if (exec_container(cont, request, *response, stdinfd, stdout_handler, stderr_handler) != 0) { + ret = -1; +- goto pack_err_response; ++ // pack err response in exec_container, there is no need to pack here. ++ goto out; + } + + goto out; +-- +2.25.1 + diff --git a/0038-CI-not-enable-remote-ro-for-ut.patch b/0038-CI-not-enable-remote-ro-for-ut.patch new file mode 100644 index 0000000..cc41858 --- /dev/null +++ b/0038-CI-not-enable-remote-ro-for-ut.patch @@ -0,0 +1,41 @@ +From c414e2226542660579e14b9401fc28bc91b709c7 Mon Sep 17 00:00:00 2001 +From: "Neil.wrz" +Date: Mon, 6 Mar 2023 18:59:43 -0800 +Subject: [PATCH 38/53] CI not enable remote ro for ut + +Signed-off-by: Neil.wrz +--- + CI/make-and-install.sh | 2 +- + CI/test_cases/image_cases/ro_separate.sh | 3 +++ + 2 files changed, 4 insertions(+), 1 deletion(-) + +diff --git a/CI/make-and-install.sh b/CI/make-and-install.sh +index fa9c2250..faeaf005 100755 +--- a/CI/make-and-install.sh ++++ b/CI/make-and-install.sh +@@ -103,7 +103,7 @@ rm -rf build + mkdir build + cd build + if [[ ${enable_gcov} -ne 0 ]]; then +- cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_UT=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON .. ++ cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_UT=ON -DENABLE_METRICS=ON .. + else + cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DENABLE_EMBEDDED=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON .. + fi +diff --git a/CI/test_cases/image_cases/ro_separate.sh b/CI/test_cases/image_cases/ro_separate.sh +index 47e04abb..df45e120 100644 +--- a/CI/test_cases/image_cases/ro_separate.sh ++++ b/CI/test_cases/image_cases/ro_separate.sh +@@ -45,6 +45,9 @@ function test_separate_ro() + isula stop test_separate + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stop container failed" && ((ret++)) + ++ isula rm test_separate ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - remove container failed" && ((ret++)) ++ + isula rmi busybox + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - remove image failed" && ((ret++)) + +-- +2.25.1 + diff --git a/0039-use-auto-free-to-proc_t.patch b/0039-use-auto-free-to-proc_t.patch new file mode 100644 index 0000000..25de87e --- /dev/null +++ b/0039-use-auto-free-to-proc_t.patch @@ -0,0 +1,43 @@ +From 0a68a53ccb6582384dad478d4197ec9386306027 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Mon, 6 Mar 2023 10:56:16 +0800 +Subject: [PATCH 39/53] use auto free to proc_t + +Signed-off-by: zhongtao +--- + src/daemon/modules/runtime/isula/isula_rt_ops.c | 12 ++---------- + 1 file changed, 2 insertions(+), 10 deletions(-) + +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index 536e36b2..5ec0e639 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -951,8 +951,8 @@ int rt_isula_start(const char *id, const char *runtime, const rt_start_params_t + pid_t shim_pid = -1; + int ret = -1; + int splice_ret = 0; +- proc_t *proc = NULL; +- proc_t *p_proc = NULL; ++ __isula_auto_free proc_t *proc = NULL; ++ __isula_auto_free proc_t *p_proc = NULL; + + if (id == NULL || runtime == NULL || params == NULL || pid_info == NULL) { + ERROR("nullptr arguments not allowed"); +@@ -1009,14 +1009,6 @@ out: + show_shim_runtime_errlog(workdir); + shim_kill_force(workdir); + } +- +- if (proc != NULL) { +- free(proc); +- } +- if (p_proc != NULL) { +- free(p_proc); +- } +- + return ret; + } + +-- +2.25.1 + diff --git a/0040-modifying-cpurt-file-permissions.patch b/0040-modifying-cpurt-file-permissions.patch new file mode 100644 index 0000000..8eb5455 --- /dev/null +++ b/0040-modifying-cpurt-file-permissions.patch @@ -0,0 +1,72 @@ +From 2d9b9d88f3150027609fe984930af1dcf06dfd00 Mon Sep 17 00:00:00 2001 +From: songbuhuang <544824346@qq.com> +Date: Thu, 9 Mar 2023 18:32:19 +0800 +Subject: [PATCH 40/53] modifying cpurt file permissions + +Signed-off-by: songbuhuang <544824346@qq.com> +--- + src/common/constants.h | 4 ++++ + src/daemon/common/sysinfo.c | 3 +-- + src/daemon/executor/container_cb/execution.c | 3 ++- + 3 files changed, 7 insertions(+), 3 deletions(-) + +diff --git a/src/common/constants.h b/src/common/constants.h +index b43d8c80..06015f1e 100644 +--- a/src/common/constants.h ++++ b/src/common/constants.h +@@ -22,6 +22,10 @@ extern "C" { + + /* mode of file and directory */ + ++#define DEFAULT_CGROUP_FILE_MODE 0644 ++ ++#define DEFAULT_CGROUP_DIR_MODE 0755 ++ + #define DEFAULT_SECURE_FILE_MODE 0640 + + #define DEFAULT_SECURE_DIRECTORY_MODE 0750 +diff --git a/src/daemon/common/sysinfo.c b/src/daemon/common/sysinfo.c +index 7559d653..baf53510 100644 +--- a/src/daemon/common/sysinfo.c ++++ b/src/daemon/common/sysinfo.c +@@ -27,6 +27,7 @@ + #include + #include + ++#include "constants.h" + #include "err_msg.h" + #include "utils.h" + #include "utils_array.h" +@@ -73,8 +74,6 @@ + + #define CGROUP_MOUNTPOINT "/sys/fs/cgroup" + #define CGROUP_ISULAD_PATH CGROUP_MOUNTPOINT"/isulad" +-#define DEFAULT_CGROUP_DIR_MODE 0755 +-#define DEFAULT_CGROUP_FILE_MODE 0644 + #define CGROUP2_CONTROLLERS_PATH CGROUP_MOUNTPOINT"/cgroup.controllers" + #define CGROUP2_SUBTREE_CONTROLLER_PATH CGROUP_MOUNTPOINT"/cgroup.subtree_control" + #define CGROUP2_CPUSET_CPUS_EFFECTIVE_PATH CGROUP_MOUNTPOINT"/cpuset.cpus.effective" +diff --git a/src/daemon/executor/container_cb/execution.c b/src/daemon/executor/container_cb/execution.c +index 198052d3..c2a0bdb9 100644 +--- a/src/daemon/executor/container_cb/execution.c ++++ b/src/daemon/executor/container_cb/execution.c +@@ -46,6 +46,7 @@ + #include "isulad_config.h" + #include "sysinfo.h" + #include "container_api.h" ++#include "constants.h" + #include "specs_api.h" + #include "execution_extend.h" + #include "execution_information.h" +@@ -324,7 +325,7 @@ static int maybe_create_cpu_realtime_file(int64_t value, const char *file, const + return 0; + } + +- ret = util_mkdir_p(path, CONFIG_DIRECTORY_MODE); ++ ret = util_mkdir_p(path, DEFAULT_CGROUP_DIR_MODE); + if (ret != 0) { + ERROR("Failed to mkdir: %s", path); + return -1; +-- +2.25.1 + diff --git a/0041-use-CURLOPT_XFERINFOFUNCTION-instead-of-deprecated-C.patch b/0041-use-CURLOPT_XFERINFOFUNCTION-instead-of-deprecated-C.patch new file mode 100644 index 0000000..af12eec --- /dev/null +++ b/0041-use-CURLOPT_XFERINFOFUNCTION-instead-of-deprecated-C.patch @@ -0,0 +1,131 @@ +From d0b0baa3f2624b6de0ca92c051c154f0cff43f1a Mon Sep 17 00:00:00 2001 +From: zhangxiaoyu +Date: Tue, 14 Mar 2023 10:33:38 +0800 +Subject: [PATCH 41/53] use CURLOPT_XFERINFOFUNCTION instead of deprecated + CURLOPT_PROGRESSFUNCTION since curl 7.32.0 + +Signed-off-by: zhangxiaoyu +--- + .../modules/image/oci/registry/http_request.c | 12 +++++++++++ + src/utils/http/http.c | 21 ++++++++++++++----- + src/utils/http/http.h | 7 +++++++ + test/image/oci/registry/registry_ut.cc | 8 +++++++ + 4 files changed, 43 insertions(+), 5 deletions(-) + +diff --git a/src/daemon/modules/image/oci/registry/http_request.c b/src/daemon/modules/image/oci/registry/http_request.c +index f29c2017..ce8b7667 100644 +--- a/src/daemon/modules/image/oci/registry/http_request.c ++++ b/src/daemon/modules/image/oci/registry/http_request.c +@@ -691,6 +691,16 @@ static int progress(void *p, double dltotal, double dlnow, double ultotal, doubl + return 0; + } + ++static int xfer(void *p, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) ++{ ++ bool *cancel = p; ++ if (*cancel) { ++ // return nonzero code means abort transition ++ return -1; ++ } ++ return 0; ++} ++ + int http_request_file(pull_descriptor *desc, const char *url, const char **custom_headers, char *file, + resp_data_type type, CURLcode *errcode) + { +@@ -721,6 +731,8 @@ int http_request_file(pull_descriptor *desc, const char *url, const char **custo + options->show_progress = 1; + options->progressinfo = &desc->cancel; + options->progress_info_op = progress; ++ options->xferinfo = &desc->cancel; ++ options->xferinfo_op = xfer; + options->timeout = true; + + ret = setup_common_options(desc, options, url, custom_headers); +diff --git a/src/utils/http/http.c b/src/utils/http/http.c +index bf163d86..986f1f0d 100644 +--- a/src/utils/http/http.c ++++ b/src/utils/http/http.c +@@ -219,12 +219,23 @@ static void http_custom_general_options(CURL *curl_handle, const struct http_get + /* disable progress meter, set to 0L to enable and disable debug output */ + if (options->show_progress == 0) { + curl_easy_setopt(curl_handle, CURLOPT_NOPROGRESS, 1L); +- } else if (options->show_progress && options->progressinfo && options->progress_info_op) { +- curl_easy_setopt(curl_handle, CURLOPT_PROGRESSFUNCTION, options->progress_info_op); +- /* pass the struct pointer into the progress function */ +- curl_easy_setopt(curl_handle, CURLOPT_PROGRESSDATA, options->progressinfo); +- curl_easy_setopt(curl_handle, CURLOPT_NOPROGRESS, 0L); + } else { ++ /* libcurl support option CURLOPT_XFERINFOFUNCTION when version >= 7.32.0 ++ * #define CURL_VERSION_BITS(x,y,z) ((x)<<16|(y)<<8|(z)) ++ * CURL_VERSION_BITS(7,32,0) = 0x072000 */ ++#if (LIBCURL_VERSION_NUM >= 0x072000) ++ if (options->xferinfo && options->xferinfo_op) { ++ curl_easy_setopt(curl_handle, CURLOPT_XFERINFOFUNCTION, options->xferinfo_op); ++ /* pass the struct pointer into the progress function */ ++ curl_easy_setopt(curl_handle, CURLOPT_XFERINFODATA, options->xferinfo); ++ } ++#else ++ if (options->progressinfo && options->progress_info_op) { ++ curl_easy_setopt(curl_handle, CURLOPT_PROGRESSFUNCTION, options->progress_info_op); ++ /* pass the struct pointer into the progress function */ ++ curl_easy_setopt(curl_handle, CURLOPT_PROGRESSDATA, options->progressinfo); ++ } ++#endif + curl_easy_setopt(curl_handle, CURLOPT_NOPROGRESS, 0L); + } + +diff --git a/src/utils/http/http.h b/src/utils/http/http.h +index 343d92c3..cdd6d64f 100644 +--- a/src/utils/http/http.h ++++ b/src/utils/http/http.h +@@ -17,6 +17,7 @@ + + #include + #include ++#include + + #ifdef __cplusplus + extern "C" { +@@ -25,6 +26,9 @@ extern "C" { + typedef int(*progress_info_func)(void *p, + double dltotal, double dlnow, + double ultotal, double ulnow); ++typedef int(*xferinfo_func)(void *p, ++ curl_off_t dltotal, curl_off_t dlnow, ++ curl_off_t ultotal, curl_off_t ulnow); + + struct http_get_options { + unsigned with_head : 1, /* if set, means write output with response HEADER */ +@@ -77,6 +81,9 @@ struct http_get_options { + + void *progressinfo; + progress_info_func progress_info_op; ++ ++ void *xferinfo; ++ xferinfo_func xferinfo_op; + }; + + #define HTTP_RES_OK 0 +diff --git a/test/image/oci/registry/registry_ut.cc b/test/image/oci/registry/registry_ut.cc +index 8d9ea92b..f4f8a763 100644 +--- a/test/image/oci/registry/registry_ut.cc ++++ b/test/image/oci/registry/registry_ut.cc +@@ -221,6 +221,14 @@ int invokeHttpRequestV2(const char *url, struct http_get_options *options, long + if (options->progress_info_op(options->progressinfo, 0, 0, 0, 0) != 0) { + return -1; + } ++ ++ cancel = (bool *)options->xferinfo; ++ while (!(*cancel)) { ++ sleep(1); // schedule out to let cancel variable set to be true ++ } ++ if (options->xferinfo_op(options->xferinfo, 0, 0, 0, 0) != 0) { ++ return -1; ++ } + } + } else if (util_has_prefix(url, "http://hub-mirror.c.163.com/v2/library/busybox/blobs/sha256:91f30d77")) { + if (retry) { +-- +2.25.1 + diff --git a/0042-bugfix-remote-ro-try-add-or-remove-image-layer-twice.patch b/0042-bugfix-remote-ro-try-add-or-remove-image-layer-twice.patch new file mode 100644 index 0000000..2741ecf --- /dev/null +++ b/0042-bugfix-remote-ro-try-add-or-remove-image-layer-twice.patch @@ -0,0 +1,99 @@ +From a08e511eb6ee1955bb62e774190dfe7a7599fbdd Mon Sep 17 00:00:00 2001 +From: "Neil.wrz" +Date: Tue, 7 Mar 2023 23:59:56 -0800 +Subject: [PATCH 42/53] bugfix remote ro try add or remove image/layer twice + +Signed-off-by: Neil.wrz +--- + .../image/oci/storage/image_store/image_store.c | 14 ++++++++++++++ + .../oci/storage/layer_store/layer_remote_impl.c | 2 +- + .../image/oci/storage/layer_store/layer_store.c | 11 +++++++++++ + 3 files changed, 26 insertions(+), 1 deletion(-) + +diff --git a/src/daemon/modules/image/oci/storage/image_store/image_store.c b/src/daemon/modules/image/oci/storage/image_store/image_store.c +index caff3705..84187ded 100644 +--- a/src/daemon/modules/image/oci/storage/image_store/image_store.c ++++ b/src/daemon/modules/image/oci/storage/image_store/image_store.c +@@ -3668,6 +3668,11 @@ int append_image_by_directory_with_lock(const char *id) + return -1; + } + ++ if (map_search(g_image_store->byid, (void *)id) != NULL ) { ++ DEBUG("remote image already exist, not added: %s", id); ++ goto out; ++ } ++ + nret = snprintf(image_path, sizeof(image_path), "%s/%s", g_image_store->dir, id); + if (nret < 0 || (size_t)nret >= sizeof(image_path)) { + ERROR("Failed to get image path"); +@@ -3675,6 +3680,8 @@ int append_image_by_directory_with_lock(const char *id) + } + + ret = append_image_by_directory(image_path); ++ ++out: + image_store_unlock(); + + return ret; +@@ -3689,7 +3696,14 @@ int remove_image_from_memory_with_lock(const char *id) + return -1; + } + ++ if (map_search(g_image_store->byid, (void *)id) == NULL) { ++ DEBUG("remote image already remvoed, don't delete twice: %s", id); ++ goto out; ++ } ++ + ret = remove_image_from_memory(id); ++ ++out: + image_store_unlock(); + + return ret; +diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c b/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c +index d03fc20b..d676458c 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c ++++ b/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c +@@ -175,7 +175,7 @@ static int remote_support_add(void *data) + } + + if (add_one_remote_layer(data, array_added[i]) != 0) { +- ERROR("Failed to add remote overlay layer: %s", array_added[i]); ++ ERROR("Failed to add remote layer: %s", array_added[i]); + ret = -1; + } + } +diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c +index c00c3356..e88067bc 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c ++++ b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c +@@ -1852,6 +1852,11 @@ int load_one_layer(const char *id) + return -1; + } + ++ if (map_search(g_metadata.by_id, (void *)id) != NULL) { ++ DEBUG("remote layer already exist, not added: %s", id); ++ goto unlock_out; ++ } ++ + tl = load_one_layer_from_json(id); + if (tl == NULL) { + ret = -1; +@@ -2482,8 +2487,14 @@ int remove_memory_stores_with_lock(const char *id) + ERROR("Failed to lock layer store when handle: %s", id); + return -1; + } ++ if (map_search(g_metadata.by_id, (void *)id) == NULL) { ++ DEBUG("remote layer already removed, don't delete: %s", id); ++ goto unlock_out; ++ } + + ret = remove_memory_stores(id); ++ ++unlock_out: + layer_store_unlock(); + + return ret; +-- +2.25.1 + diff --git a/0043-bugfix-can-t-delete-layers-under-dir-overlay-layers.patch b/0043-bugfix-can-t-delete-layers-under-dir-overlay-layers.patch new file mode 100644 index 0000000..a4d9296 --- /dev/null +++ b/0043-bugfix-can-t-delete-layers-under-dir-overlay-layers.patch @@ -0,0 +1,34 @@ +From c8702b62fd6016a96794d74abcae2e74551a9c07 Mon Sep 17 00:00:00 2001 +From: "Neil.wrz" +Date: Tue, 14 Mar 2023 20:32:23 -0700 +Subject: [PATCH 43/53] bugfix can't delete layers under dir overlay-layers + +Signed-off-by: Neil.wrz +--- + .../modules/image/oci/storage/layer_store/layer_store.c | 8 ++++++-- + 1 file changed, 6 insertions(+), 2 deletions(-) + +diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c +index e88067bc..4edd0cad 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c ++++ b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c +@@ -1409,10 +1409,14 @@ static int do_delete_layer(const char *id) + } + + #ifdef ENABLE_REMOTE_LAYER_STORE +- if (l->slayer->writable) { ++ if (!g_enable_remote_layer) { + ret = layer_store_remove_layer(l->slayer->id); + } else { +- ret = remote_layer_remove_ro_dir(l->slayer->id); ++ if (l->slayer->writable) { ++ ret = layer_store_remove_layer(l->slayer->id); ++ } else { ++ ret = remote_layer_remove_ro_dir(l->slayer->id); ++ } + } + #else + ret = layer_store_remove_layer(l->slayer->id); +-- +2.25.1 + diff --git a/0044-doc-add-document-about-support-remote-ro-directory.patch b/0044-doc-add-document-about-support-remote-ro-directory.patch new file mode 100644 index 0000000..71af0e9 --- /dev/null +++ b/0044-doc-add-document-about-support-remote-ro-directory.patch @@ -0,0 +1,396 @@ +From 6911c68b96f3a0fc35b40e5c387da51cd84e5461 Mon Sep 17 00:00:00 2001 +From: "Neil.wrz" +Date: Mon, 13 Feb 2023 03:34:31 -0800 +Subject: [PATCH 44/53] doc: add document about support remote ro directory + +--- + docs/design/README.md | 4 +- + docs/design/README_zh.md | 6 +- + .../design/detailed/Image/remote_ro_design.md | 333 ++++++++++++++++++ + 3 files changed, 340 insertions(+), 3 deletions(-) + create mode 100644 docs/design/detailed/Image/remote_ro_design.md + +diff --git a/docs/design/README.md b/docs/design/README.md +index a9d357b4..cf29c0a1 100644 +--- a/docs/design/README.md ++++ b/docs/design/README.md +@@ -34,6 +34,8 @@ This section contains some design documents for users who want to learn more abo + + - You can see how the isula search are designed in [image_search_design](./detailed/Image/image_search_design_zh.md) + ++- You can see how the remote ro are designed in [remote_ro_design](./detailed/Image/remote_ro_design.md) ++ + ## Network + + - You can see how the cni operator modules are designed in [cni_operator_design](./detailed/Network/cni_operator_design.md). +@@ -54,4 +56,4 @@ This section contains some design documents for users who want to learn more abo + + ## Volume + +-- You can see how the local volume modules are designed in [local_volume_design](./detailed/Volume/local_volume_design.md). +\ No newline at end of file ++- You can see how the local volume modules are designed in [local_volume_design](./detailed/Volume/local_volume_design.md). +diff --git a/docs/design/README_zh.md b/docs/design/README_zh.md +index f51930e4..1f1c94b5 100644 +--- a/docs/design/README_zh.md ++++ b/docs/design/README_zh.md +@@ -32,7 +32,9 @@ + + - 查看 registry 模块的设计文档： [registry_degisn](./detailed/Image/registry_degisn_zh.md) 。 + +-- 查看 isula search 的设计文档：[image_search_design](./detailed/Image/image_search_design_zh.md) ++- 查看 isula search 的设计文档：[image_search_design](./detailed/Image/image_search_design_zh.md) 。 ++ ++- 查看 ro目录分离的设计文档： [remote_ro_design](./detailed/Image/remote_ro_design.md) 。 + + ## Network + +@@ -54,4 +56,4 @@ + + ## Volume + +-- 查看 local volume 模块的设计文档： [local_volume_design](./detailed/Volume/local_volume_design_zh.md). +\ No newline at end of file ++- 查看 local volume 模块的设计文档： [local_volume_design](./detailed/Volume/local_volume_design_zh.md). +diff --git a/docs/design/detailed/Image/remote_ro_design.md b/docs/design/detailed/Image/remote_ro_design.md +new file mode 100644 +index 00000000..fee33835 +--- /dev/null ++++ b/docs/design/detailed/Image/remote_ro_design.md +@@ -0,0 +1,333 @@ ++| Author | 王润泽 | ++| ------ | ---------------------- | ++| Date | 2023-2-13 | ++| Email | wangrunze13@huawei.com | ++ ++# 1. 方案目标 ++目标有两个： ++1. 把isulad当前的layer store里的RO层分离出来，把RW layer和RO layer分开到不同到目录存储。 ++2. isulad在运行时如果在相关目录里恢复了正确的镜像数据(image和layer数据), 可实现不重启isulad, 直接使用新恢复的镜像。如果移除当前没有容器正在使用的镜像数据，可实现不重启isulad，isulad更新当前管理的镜像列表去除该镜像。只考虑完全正确的新增和删除。 ++ ++ ++## 1.1 用法说明 ++通过源码编译打开编译选项来开启功能，cmake添加`cmake -DENABLE_REMOTE_LAYER_STORE=ON ..`, 然后`make -j`即可。启动iSulad之前，还需要在配置文件`/etc/isulad/daemon.json`里面添加`"storage-enable-remote-layer": true`来打开开关。 ++ ++# 2. 总体设计 ++ ++*Modules Dependencies* ++``` ++=> New Added Module: ++ ++ +===================================+ ++ | Remote Supporter Module | ++ +===================================+ ++ | | ++ | +-------------------------+ | ++ | | maintainer submod | | ++ | +-------------------------+ | ++ | | global data initer | | ++ | | symbol link maintainer | | ++ | | global data getter | | ++ | +-------------------------+ | ++ | | ++ | +-------------------------+ | ++ | | Supporter submod | | ++ | +-------------------------+ | ++ | | supporter interface | | ++ | | overlay supporter impl | | ++ | | layer supporter impl | | ++ | | image supporter impl | | ++ | | remote refresh thread | | ++ | +-------------------------+ | ++ +-----------------------------------+ ++ ++ ++=> Modified Modules: ++ ++ +===================================+ ++ | Storage Module | ++ +===================================+ ++ | | ++ | +-------------------------+ | ++ | | Image Store submod | | ++ | | Added Functions | | ++ | +-------------------------+ | ++ | | add image in memory | | ++ | | delete image in memory | | ++ | | get image top layer | | ++ | | valid image manifest | | ++ | +-------------------------+ | ++ | | ++ | +-------------------------+ | ++ | | Layer Store submod | | ++ | | Added Functions | | ++ | +-------------------------+ | ++ | | add layer in memory | | ++ | | delete layer in memory | | ++ | +-------------------------+ | ++ | | ++ | +-------------------------+ | ++ | | Driver Overlay submod | | ++ | | Added Functions | | ++ | +-------------------------+ | ++ | | - | | ++ | +-------------------------+ | ++ +-----------------------------------+ ++ ++ ++=> Modules Dependencies: ++ ++ +-------------------------+ +---------------+ ++ | Supporter submod | | Image Store | ++ +-------------------------+ +----▶| submod ---------+ ++ | supporter interface | | +---------------+ | ++ | overlay supporter impl -----+ | ++ | layer supporter impl -----+ | +++----------------+ | image supporter impl | | +----------------+ | ++| storage module |-------▶| remote refresh thread | +----▶| Layer Store | | init +-----------------------+ +++----------------+ +-------------------------+ | submod --------+--------▶| maintainer submod | ++ | +----------------+ | +-----------------------+ ++ | | ▲ ++ | | | ++ | +----------------+ | | ++ | | Layer Store | | | ++ | | submod --------+ | ++ | +----------------+ | ++ | | ++ +------------------------------------------------------------------------+ ++ get global data ++``` ++ ++ ++总体来说有两部分的功能： ++- iSulad原有的image storage适配分离的RO目录结构，*分离的RO目录*可用于远程挂载 ++- iSulad实例同步内存数据，镜像数据和layer数据*定期更新*，不通过`isula pull` 和 `isula rmi` 等命令，直接通过分离目录里面的数据来更新镜像数据。 ++ ++*分离RO目录* ++修改前后storage目录结构对比： ++ ++``` ++old: ++overlay-layer ++├── b703587 ++│ └── layer.json ++└── b64792c ++ └── layer.json ++ └── b64792.tar.gz ++ ++new: ++overlay-layer ++├── b64792c -> ../RO/b64792c ++├── b703587 ++│ └── layer.json ++└── RO ++ └── b64792c ++ └── layer.json ++ └── b64792.tar.gz ++``` ++ ++以overlay-layers目录为例，创建新layer时，如果是只读层，就把层数据放到RO目录下，在RO上层目录创建软连接指向真实数据。删除layer时需要额外删除软连接。 ++ ++ ++*定期更新* ++定期更新通过启动一个线程周期扫描`overlay`, `overlay-layers`, `overlay-image`这三个目录，通过比较当前时刻与上一时刻的目录差异，来获取镜像和层的删减情况，进而同步isulad的storage内存数据和维护软链接。 ++ ++``` +++---------------------+ +---------------------+ +---------------------+ +-----------------------+ ++| refresh thread loop | | overlay remote impl | | layer remote impl | | image remote impl | +++---------------------+ +---------------------+ +---------------------+ +-----------------------+ ++ | | | | ++ | refresh start | | | ++ |-----------------------------▶| | | ++ | | overlay dir scan | | ++ | | | | ++ | | to added layers | | ++ | | memory and symlink add | | ++ | | to deleted layers | | ++ | | memory and symlink del | | ++ | | valid overlay layers | | ++ | |---------------------------------▶| | ++ | | next scan | | ++ | | | | ++ | | | | ++ | | | | ++ | | | overlay-layers dir scan | ++ | | check overlay layer ready | | ++ | |◀---------------------------------| to added layers | ++ | |---------------------------------▶| filter invalid layers | ++ | | result | memory and symlink add | ++ | | | to deleted layers | ++ | | | memory and symlink del | ++ | | | valid overlay layers | ++ | | |---------------------------------▶| ++ | | | next scan | ++ | | | | ++ | | | | ++ | | | | ++ | | | | overlay-image dir scan ++ | | | check layers ready | ++ | | |◀---------------------------------| to added images ++ | | |---------------------------------▶| filter invalid images ++ | | | result | memory add images ++ | | | | to deleted images ++ | | | | memory del images ++ | | | | ++ |◀---------------------------------------------------------------------------------------------------| ++ | refresh end | | | ++ | | | | +++---------------------+ +---------------------+ +---------------------+ +-----------------------+ ++| refresh thread loop | | image remote module | | layer remote module | | overlay remote module | +++---------------------+ +---------------------+ +---------------------+ +-----------------------+ ++ ++``` ++ ++# 3. 接口描述 ++ ++```c ++// 初始化remote模块里的layer data ++int remote_layer_init(const char *root_dir); ++ ++// 初始化remote模块里的overlay data ++int remote_overlay_init(const char *driver_home); ++ ++// 清理remote模块的资源 ++void remote_maintain_cleanup(); ++ ++// 启动 定期更新的thread ++int start_refresh_thread(void); ++ ++// 创建新layer目录 ++int remote_layer_build_ro_dir(const char *id); ++ ++// 创建新overlay目录 ++int remote_overlay_build_ro_dir(const char *id); ++ ++// 删除layer目录 ++int remote_layer_remove_ro_dir(const char *id); ++ ++// 删除overlay目录 ++int remote_overlay_remove_ro_dir(const char *id); ++``` ++ ++# 4. 详细设计 ++分离RO目录的关键在于适配原来的代码逻辑，原先的代码在操作镜像和层的时候，不管是RO层还是RW层，从创建到删除都是在当前目录下进行的，这就是我们额外创建一个软连接的作用: ++- RO目录的作用是为了支持远程挂载 ++- 软连接的作用是模拟原来的目录结构 ++ ++这样以来，image module的逻辑几乎不需要改动，除了以下几点需要注意： ++- 创建和删除的时候需要处理一个额外的资源：软连接，之前只需要关注目录即可，现在如果创建的是只读层，就需要额外创建软连接，如果删除的是只读层，就需要额外删除软连接 ++- 以`overlay-layers`目录为例，isulad启动时会以正则规则扫描当前目录下的子目录是否合法，所以需要屏蔽`RO`目录 ++ ++定时刷新的逻辑如下： ++以`overlay-image`目录的刷新为例，通过维护两个集合`new` 和 `old`, 这两个集合初始都为空，通过扫描目录里面所有的子目录，把合法的image id 加入`new`集合， 通过计算两个集合的差， 在集合`new`里面存在而在集合`old`里面不存在的id则为新增加的镜像， 在集合`old`里面存在而在集合`new`里面不存在的id则为删除的镜像。处理新增加的镜像还需要额外的一个判断，就是判断镜像的层数据是否已经加载，如果没加载则该镜像本轮不加载。`overlay-layers` 和 `overlay` 目录的处理逻辑类似。 ++ ++ ++*可能的使用场景* ++一个可能的使用场景就是通过远程文件共享(nfs)让多台启动的isulad实例共享某些只读的数据；具体来说，在两个host A和B上都启动了iSulad, 如果A pull或者load了镜像busybox, 那么B上的isulad同样可以使用这个镜像。 ++ ++``` ++operations: ++ +++--------------------+ +--------------------+ +--------------------+ ++| isula pull busybox | | without pull | | without pull | ++| isula pull nginx | | isula run busybox | | isula run ngxinx | ++| isula pull ... | | isula run ... | | isula run centos | +++--------------------+ +--------------------+ +--------------------+ ++ | | | ++ ▼ ▼ ▼ +++======================+ +======================+ +======================+ ++| isulad on Host A | | isulad on Host B | | isulad on Host C | +++======================+ +======================+ +======================+ ++| image store module | | image store module | | image store module | +++----------------------+ +----------------------+ +----------------------+ ++| refresh thread off | | refresh thread on | | refresh thread on | +++----------------------+ +----------------------+ +----------------------+ ++| local rw | remote ro | | local rw | remote ro | | local rw | remote ro | +++----------------------+ +----------------------+ +----------------------+ ++ | | | ++ | enable nfs | mounted on | mounted on ++ ▼ ▼ ▼ ++ +=====================================================================+ ++ | nfs directory over network | ++ +=====================================================================+ ++ | image store | ++ +---------------------------------------------------------------------+ ++ | image | image | image | image | image | ++ | busybox | nginx | my-app | ubuntu | centos | ++ | 4MB | 100MB | 1.2GB | 5MB | 6MB | ++ +---------------------------------------------------------------------+ ++ | layers store | ++ +---------------------------------------------------------------------+ ++ | layer | layer | layer | ++ | 05c361054 | 8a9d75caad | 789d605ac | ++ +---------------------------------------------------------------------+ ++``` ++ ++*同步问题* ++共享资源并发使用发生竞争的条件是：`two process access the same resource concurrently and at least one of the access is a writer`。这里的共享资源有： ++ ++``` +++============================+ +=====================================+ ++| Sharing Resource | | Storage | +++============================+ +=====================================+ ++| read-only overlay layers | mounted | /var/lib/isulad/overlay/RO | +++----------------------------+ ======▶ +-------------------------------------+ ++| reald-only layers metadata | shared | /var/lib/isulad/overlay-layers/RO | +++----------------------------+ +-------------------------------------+ ++| reald-only images | | /var/lib/isulad/overlay-images | +++----------------------------+ +-------------------------------------+ ++ ++``` ++而分布在不同host上的isulad进程通过网络共享这些资源，如果不考虑新增删除的情况，不会出现资源竞争，所有的节点都是reader。 ++ ++对于主节点pull镜像，其他节点使用镜像的情况，主节点是writer，其他节点是reader。这时候可能出现的问题是主节点pull镜像的流程没有完全结束，但是其他节点开始使用这个不完整的镜像。对于这个问题的解决方案是通过扫描image目录来新增镜像，通过这种方式能确保一定该新增镜像的信息完整。 ++ ++``` +++---------------------+ +--------------------+ +-----------------------+ +-----------------------+ ++| registry | | image module | | layer store module | | driver overlay module | +++---------------------+ +--------------------+ +-----------------------+ +-----------------------+ ++ | | | | ++ | | | | ++ | registry_pull | | | ++ | fetch_manifest | | | ++ | check reuse and fetch | | | ++ | +----------------+ | | | ++ | | register_layer | | | | ++ | +----------------+ | | | ++ |-----------------------------------------------------------------▶| | ++ | | | layer_store_create | ++ | | |--------------------------------▶| ++ | | | | driver_create_layer ++ | | | | +--------------------+ ++ | | | | | setup overlay dir | ++ | | | | +--------------------+ ++ | | | | driver_create_layer done ++ | | |◀--------------------------------| ++ | | | +------------+ | ++ | | | | save layer | | ++ | | | +------------+ | ++ | | | layer create done | ++ |◀-----------------------------------------------------------------| | ++ | all layer setup | | | ++ | +----------------+ | | | ++ | | register image | | | | ++ | +----------------+ | | | ++ |-------------------------------▶| | | ++ | | storage_img_create | | ++ | | set img top layer | | ++ | | img create | | ++ | | +------------+ | | ++ | | | save image | | | ++ | | +------------+ | | ++ | | create image done | | ++ |◀-------------------------------| | | ++ | pull img done | | | ++ | | | | ++ | | | | +++---------------------+ +--------------------+ +-----------------------+ +-----------------------+ ++| registry | | image module | | layer store module | | driver overlay module | +++---------------------+ +--------------------+ +-----------------------+ +-----------------------+ ++ ++``` ++ ++至于主节点删除镜像的情况，主节点是writer，其他节点是reader，可能出现的情况是其他节点还有容器在使用镜像的时候，镜像被删除，但是根据需求场景暂不处理这种情况。其他的处理与新增镜像相同，依然以image dir作为入口，扫描发现删除的镜像。删除时需要关注layer和overlay目录下的软链接。 +-- +2.25.1 + diff --git a/0045-Refine-a-minor-log-message.patch b/0045-Refine-a-minor-log-message.patch new file mode 100644 index 0000000..675984b --- /dev/null +++ b/0045-Refine-a-minor-log-message.patch @@ -0,0 +1,34 @@ +From 9b7e37018612f7443a0f8f966106995da31cc292 Mon Sep 17 00:00:00 2001 +From: sailorvii +Date: Fri, 17 Mar 2023 06:34:37 +0000 +Subject: [PATCH 45/53] Refine a minor log message. + +--- + src/daemon/modules/network/native/adaptor_native.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/src/daemon/modules/network/native/adaptor_native.c b/src/daemon/modules/network/native/adaptor_native.c +index 02ff3642..8bc386d1 100644 +--- a/src/daemon/modules/network/native/adaptor_native.c ++++ b/src/daemon/modules/network/native/adaptor_native.c +@@ -1764,8 +1764,7 @@ bool has_connected_container(native_network *network) + message[pos - 1] = '\0'; + } + +- ERROR("network %s has connected containers [ %s ]", network->conflist->list->name, message); +- isulad_set_error_message("network %s has connected containers [ %s ]", network->conflist->list->name, message); ++ INFO("network %s has connected containers [ %s ]", network->conflist->list->name, message); + + out: + native_network_unlock(network); +@@ -1923,6 +1922,7 @@ int native_config_remove(const char *name, char **res_name) + } + + if (has_connected_container(network)) { ++ isulad_set_error_message("network %s has connected containers", network->conflist->list->name); + ret = -1; + goto out; + } +-- +2.25.1 + diff --git a/0046-modify-the-return-value-of-the-util_waitpid_with_tim.patch b/0046-modify-the-return-value-of-the-util_waitpid_with_tim.patch new file mode 100644 index 0000000..f5a1aed --- /dev/null +++ b/0046-modify-the-return-value-of-the-util_waitpid_with_tim.patch @@ -0,0 +1,44 @@ +From ea8962c6ee3ed05cfabe84c58c47cf56e3388242 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Mon, 20 Mar 2023 14:17:00 +0800 +Subject: [PATCH 46/53] modify the return value of the + util_waitpid_with_timeout to status + +Signed-off-by: zhongtao +--- + src/utils/cutils/utils.c | 5 +++-- + 1 file changed, 3 insertions(+), 2 deletions(-) + +diff --git a/src/utils/cutils/utils.c b/src/utils/cutils/utils.c +index 983d81d8..3500d8f8 100644 +--- a/src/utils/cutils/utils.c ++++ b/src/utils/cutils/utils.c +@@ -319,6 +319,7 @@ rep: + int util_waitpid_with_timeout(pid_t pid, const int64_t timeout, handle_timeout_callback_t cb) + { + int nret = 0; ++ int st; + time_t start_time = time(NULL); + time_t end_time; + double interval; +@@ -328,7 +329,7 @@ int util_waitpid_with_timeout(pid_t pid, const int64_t timeout, handle_timeout_c + } + + for (;;) { +- nret = waitpid(pid, NULL, WNOHANG); ++ nret = waitpid(pid, &st, WNOHANG); + if (nret == pid) { + break; + } +@@ -347,7 +348,7 @@ int util_waitpid_with_timeout(pid_t pid, const int64_t timeout, handle_timeout_c + // sleep some time instead to avoid cpu full running and then retry. + usleep(100); + } +- return 0; ++ return st; + } + + int util_wait_for_pid_status(pid_t pid) +-- +2.25.1 + diff --git a/0047-fix-util_getgrent_r-overflow.patch b/0047-fix-util_getgrent_r-overflow.patch new file mode 100644 index 0000000..038aabc --- /dev/null +++ b/0047-fix-util_getgrent_r-overflow.patch @@ -0,0 +1,74 @@ +From fc853c60c56fc3e8dc3c78a8081fea2f9ecc64dc Mon Sep 17 00:00:00 2001 +From: zhangxiaoyu +Date: Tue, 21 Mar 2023 18:52:43 +0800 +Subject: [PATCH 47/53] fix util_getgrent_r overflow + +Signed-off-by: zhangxiaoyu +--- + src/utils/cutils/utils_pwgr.c | 3 +++ + test/cutils/utils_pwgr/long_sample | 1 + + test/cutils/utils_pwgr/utils_pwgr_ut.cc | 28 +++++++++++++++++++++++++ + 3 files changed, 32 insertions(+) + create mode 100644 test/cutils/utils_pwgr/long_sample + +diff --git a/src/utils/cutils/utils_pwgr.c b/src/utils/cutils/utils_pwgr.c +index 75efadb2..064bae1f 100644 +--- a/src/utils/cutils/utils_pwgr.c ++++ b/src/utils/cutils/utils_pwgr.c +@@ -157,6 +157,9 @@ static char **hold_string_list(char **line, char *buf_start, char *buf_end, cons + } + + out: ++ if ((char *)(walker + 2) > buf_end) { ++ return NULL; ++ } + *walker = NULL; + return result; + } +diff --git a/test/cutils/utils_pwgr/long_sample b/test/cutils/utils_pwgr/long_sample +new file mode 100644 +index 00000000..7e991ffd +--- /dev/null ++++ b/test/cutils/utils_pwgr/long_sample +@@ -0,0 +1 @@ ++adm:x:4:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +\ No newline at end of file +diff --git a/test/cutils/utils_pwgr/utils_pwgr_ut.cc b/test/cutils/utils_pwgr/utils_pwgr_ut.cc +index 431372d3..20de8a89 100644 +--- a/test/cutils/utils_pwgr/utils_pwgr_ut.cc ++++ b/test/cutils/utils_pwgr/utils_pwgr_ut.cc +@@ -140,3 +140,31 @@ TEST(utils_pwgr, test_getgrent_r) + + fclose(f_gr); + } ++ ++TEST(utils_pwgr, test_long_getgrent_r) ++{ ++ std::string path = "../../../../test/cutils/utils_pwgr/long_sample"; ++ FILE *f_gr = fopen(path.c_str(), "r"); ++ ASSERT_NE(f_gr, nullptr); ++ ++ struct group gr { ++ 0 ++ }; ++ struct group *pgr = nullptr; ++ char mark_before[BUFSIZ] = { 0 }; ++ char buf[BUFSIZ] = { 0 }; ++ char mark_after[BUFSIZ] = { 0 }; ++ ++ for (int num = 0; num < BUFSIZ; num++) { ++ mark_before[num] = 0xEE; ++ mark_after[num] = 0xDD; ++ } ++ ++ (void)util_getgrent_r(f_gr, &gr, buf, sizeof(buf), &pgr); ++ for (int num = 0; num < BUFSIZ; num++) { ++ ASSERT_EQ(mark_before[num], (char)0xEE); ++ ASSERT_EQ(mark_after[num], (char)0xDD); ++ } ++ ++ fclose(f_gr); ++} +-- +2.25.1 + diff --git a/0048-add-ut-for-runc.patch b/0048-add-ut-for-runc.patch new file mode 100644 index 0000000..e7e4a01 --- /dev/null +++ b/0048-add-ut-for-runc.patch @@ -0,0 +1,188 @@ +From c20a580a7d8c6a8a16273513c6d11edd827b90b0 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Thu, 23 Feb 2023 14:39:23 +0800 +Subject: [PATCH 48/53] add ut for runc + +Signed-off-by: zhongtao +--- + .../modules/runtime/isula/isula_rt_ops.c | 5 ++ + test/cutils/utils_utils/CMakeLists.txt | 2 + + test/cutils/utils_utils/utils_utils_ut.cc | 61 +++++++++++++++++++ + test/runtime/isula/isula_rt_ops_ut.cc | 56 +++++++++++++++++ + 4 files changed, 124 insertions(+) + +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index 5ec0e639..fe0f227c 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -1553,6 +1553,11 @@ out: + + int rt_isula_kill(const char *id, const char *runtime, const rt_kill_params_t *params) + { ++ if (id == NULL || runtime == NULL || params == NULL || params->pid < 0) { ++ ERROR("Invalid arguments not allowed"); ++ return -1; ++ } ++ + if (util_process_alive(params->pid, params->start_time) == false) { + if (params->signal == params->stop_signal || params->signal == SIGKILL) { + WARN("Process %d is not alive", params->pid); +diff --git a/test/cutils/utils_utils/CMakeLists.txt b/test/cutils/utils_utils/CMakeLists.txt +index 99a83e7a..6d276390 100644 +--- a/test/cutils/utils_utils/CMakeLists.txt ++++ b/test/cutils/utils_utils/CMakeLists.txt +@@ -12,6 +12,8 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/map + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils + ) ++ ++set_target_properties(${EXE} PROPERTIES LINK_FLAGS "-Wl,--wrap,waitpid") + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) + set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/cutils/utils_utils/utils_utils_ut.cc b/test/cutils/utils_utils/utils_utils_ut.cc +index 5308351a..18f0a506 100644 +--- a/test/cutils/utils_utils/utils_utils_ut.cc ++++ b/test/cutils/utils_utils/utils_utils_ut.cc +@@ -15,6 +15,44 @@ + + #include + #include "utils.h" ++#include "mock.h" ++ ++static pid_t test_pid = -1; ++ ++extern "C" { ++ DECLARE_WRAPPER_V(waitpid, pid_t, (__pid_t pid, int *stat_loc, int options)); ++ DEFINE_WRAPPER_V(waitpid, pid_t, (__pid_t pid, int *stat_loc, int options),(pid, stat_loc, options)); ++} ++ ++static pid_t waitpid_none_zero(__pid_t pid, int *stat_loc, int options) ++{ ++ *stat_loc = 256; ++ return test_pid; ++} ++ ++static pid_t waitpid_zero(__pid_t pid, int *stat_loc, int options) ++{ ++ *stat_loc = 0; ++ return test_pid; ++} ++ ++#define ExitSignalOffset 128 ++static int status_to_exit_code(int status) ++{ ++ int exit_code = 0; ++ ++ if (WIFEXITED(status)) { ++ exit_code = WEXITSTATUS(status); ++ } else { ++ exit_code = -1; ++ } ++ if (WIFSIGNALED(status)) { ++ int signal; ++ signal = WTERMSIG(status); ++ exit_code = ExitSignalOffset + signal; ++ } ++ return exit_code; ++} + + TEST(utils_utils, test_util_mem_realloc) + { +@@ -284,4 +322,27 @@ TEST(utils_utils, test_do_retry_call) + DO_RETRY_CALL(10, 100, nret, retry_call_test, 11); + ASSERT_EQ(global_total, 10); + ASSERT_EQ(nret, -1); ++} ++ ++TEST(utils_utils, test_util_waitpid_with_timeout) ++{ ++ int64_t timeout = 2; ++ pid_t pid = getpid(); ++ int status = 0; ++ ++ test_pid = pid; ++ MOCK_SET_V(waitpid, waitpid_none_zero); ++ status = util_waitpid_with_timeout(test_pid, timeout, nullptr); ++ ASSERT_EQ(status, 256); ++ ASSERT_EQ(status_to_exit_code(status), 1); ++ MOCK_CLEAR(waitpid); ++ ++ MOCK_SET_V(waitpid, waitpid_zero); ++ status = util_waitpid_with_timeout(test_pid, timeout, nullptr); ++ ASSERT_EQ(status, 0); ++ ASSERT_EQ(status_to_exit_code(status), 0); ++ MOCK_CLEAR(waitpid); ++ ++ ASSERT_EQ(util_waitpid_with_timeout(pid, timeout, nullptr), -1); ++ + } +\ No newline at end of file +diff --git a/test/runtime/isula/isula_rt_ops_ut.cc b/test/runtime/isula/isula_rt_ops_ut.cc +index 03c213a5..f37e62a0 100644 +--- a/test/runtime/isula/isula_rt_ops_ut.cc ++++ b/test/runtime/isula/isula_rt_ops_ut.cc +@@ -163,3 +163,59 @@ TEST_F(IsulaRtOpsUnitTest, test_rt_isula_exec_resize) + close(fd); + ASSERT_EQ(system(rm_path.c_str()), 0); + } ++ ++TEST_F(IsulaRtOpsUnitTest, test_rt_isula_update) ++{ ++ rt_update_params_t params = {}; ++ ASSERT_EQ(rt_isula_update(nullptr, nullptr, nullptr), -1); ++ ++ ASSERT_EQ(rt_isula_update("123", nullptr, nullptr), -1); ++ ASSERT_EQ(rt_isula_update("123", "runtime", nullptr), -1); ++ ASSERT_EQ(rt_isula_update("123", "runtime", ¶ms), -1); ++} ++ ++TEST_F(IsulaRtOpsUnitTest, test_rt_isula_pause) ++{ ++ rt_pause_params_t params = {}; ++ ASSERT_EQ(rt_isula_pause(nullptr, nullptr, nullptr), -1); ++ ++ ASSERT_EQ(rt_isula_pause("123", nullptr, nullptr), -1); ++ ASSERT_EQ(rt_isula_pause("123", "runtime", nullptr), -1); ++ ASSERT_EQ(rt_isula_pause("123", "runtime", ¶ms), -1); ++} ++ ++TEST_F(IsulaRtOpsUnitTest, test_rt_isula_resume) ++{ ++ rt_resume_params_t params = {}; ++ ASSERT_EQ(rt_isula_resume(nullptr, nullptr, nullptr), -1); ++ ++ ASSERT_EQ(rt_isula_resume("123", nullptr, nullptr), -1); ++ ASSERT_EQ(rt_isula_resume("123", "runtime", nullptr), -1); ++ ASSERT_EQ(rt_isula_resume("123", "runtime", ¶ms), -1); ++} ++ ++TEST_F(IsulaRtOpsUnitTest, test_rt_isula_resources_stats) ++{ ++ rt_stats_params_t params = {}; ++ struct runtime_container_resources_stats_info stats = {}; ++ ++ ASSERT_EQ(rt_isula_resources_stats(nullptr, nullptr, nullptr, nullptr), -1); ++ ++ ASSERT_EQ(rt_isula_resources_stats("123", nullptr, nullptr, nullptr), -1); ++ ASSERT_EQ(rt_isula_resources_stats("123", "runtime", nullptr, nullptr), -1); ++ ASSERT_EQ(rt_isula_resources_stats("123", "runtime", ¶ms, nullptr), -1); ++ params.state = "/var/run/isulad/runtime"; ++ ASSERT_EQ(rt_isula_resources_stats("123", "runtime", ¶ms, &stats), -1); ++} ++ ++TEST_F(IsulaRtOpsUnitTest, test_rt_isula_kill) ++{ ++ rt_kill_params_t kill_params = { ++ .pid = -1, ++ }; ++ ASSERT_EQ(rt_isula_kill(nullptr, nullptr, nullptr), -1); ++ ++ ASSERT_EQ(rt_isula_kill("123", nullptr, nullptr), -1); ++ ASSERT_EQ(rt_isula_kill("123", "runtime", nullptr), -1); ++ ASSERT_EQ(rt_isula_kill("123", "runtime", &kill_params), -1); ++} +\ No newline at end of file +-- +2.25.1 + diff --git a/0049-add-runc-doc.patch b/0049-add-runc-doc.patch new file mode 100644 index 0000000..11f8de2 --- /dev/null +++ b/0049-add-runc-doc.patch @@ -0,0 +1,506 @@ +From eb3e8dd9e87e926a121715d103cc6c3cb20388a7 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Fri, 17 Feb 2023 14:33:15 +0800 +Subject: [PATCH 49/53] add runc doc + +Signed-off-by: zhongtao +--- + docs/design/README_zh.md | 1 + + .../design/detailed/Runtime/runc_design_zh.md | 264 ++++++++++++++++++ + docs/images/isulad_shim_flow_chart.svg | 16 ++ + docs/images/k8s_isulad_flow_chart.svg | 16 ++ + docs/manual/README_zh.md | 6 +- + docs/manual/runc_config_zh.md | 134 +++++++++ + 6 files changed, 436 insertions(+), 1 deletion(-) + create mode 100644 docs/design/detailed/Runtime/runc_design_zh.md + create mode 100755 docs/images/isulad_shim_flow_chart.svg + create mode 100644 docs/images/k8s_isulad_flow_chart.svg + create mode 100644 docs/manual/runc_config_zh.md + +diff --git a/docs/design/README_zh.md b/docs/design/README_zh.md +index 1f1c94b5..e7763a42 100644 +--- a/docs/design/README_zh.md ++++ b/docs/design/README_zh.md +@@ -49,6 +49,7 @@ + ## Runtime + + - 查看 runtime 模块的设计文档： [runtime_design](./detailed/Runtime/runtime_design_zh.md) 。 ++- 查看 isulad 支持runc 的设计文档：[runc_design](./detailed/Runtime/runc_design_zh.md)。 + + ## Security + +diff --git a/docs/design/detailed/Runtime/runc_design_zh.md b/docs/design/detailed/Runtime/runc_design_zh.md +new file mode 100644 +index 00000000..3af3ea02 +--- /dev/null ++++ b/docs/design/detailed/Runtime/runc_design_zh.md +@@ -0,0 +1,264 @@ ++| Author | zhongtao | ++| ------ | --------------------- | ++| Date | 2023-03-21 | ++| Email | zhongtao17@huawei.com | ++ ++# 方案目标 ++ ++isulad 支持使用runc作为容器runtime。用户可以在启动容器时配置`--runtime=runc`创建runtime为runc的容器： ++```bash ++$ isula run -tid -n test --runtime=runc busybox sh ++eb77c672a148cc4cefa0d6e7c5847f5a87d0c5353e245461b68820bd9af90c67 ++$ isula inspect eb77 | grep -i runc ++ "ResolvConfPath": "/var/lib/isulad/engines/runc/eb77c672a148cc4cefa0d6e7c5847f5a87d0c5353e245461b68820bd9af90c67/resolv.conf", ++ "HostsPath": "/var/lib/isulad/engines/runc/eb77c672a148cc4cefa0d6e7c5847f5a87d0c5353e245461b68820bd9af90c67/hosts", ++ "LogPath": "/var/lib/isulad/engines/runc/eb77c672a148cc4cefa0d6e7c5847f5a87d0c5353e245461b68820bd9af90c67/console.log", ++ "Runtime": "runc", ++ "log.console.file": "/var/lib/isulad/engines/runc/eb77c672a148cc4cefa0d6e7c5847f5a87d0c5353e245461b68820bd9af90c67/console.log", ++``` ++ ++同时，也可以在`/etc/isulad/daemon.json`中配置`default-runtime`为runc并重启isulad，修改isulad创建容器时默认使用的runtime。 ++```sh ++$ vim /etc/isulad/daemon.json ++ ... ++ "default-runtime": "runc" ++ ... ++$ sudo isulad ++``` ++ ++# 总体设计 ++ ++由于isulad与runc之间的交互存在gap，且将容器创建成功之后，容器进程的生命周期与isulad进程的生命周期没有必然联系，因此我们设计了一个isulad-shim进程，用于isulad与runc的交互并将isulad与容器实例解耦。同时，由于只有create以及exec涉及到在容器中新建进程，因此只有这两个子命令需要创建isulad-shim。其他的子命令直接通过调用runc二进制实现。 ++ ++## 时序图 ++ ++```mermaid ++sequenceDiagram ++ participant isula ++ participant kubelet ++ participant isulad ++ participant supervisor ++ participant shim ++ participant runc ++ participant container ++ ++ isula->>isulad: request ++ kubelet->>isulad:request ++ alt create ++ isulad->>shim:shim_creat() ++ shim->>runc: execvp(runc, params) ++ runc ->> container:create request ++ container ->> runc:success ++ runc ->> shim:get process pid ++ isulad ->> isulad:get process pid ++ isulad ->> supervisor:add monitor ++ loop epoll exit_fd ++ supervisor ->> shim:if exit? ++ end ++ else exec ++ isulad->>shim:shim_creat() ++ par ++ shim->>runc: execvp(runc, params) ++ runc ->> container:exec request ++ container ->> runc:success ++ runc ->> shim:get process pid ++ shim ->> container:wait process pid ++ shim ->> shim:exit ++ and ++ isulad ->> isulad: wait isulad-shim pid ++ end ++ else others container cmd ++ isulad->>runc: runtime_call_simple() ++ runc ->> container:cmd ++ container ->>runc:success ++ runc ->>isulad:success ++ end ++ isulad ->> isula:response ++ isulad ->> kubelet:response ++``` ++ ++# 接口描述 ++ ++## cri接口 ++ ++### PodSandboxManagerService ++ ++```h ++auto RunPodSandbox(const runtime::v1alpha2::PodSandboxConfig &config, const std::string &runtimeHandler,Errors &error) -> std::string; ++ ++void StopPodSandbox(const std::string &podSandboxID, Errors &error); ++ ++void RemovePodSandbox(const std::string &podSandboxID, Errors &error); ++ ++auto PodSandboxStatus(const std::string &podSandboxID, Errors &error) ++ -> std::unique_ptr; ++ ++void ListPodSandbox(const runtime::v1alpha2::PodSandboxFilter *filter,std::vector> *pods, Errors &error); ++ ++// This feature is temporarily not supported ++void PortForward(const runtime::v1alpha2::PortForwardRequest &req,runtime::v1alpha2::PortForwardResponse *resp,Errors &error); ++ ... ... ++}; ++} // namespace CRI ++``` ++ ++### ContainerManagerService ++ ++```c ++auto CreateContainer(const std::string &podSandboxID, const runtime::v1alpha2::ContainerConfig &containerConfig,const runtime::v1alpha2::PodSandboxConfig &podSandboxConfig, Errors &error)-> std::string override; ++ ++void StartContainer(const std::string &containerID, Errors &error) override; ++ ++void StopContainer(const std::string &containerID, int64_t timeout, Errors &error) override; ++ ++void RemoveContainer(const std::string &containerID, Errors &error) override; ++ ++void ListContainers(const runtime::v1alpha2::ContainerFilter *filter, ++ std::vector> *containers, Errors &error) override; ++ ++void ListContainerStats(const runtime::v1alpha2::ContainerStatsFilter *filter,std::vector> *containerstats,Errors &error) override; ++ ++auto ContainerStats(const std::string &containerID, Errors &error) ++ -> std::unique_ptr override; ++ ++auto ContainerStatus(const std::string &containerID, Errors &error) -> std::unique_ptr override; ++ ++void ExecSync(const std::string &containerID, const google::protobuf::RepeatedPtrField &cmd, int64_t timeout, runtime::v1alpha2::ExecSyncResponse *reply, Errors &error) override; ++ ++void Exec(const runtime::v1alpha2::ExecRequest &req, runtime::v1alpha2::ExecResponse *resp, Errors &error) override; ++ ++void Attach(const runtime::v1alpha2::AttachRequest &req, runtime::v1alpha2::AttachResponse *resp, Errors &error) override; ++ ++void UpdateContainerResources(const std::string &containerID, ++ const runtime::v1alpha2::LinuxContainerResources &resources, Errors &error) override; ++``` ++ ++### RuntimeManagerService ++ ++```c ++void UpdateRuntimeConfig(const runtime::v1alpha2::RuntimeConfig &config, Errors &error) override; ++ ++// 对应crictl info ++auto Status(Errors &error) -> std::unique_ptr override; ++``` ++ ++### ImageManagerService ++ ++```c ++void ListImages(const runtime::v1alpha2::ImageFilter &filter, std::vector> *images, Errors &error) override; std::unique_ptr ImageStatus(const runtime::v1alpha2::ImageSpec &image, Errors &error) override; ++ ++std::string PullImage(const runtime::v1alpha2::ImageSpec &image, const runtime::v1alpha2::AuthConfig &auth, Errors &error) override; ++ ++void RemoveImage(const runtime::v1alpha2::ImageSpec &image, Errors &error) override; ++ ++void ImageFsInfo(std::vector> *usages, Errors &error) override ++``` ++ ++ ++ ++## isula_rt_ops模块 ++ ++```c ++// 检测runtime是否为isula_rt_ops模块处理的目标runtime ++bool rt_isula_detect(const char *runtime); ++ ++int rt_isula_create(const char *name, const char *runtime, const rt_create_params_t *params); ++ ++int rt_isula_start(const char *name, const char *runtime, const rt_start_params_t *params, pid_ppid_info_t *pid_info); ++ ++// restart not implemented ++int rt_isula_restart(const char *name, const char *runtime, const rt_restart_params_t *params); ++ ++int rt_isula_clean_resource(const char *name, const char *runtime, const rt_clean_params_t *params); ++ ++int rt_isula_rm(const char *name, const char *runtime, const rt_rm_params_t *params); ++ ++int rt_isula_exec(const char *id, const char *runtime, const rt_exec_params_t *params, int *exit_code); ++ ++int rt_isula_status(const char *name, const char *runtime, const rt_status_params_t *params, ++ struct runtime_container_status_info *status); ++ ++// isula attach not support on isulad-shim ++int rt_isula_attach(const char *id, const char *runtime, const rt_attach_params_t *params); ++ ++int rt_isula_update(const char *id, const char *runtime, const rt_update_params_t *params); ++ ++int rt_isula_pause(const char *id, const char *runtime, const rt_pause_params_t *params); ++ ++int rt_isula_resume(const char *id, const char *runtime, const rt_resume_params_t *params); ++ ++// isula top/listpids not support on isulad-shim ++int rt_isula_listpids(const char *name, const char *runtime, const rt_listpids_params_t *params, ++ rt_listpids_out_t *out); ++ ++int rt_isula_resources_stats(const char *name, const char *runtime, const rt_stats_params_t *params, ++ struct runtime_container_resources_stats_info *rs_stats); ++// rt_isula_resize not impl ++int rt_isula_resize(const char *id, const char *runtime, const rt_resize_params_t *params); ++ ++int rt_isula_exec_resize(const char *id, const char *runtime, const rt_exec_resize_params_t *params); ++ ++int rt_isula_kill(const char *id, const char *runtime, const rt_kill_params_t *params); ++ ++``` ++ ++ ++ ++# 详细设计 ++ ++## create 实现流程 ++ ++isulad端： ++ ++1. 创建process文件：create_process_json_file(); ++2. 获得runtime二进制：get_runtime_cmd(runtime, &cmd); ++3. 利用两次fork()创建isulad-shim进程， 此时的isulad-shim进程是1号进程的子进程，与isulad无父子关系:shim_create(); ++4. 若shim创建成功则直接返回成功，若失败则调用runtime_call_delete_force()将容器进程都force delete。 ++ ++isulad-shim端： ++ ++1. 根据解析process文件新建一个process，new_process()； ++2. 开启exit_fifo：open_no_inherit("exit_fifo", O_WRONLY, -1); ++3. 为io_copy创建io_epoll_loop线程：process_io_init(); ++4. 创建进行io copy的线程，并将isulad的console与runtime的console连接起来:open_io(); ++5. 创建运行runtime的子进程，获得容器中进程的pid，create_process(); ++6. 循环wait子进程，直到wait到的子进程为容器进程pid则退出shim进程，process_signal_handle_routine(); ++ ++## exec 实现流程 ++ ++isulad端： ++ ++1. 创建process文件：create_process_json_file(); ++2. 获得runtime二进制：get_runtime_cmd(runtime, &cmd); ++3. 若`--detach=false`，直接fork()创建isulad-shim进程，此时的isulad-shim进程是isulad进程的子进程，isulad进程wait等待isulad-shim进程退出；若-`-detach=true`，则与create一样，创建独立的isulad-shim进程:shim_create(); ++4. 循环读取pid文件获得容器进程id说明exec命令成功，get_container_process_pid。 ++ ++isulad-shim端： ++ ++1. 根据解析process文件新建一个process，new_process()； ++2. 为io_copy创建io_epoll_loop线程：process_io_init(); ++3. 创建进行io copy的线程，并将isulad的console与runtime的console连接起来:open_io(); ++4. 创建运行runtime的子进程，获得容器中进程的pid，create_process(); ++5. 循环wait子进程，直到wait到的子进程为容器进程pid则退出shim进程，process_signal_handle_routine(); ++ ++ ++ ++## start 实现流程 ++ ++isulad端： ++ ++1. 分别读取pid以及shim-pid文件获得容器进程pid以及isulad-shim pid； ++2. 依次获得容器进程以及isulad-shim进程的proc信息：util_get_process_proc_info(); ++3. 根据proc信息为pid_ppid_info_t结构体赋值； ++4. 直接调用runc二进制start容器：runtime_call_simple(); ++5. 之后isulad根据监听exit_fifo_fd，感知容器是否退出，从而更新容器状态。 ++ ++其他子命令与satrt类似，均是调用runtime_call_simple()函数直接调用runc二进制，此处不再赘述。 ++ ++ ++ ++## 流程图 ++ ++### isulad与isulad-shim交互流程图 ++ ++![isulad_shim_flow_chart](../../../images/isulad_shim_flow_chart.svg) +diff --git a/docs/images/isulad_shim_flow_chart.svg b/docs/images/isulad_shim_flow_chart.svg +new file mode 100755 +index 00000000..9a509b80 +--- /dev/null ++++ b/docs/images/isulad_shim_flow_chart.svg +@@ -0,0 +1,16 @@ ++ ++ ++ ++ ++ ++ ++ isulad-shimmain()isulad-shim id bundle runtime info 2m0s从process.json中加载process：new_processif p->state->exec?将create的process存储在process.jsonwe文件中：create_process_json_fileisuladget_runtime_cmdret = shim_createret = -1runtime_call_delete_forcert_isula_creatert_isula_execget_runtime_cmdret = shim_createget_container_process_pid将exec的process存储在process.jsonwe文件中create_process_json_fileget_container_process_pidfile_read_int(shim_pid_file_name, &shim_pid);util_get_process_proc_inforuntime_call_simple(workdir,runtime, "start", NULL, 0, id, NULL)rt_isula_startfg=false;exit_code=NULLshim_createfork()若设置了--detach，则fg=true;否则为false；exit_code!=NULLparent(isulad)child(shim)从读fifo中读取isulad-shimxier写入的内容num = util_read_nointrif num > 0nowait isulad-shim或者shim parent 进程util_waitpid_with_timeoutif ret != 0show_shim_runtime_errlogif timeout <= 0kill(pid, SIGKILL)yeschdir(workdir)if fgnounsetenv("NOTIFY_SOCKET")fork()parent(shim parent)child(isulad-shim)将shim的pid写入文件中：file_write_int(fpid, pid)_exit(EXIT_SUCCESS);setsid()将从父进程继承的fd关闭util_check_inheritedexecvp(SHIM_BINARY, (char * const *)params);设置启动isulad-shim超时的定时器set_timeout_exit(DEFAULT_TIMEOUT);set_subreaper()解析参数：parse_argsyes打开exit_fd用于感知退出：open_no_inherit("exit_fifo", O_WRONLY, -1)为isulad与runtime之间的io copychuan构建main loop和epoll：：process_io_init(p);open_io(p, &tid_accept);no创建io copy线程：start_io_copy_threads(p);if (p->state->terminal) true开勇socket实现isulad与runtime的通信：open_terminal_io(p, tid_accept);利用pipe实现通信：open_generic_io(p);falsecreate_process(p);fork()child:runtime processparent:isulad-shim process拼接params，并调用runc二进制执行命令：exec_runtime_process(p, exec_fd[1]);读取runtime写入的信息： read_nointr(exec_fd[0],exec_buff, sizeof(exec_buff) - 1);等待runtime子进程退出：waitpid(pid, NULL, 0);获得runtime写入的容器中进程的pid：read_text_file("pid");将其赋值给process：p->ctr_pid = ctr_pid;清除超时计时器：released_timeout_exit();process_signal_handle_routine(p, tid_accept);wait任意一个子进程退出，当pid为ctr_pid时ret为0且exit_shim = true：reap_container(p->ctr_pid, &status);if ret == 0? noyesif exit_shim? yes调用runc kill命令杀死容器进程：process_kill_all(p);等待所有子进程被杀死：DO_RETRY_CALL(120, 1000000, nret, try_wait_all_child);调用runc delete命令删除容器进程：process_delete(p)if p->exit_fd > 0yeswrite_nointr(p->exit_fd, &status, sizeof(int));pthread_timedjoin_np(tid_accept, NULL, &ts);nodestroy_io_thread(p, i); +\ No newline at end of file +diff --git a/docs/images/k8s_isulad_flow_chart.svg b/docs/images/k8s_isulad_flow_chart.svg +new file mode 100644 +index 00000000..700b5b6d +--- /dev/null ++++ b/docs/images/k8s_isulad_flow_chart.svg +@@ -0,0 +1,16 @@ ++ ++ ++ ++ ++ ++ ++ kubeletclientisuladservergrpcCRIPodSandboxManagerServiceruntime service implImageManagerServiceservice_executor_t：callback函数Network::PluginManagerRunPodSandbox1. Pull the image for the sandboxEnsureSandboxImageExists2. Create the sandbox containerCreateSandboxContainerRuntimeVersionerServiceContainerManagerServicePodSandboxManagerServiceRuntimeManagerServiceRuntimeManagerServiceContainerManagerServiceRuntimeVersionerServiceservice_executor_t：callback函数Network::PluginManagerservice_executor_tservice_executor_tNetwork::PluginManager1. 创建create容器的requestGenerateSandboxCreateContainerRequest2. 发送请求m_cb->container.create(create_request, &create_response)3: Enable network SetNetworkReady4: Inspect container：直接从container_t中获得容器信息：inspect_data =CRIHelpers::InspectContainer5: Get networking info GetSandboxNetworkInfo 6: Mount network namespace when network mode is cninamespace_is_cn；prepare_network_namespace7: Setup networking for the sandbox. SetupSandboxNetwork8: Start the sandbox container. StartSandboxContainer 9: Save network settings json to disk ips_request = GenerateUpdateNetworkSettingsReqestinput：run pod的config文件解析出来的结构体output：pause容器的idCreateContainerinput：podSandboxID、containerConfig、podSandboxConfigoutput：pod中容器的id1. 根据podsandbox的id或者名字获得容器idGetRealContainerOrSandboxID2. 获得pod的runtimeGetContainerOrSandboxRuntime3. 创建pod中的容器的create请求GenerateCreateContainerRequest4. 调用创建cbm_cb->container.create +\ No newline at end of file +diff --git a/docs/manual/README_zh.md b/docs/manual/README_zh.md +index 8de5604b..1f3f37f8 100644 +--- a/docs/manual/README_zh.md ++++ b/docs/manual/README_zh.md +@@ -22,4 +22,8 @@ Device Mapper 是一个基于内核的框架，它支持 Linux 上的许多高 + + ## isula search使用指南 + +-关于如何使用isula search请参考[isula_search](isula_search_zh.md) +\ No newline at end of file ++关于如何使用isula search请参考[isula_search](isula_search_zh.md) ++ ++## runc使用指南 ++ ++关于如何在isulad中使用runc请参考[runc_config](runc_config_zh.md) +\ No newline at end of file +diff --git a/docs/manual/runc_config_zh.md b/docs/manual/runc_config_zh.md +new file mode 100644 +index 00000000..003cd2c5 +--- /dev/null ++++ b/docs/manual/runc_config_zh.md +@@ -0,0 +1,134 @@ ++# runc使用指南 ++本文主要是指导iSulad社区开发者和使用者，如何配置isulad使用runc作为runtime创建容器。 ++ ++## 一、runc的安装 ++ ++`tips`: 在安装runc之前需要安装好go环境。 ++ ++isulad当前推荐的runc验证版本为v1.0.0-rc5。 ++ ++runc可以使用以下两种安装方式： ++ ++1. 直接使用包管理器安装runc: ++ ++```sh ++# centOS ++sudo yum install runc ++# Ubuntu ++sudo apt-get install runc ++``` ++ ++2. 源码编译安装runc（注意建议切换成isulad推荐的runc版本：`git checkout v1.0.0-rc5`） ++ ++```sh ++# 在GOPATH/src下创建 'github.com/opencontainers' 文件夹 ++cd github.com/opencontainers ++git clone https://github.com/opencontainers/runc ++cd runc ++ ++make ++sudo make install ++``` ++ ++还可以使用go get安装到`GOPATH`路径下（需要在GOPATH/src下创建github.com父文件夹）： ++ ++```sh ++go get github.com/opencontainers/runc ++cd $GOPATH/src/github.com/opencontainers/runc ++make ++sudo make install ++``` ++ ++最终安装好的runc会在`/usr/local/sbin/runc`目录下。 ++ ++## 二、配置iSulad使用runc ++ ++### 配置文件配置 ++ ++1. 修改isulad的daemon.json，配置isulad默认使用的runtime。 ++ ++```sh ++$ vim /etc/isulad/daemon.json ++ ... ++ "default-runtime": "runc" ++ ... ++``` ++ ++2. 也可以在配置文件中配置runtimes，在其中指定使用的`path`(用于修改isulad使用的runc路径)以及`runtime-args`(对runtime所有命令配置的参数)。 ++ ++```sh ++"runtimes": { ++ "runc": { ++ "path": "/usr/local/sbin/runc", ++ "runtime-args": [ ++ ] ++ } ++ }, ++``` ++ ++之后使用root权限启动isulad服务，使修改后的配置生效即可： ++ ++```sh ++$ sudo isulad ++``` ++ ++### 单个容器配置 ++ ++使用`--runtime=runc`启动一个runtime为runc的容器。 ++ ++```sh ++isula run -tid -n test --runtime=runc busybox sh ++``` ++ ++## 三、K8s中配置pod的runtime为runc ++ ++如何与kubernetes集成请参考[k8s_integration](https://gitee.com/openeuler/iSulad/blob/master/docs/manual/k8s_integration_zh.md)。 ++ ++### 全局配置 ++ ++直接参照第二节中配置文件配置的方式修改isulad默认使用的runtime为runc，则后续使用k8s启动容器时会默认使用的runtime即为runc。 ++ ++### 使用RuntimeClass配置 ++ ++RuntimeClass 是K8s的一种内置集群资源，是一种容器运行时配置，用于运行pod中的容器。 ++ ++1. 在`/etc/isulad/daemon.json`中配置`isulad`： ++ ++ ```json ++ "runtimes": { ++ "runc-runtime": { ++ "path": "/usr/local/sbin/runc", ++ "runtime-args": [ ++ ] ++ } ++ }, ++ ``` ++ ++2. 定义 `runc-runtime.yaml`，例如创建一个`runc-runtime.yaml`内容如下：(注意handler需要与daemon.json中的名称一致) ++ ++ ```yamlapiVersion: v1 ++ apiVersion: node.k8s.io/v1beta1 ++ kind: RuntimeClass ++ metadata: ++ name: runc-runtime ++ handler: runc-runtime ++ ``` ++ ++ 之后运行`kubectl apply -f runc-runtime.yaml`命令在kubectl中让这个配置生效。 ++ ++3. 之后在创建pod时，可以在其定义的yaml文件中的`spec.runtimeClassName`中设置pod使用的runtime： ++ ++```yaml ++apiVersion: v1 ++kind: Pod ++metadata: ++ name: runc-pod-example ++spec: ++ runtimeClassName: runc-runtime ++ containers: ++ - name: runc-pod ++ image: busybox:latest ++ command: ["/bin/sh"] ++ args: ["-c", "sleep 1000"] ++``` ++ +-- +2.25.1 + diff --git a/0050-fix-isula_rt_ops_ut-bugs.patch b/0050-fix-isula_rt_ops_ut-bugs.patch new file mode 100644 index 0000000..6f96568 --- /dev/null +++ b/0050-fix-isula_rt_ops_ut-bugs.patch @@ -0,0 +1,28 @@ +From e2382f841c7bc3215793fdd8ce29132871281810 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Wed, 22 Mar 2023 08:02:51 +0800 +Subject: [PATCH 50/53] fix isula_rt_ops_ut bugs + +Signed-off-by: zhongtao +--- + test/runtime/isula/isula_rt_ops_ut.cc | 3 +++ + 1 file changed, 3 insertions(+) + +diff --git a/test/runtime/isula/isula_rt_ops_ut.cc b/test/runtime/isula/isula_rt_ops_ut.cc +index f37e62a0..9e014ac3 100644 +--- a/test/runtime/isula/isula_rt_ops_ut.cc ++++ b/test/runtime/isula/isula_rt_ops_ut.cc +@@ -211,7 +211,10 @@ TEST_F(IsulaRtOpsUnitTest, test_rt_isula_resources_stats) + TEST_F(IsulaRtOpsUnitTest, test_rt_isula_kill) + { + rt_kill_params_t kill_params = { ++ .signal = SIGTERM, ++ .stop_signal = SIGKILL, + .pid = -1, ++ .start_time = 12345, + }; + ASSERT_EQ(rt_isula_kill(nullptr, nullptr, nullptr), -1); + +-- +2.25.1 + diff --git a/0051-refactor-remote-ro-code.patch b/0051-refactor-remote-ro-code.patch new file mode 100644 index 0000000..c9269cb --- /dev/null +++ b/0051-refactor-remote-ro-code.patch @@ -0,0 +1,2089 @@ +From d4816441dc5ad67cc8cbab565e5ddc0eaf838611 Mon Sep 17 00:00:00 2001 +From: "Neil.wrz" +Date: Mon, 6 Mar 2023 17:43:17 -0800 +Subject: [PATCH 51/53] refactor remote ro code + +Signed-off-by: Neil.wrz +--- + CI/make-and-install.sh | 2 +- + .../oci/storage/image_store/CMakeLists.txt | 3 - + .../oci/storage/image_store/image_store.c | 37 +-- + .../oci/storage/image_store/image_store.h | 13 +- + .../oci/storage/layer_store/CMakeLists.txt | 3 - + .../graphdriver/overlay2/CMakeLists.txt | 3 - + .../graphdriver/overlay2/driver_overlay2.h | 8 - + .../oci/storage/layer_store/layer_store.c | 230 +++++++++--------- + .../oci/storage/layer_store/layer_store.h | 9 +- + .../image_remote_impl.c | 61 +++-- + .../layer_remote_impl.c | 71 +++--- + .../overlay_remote_impl.c | 103 +++++--- + .../remote_layer_support/remote_support.c | 141 ++++++----- + .../remote_layer_support/remote_support.h | 62 +++-- + .../ro_symlink_maintain.c | 124 +--------- + .../ro_symlink_maintain.h | 19 +- + .../modules/image/oci/storage/storage.c | 4 +- + src/utils/cutils/map/rb_tree.c | 1 + + test/image/oci/registry/CMakeLists.txt | 2 + + test/image/oci/storage/CMakeLists.txt | 3 + + test/image/oci/storage/images/CMakeLists.txt | 2 + + test/image/oci/storage/layers/CMakeLists.txt | 4 + + .../remote_layer_support/CMakeLists.txt | 44 ++++ + .../remote_layer_support/remote_layer_ut.cc | 93 +++++++ + test/image/oci/storage/rootfs/CMakeLists.txt | 2 + + test/mocks/remote_store_mock.cc | 68 ++++++ + test/mocks/remote_store_mock.h | 40 +++ + 27 files changed, 683 insertions(+), 469 deletions(-) + rename src/daemon/modules/image/oci/storage/{image_store => remote_layer_support}/image_remote_impl.c (71%) + rename src/daemon/modules/image/oci/storage/{layer_store => remote_layer_support}/layer_remote_impl.c (76%) + rename src/daemon/modules/image/oci/storage/{layer_store/graphdriver/overlay2 => remote_layer_support}/overlay_remote_impl.c (72%) + create mode 100644 test/image/oci/storage/remote_layer_support/CMakeLists.txt + create mode 100644 test/image/oci/storage/remote_layer_support/remote_layer_ut.cc + create mode 100644 test/mocks/remote_store_mock.cc + create mode 100644 test/mocks/remote_store_mock.h + +diff --git a/CI/make-and-install.sh b/CI/make-and-install.sh +index faeaf005..fa9c2250 100755 +--- a/CI/make-and-install.sh ++++ b/CI/make-and-install.sh +@@ -103,7 +103,7 @@ rm -rf build + mkdir build + cd build + if [[ ${enable_gcov} -ne 0 ]]; then +- cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_UT=ON -DENABLE_METRICS=ON .. ++ cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_UT=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON .. + else + cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DENABLE_EMBEDDED=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON .. + fi +diff --git a/src/daemon/modules/image/oci/storage/image_store/CMakeLists.txt b/src/daemon/modules/image/oci/storage/image_store/CMakeLists.txt +index 7d4fb77c..ecf21caa 100644 +--- a/src/daemon/modules/image/oci/storage/image_store/CMakeLists.txt ++++ b/src/daemon/modules/image/oci/storage/image_store/CMakeLists.txt +@@ -1,8 +1,5 @@ + # get current directory sources files + aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_image_store_srcs) +-IF (NOT ENABLE_REMOTE_LAYER_STORE) +-list(REMOVE_ITEM local_image_store_srcs "${CMAKE_CURRENT_SOURCE_DIR}/image_remote_impl.c") +-ENDIF() + + set(IMAGE_STORE_SRCS + ${local_image_store_srcs} +diff --git a/src/daemon/modules/image/oci/storage/image_store/image_store.c b/src/daemon/modules/image/oci/storage/image_store/image_store.c +index 84187ded..9a32b247 100644 +--- a/src/daemon/modules/image/oci/storage/image_store/image_store.c ++++ b/src/daemon/modules/image/oci/storage/image_store/image_store.c +@@ -132,7 +132,7 @@ static void free_image_store(image_store_t *store) + (void)map_free(store->bydigest); + store->bydigest = NULL; + +- linked_list_for_each_safe(item, &(store->images_list), next) { ++ linked_list_for_each_safe (item, &(store->images_list), next) { + linked_list_del(item); + image_ref_dec((image_t *)item->elem); + free(item); +@@ -164,7 +164,7 @@ static void image_store_digest_field_kvfree(void *key, void *value) + + free(key); + if (val != NULL) { +- linked_list_for_each_safe(item, &(val->images_list), next) { ++ linked_list_for_each_safe (item, &(val->images_list), next) { + linked_list_del(item); + free(item); + item = NULL; +@@ -500,7 +500,7 @@ static void digest_image_slice_without_value(digest_image_t *digest_filter_image + return; + } + +- linked_list_for_each_safe(item, &(digest_filter_images->images_list), next) { ++ linked_list_for_each_safe (item, &(digest_filter_images->images_list), next) { + tmp = (image_t *)item->elem; + if (strcmp(tmp->simage->id, img->simage->id) == 0) { + linked_list_del(item); +@@ -581,7 +581,7 @@ static int remove_image_from_memory(const char *id) + goto out; + } + +- linked_list_for_each_safe(item, &(g_image_store->images_list), next) { ++ linked_list_for_each_safe (item, &(g_image_store->images_list), next) { + image_t *tmp = (image_t *)item->elem; + if (strcmp(tmp->simage->id, id) != 0) { + continue; +@@ -680,7 +680,7 @@ static void free_digest_image(digest_image_t *ptr) + return; + } + +- linked_list_for_each_safe(item, &(ptr->images_list), next) { ++ linked_list_for_each_safe (item, &(ptr->images_list), next) { + linked_list_del(item); + free(item); + item = NULL; +@@ -2678,7 +2678,7 @@ int image_store_get_all_images(imagetool_images_list *images_list) + goto unlock; + } + +- linked_list_for_each_safe(item, &(g_image_store->images_list), next) { ++ linked_list_for_each_safe (item, &(g_image_store->images_list), next) { + imagetool_image_summary *imginfo = NULL; + image_t *img = (image_t *)item->elem; + imginfo = get_image_summary(img); +@@ -3099,7 +3099,7 @@ out: + return ret; + } + +-int validate_manifest_schema_version_1(const char *path, bool *valid) ++int image_store_validate_manifest_schema_version_1(const char *path, bool *valid) + { + int ret = 0; + int nret; +@@ -3506,7 +3506,7 @@ static int get_images_from_json() + continue; + } + +- if (validate_manifest_schema_version_1(image_path, &valid_v1_image) != 0) { ++ if (image_store_validate_manifest_schema_version_1(image_path, &valid_v1_image) != 0) { + ERROR("Failed to validate manifest schema version 1 format"); + continue; + } +@@ -3543,7 +3543,7 @@ static void image_store_check_all_images() + return; + } + +- linked_list_for_each_safe(item, &(g_image_store->images_list), next) { ++ linked_list_for_each_safe (item, &(g_image_store->images_list), next) { + image_t *img = (image_t *)item->elem; + if (img->spec == NULL) { + ERROR("Failed to check spec info of image: %s, try to delete", img->simage->id); +@@ -3657,18 +3657,23 @@ out: + } + + #ifdef ENABLE_REMOTE_LAYER_STORE +-int append_image_by_directory_with_lock(const char *id) ++int remote_append_image_by_directory_with_lock(const char *id) + { + int ret = 0; + int nret = 0; + char image_path[PATH_MAX] = { 0x00 }; + ++ if (id == NULL) { ++ ERROR("can't add NULL remote image"); ++ return -1; ++ } ++ + if (!image_store_lock(EXCLUSIVE)) { + ERROR("Failed to lock remote image store when handle: %s", id); + return -1; + } + +- if (map_search(g_image_store->byid, (void *)id) != NULL ) { ++ if (map_search(g_image_store->byid, (void *)id) != NULL) { + DEBUG("remote image already exist, not added: %s", id); + goto out; + } +@@ -3687,10 +3692,15 @@ out: + return ret; + } + +-int remove_image_from_memory_with_lock(const char *id) ++int remote_remove_image_from_memory_with_lock(const char *id) + { + int ret = 0; + ++ if (id == NULL) { ++ ERROR("can't remove NULL remote image"); ++ return -1; ++ } ++ + if (!image_store_lock(EXCLUSIVE)) { + ERROR("Failed to lock remote image store when handle: %s", id); + return -1; +@@ -3709,9 +3719,8 @@ out: + return ret; + } + +-char *get_top_layer_from_json(const char *img_id) ++char *remote_image_get_top_layer_from_json(const char *img_id) + { +- + char *ret = NULL; + int nret = 0; + char image_path[PATH_MAX] = { 0x00 }; +diff --git a/src/daemon/modules/image/oci/storage/image_store/image_store.h b/src/daemon/modules/image/oci/storage/image_store/image_store.h +index c3cb50e3..5164cc73 100644 +--- a/src/daemon/modules/image/oci/storage/image_store/image_store.h ++++ b/src/daemon/modules/image/oci/storage/image_store/image_store.h +@@ -28,9 +28,6 @@ + #include "isula_libutils/imagetool_image.h" + #include "isula_libutils/imagetool_images_list.h" + #include "isula_libutils/imagetool_image_summary.h" +-#ifdef ENABLE_REMOTE_LAYER_STORE +-#include "remote_support.h" +-#endif + + struct storage_module_init_options; + +@@ -112,11 +109,11 @@ void image_store_free(); + imagetool_image_summary *image_store_get_image_summary(const char *id); + + #ifdef ENABLE_REMOTE_LAYER_STORE +-remote_support *image_store_impl_remote_support(); +-int validate_manifest_schema_version_1(const char *path, bool *valid); +-int append_image_by_directory_with_lock(const char *image_dir); +-int remove_image_from_memory_with_lock(const char *id); +-char *get_top_layer_from_json(const char *img_id); /* return top layer id */ ++int image_store_validate_manifest_schema_version_1(const char *path, bool *valid); ++int remote_append_image_by_directory_with_lock(const char *image_dir); ++int remote_remove_image_from_memory_with_lock(const char *id); ++// return top layer id ++char *remote_image_get_top_layer_from_json(const char *img_id); + #endif + + #ifdef __cplusplus +diff --git a/src/daemon/modules/image/oci/storage/layer_store/CMakeLists.txt b/src/daemon/modules/image/oci/storage/layer_store/CMakeLists.txt +index e04b4ad7..c218a7c0 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/CMakeLists.txt ++++ b/src/daemon/modules/image/oci/storage/layer_store/CMakeLists.txt +@@ -1,8 +1,5 @@ + # get current directory sources files + aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_layer_store_srcs) +-IF (NOT ENABLE_REMOTE_LAYER_STORE) +-list(REMOVE_ITEM local_layer_store_srcs "${CMAKE_CURRENT_SOURCE_DIR}/layer_remote_impl.c") +-ENDIF() + add_subdirectory(graphdriver) + + set(LAYER_STORE_SRCS +diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/CMakeLists.txt b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/CMakeLists.txt +index dd4e82aa..ceed16b7 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/CMakeLists.txt ++++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/CMakeLists.txt +@@ -1,8 +1,5 @@ + # get current directory sources files + aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_overlay2_srcs) +-IF (NOT ENABLE_REMOTE_LAYER_STORE) +-list(REMOVE_ITEM local_overlay2_srcs "${CMAKE_CURRENT_SOURCE_DIR}/overlay_remote_impl.c") +-ENDIF() + + set(OVERLAY2_SRCS + ${local_overlay2_srcs} +diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.h b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.h +index 5c1d93fb..e14271b1 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.h ++++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.h +@@ -22,9 +22,6 @@ + #include + + #include "driver.h" +-#ifdef ENABLE_REMOTE_LAYER_STORE +-#include "remote_support.h" +-#endif + + struct driver_create_opts; + struct driver_mount_opts; +@@ -71,11 +68,6 @@ int overlay2_repair_lowers(const char *id, const char *parent, const struct grap + + int overlay2_get_layer_fs_info(const char *id, const struct graphdriver *driver, imagetool_fs_info *fs_info); + +-#ifdef ENABLE_REMOTE_LAYER_STORE +-remote_support *overlay_driver_impl_remote_support(void); +-bool overlay_remote_layer_valid(const char *layer_id); +-#endif +- + #ifdef __cplusplus + } + #endif +diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c +index 4edd0cad..79339757 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c ++++ b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c +@@ -1789,119 +1789,6 @@ out: + return ret; + } + +-static layer_t *load_one_layer_from_json(const char *id) +-{ +- int nret = 0; +- char *mount_point_path = NULL; +- char tmpdir[PATH_MAX] = { 0 }; +- char *rpath = NULL; +- layer_t *l = NULL; +- bool layer_valid = false; +- +- nret = snprintf(tmpdir, PATH_MAX, "%s/%s", g_root_dir, id); +- if (nret < 0 || nret >= PATH_MAX) { +- ERROR("Sprintf: %s failed", id); +- goto free_out; +- } +- +- mount_point_path = mountpoint_json_path(id); +- if (mount_point_path == NULL) { +- ERROR("Out of Memory"); +- goto free_out; +- } +- +- rpath = layer_json_path(id); +- if (rpath == NULL) { +- ERROR("%s is invalid layer", id); +- goto free_out; +- } +- +- l = load_layer(rpath, mount_point_path); +- if (l == NULL) { +- ERROR("load layer: %s failed, remove it", id); +- goto free_out; +- } +- +- if (do_validate_image_layer(tmpdir, l) != 0) { +- ERROR("%s is invalid image layer", id); +- goto free_out; +- } +- +- if (do_validate_rootfs_layer(l) != 0) { +- ERROR("%s is invalid rootfs layer", id); +- goto free_out; +- } +- +- layer_valid = true; +- +-free_out: +- free(rpath); +- free(mount_point_path); +- if (!layer_valid) { +- free_layer_t(l); +- l = NULL; +- } +- // always return true; +- // if load layer failed, just remove it +- return l; +-} +- +-int load_one_layer(const char *id) +-{ +- int ret = 0; +- layer_t *tl = NULL; +- int i = 0; +- +- if (!layer_store_lock(true)) { +- return -1; +- } +- +- if (map_search(g_metadata.by_id, (void *)id) != NULL) { +- DEBUG("remote layer already exist, not added: %s", id); +- goto unlock_out; +- } +- +- tl = load_one_layer_from_json(id); +- if (tl == NULL) { +- ret = -1; +- goto unlock_out; +- } +- +- if (!map_insert(g_metadata.by_id, (void *)tl->slayer->id, (void *)tl)) { +- ERROR("Insert id: %s for layer failed", tl->slayer->id); +- ret = -1; +- goto unlock_out; +- } +- +- for (; i < tl->slayer->names_len; i++) { +- // this should be done by master isulad +- // if (remove_name(tl->slayer->names[i])) { +- // should_save = true; +- // } +- if (!map_insert(g_metadata.by_name, (void *)tl->slayer->names[i], (void *)tl)) { +- ret = -1; +- ERROR("Insert name: %s for layer failed", tl->slayer->names[i]); +- goto unlock_out; +- } +- } +- ret = insert_digest_into_map(g_metadata.by_compress_digest, tl->slayer->compressed_diff_digest, tl->slayer->id); +- if (ret != 0) { +- ERROR("update layer: %s compress failed", tl->slayer->id); +- goto unlock_out; +- } +- +- ret = insert_digest_into_map(g_metadata.by_uncompress_digest, tl->slayer->diff_digest, tl->slayer->id); +- if (ret != 0) { +- ERROR("update layer: %s uncompress failed", tl->slayer->id); +- goto unlock_out; +- } +- +- ret = 0; +-unlock_out: +- layer_store_unlock(); +- return ret; +-} +- + static bool load_layer_json_cb(const char *path_name, const struct dirent *sub_dir, void *context) + { + #define LAYER_NAME_LEN 64 +@@ -2483,7 +2370,7 @@ container_inspect_graph_driver *layer_store_get_metadata_by_layer_id(const char + } + + #ifdef ENABLE_REMOTE_LAYER_STORE +-int remove_memory_stores_with_lock(const char *id) ++int remote_layer_remove_memory_stores_with_lock(const char *id) + { + int ret = 0; + +@@ -2496,6 +2383,11 @@ int remove_memory_stores_with_lock(const char *id) + goto unlock_out; + } + ++ if (map_search(g_metadata.by_id, (void *)id) == NULL) { ++ DEBUG("remote layer already removed, don't delete: %s", id); ++ goto unlock_out; ++ } ++ + ret = remove_memory_stores(id); + + unlock_out: +@@ -2503,4 +2395,114 @@ unlock_out: + + return ret; + } ++ ++static layer_t *load_one_layer_from_json(const char *id) ++{ ++ int nret = 0; ++ char *mount_point_path = NULL; ++ char tmpdir[PATH_MAX] = { 0 }; ++ char *rpath = NULL; ++ layer_t *l = NULL; ++ bool layer_valid = false; ++ ++ nret = snprintf(tmpdir, PATH_MAX, "%s/%s", g_root_dir, id); ++ if (nret < 0 || nret >= PATH_MAX) { ++ ERROR("Sprintf: %s failed", id); ++ goto free_out; ++ } ++ ++ mount_point_path = mountpoint_json_path(id); ++ if (mount_point_path == NULL) { ++ ERROR("Out of Memory"); ++ goto free_out; ++ } ++ ++ rpath = layer_json_path(id); ++ if (rpath == NULL) { ++ ERROR("%s is invalid layer", id); ++ goto free_out; ++ } ++ ++ l = load_layer(rpath, mount_point_path); ++ if (l == NULL) { ++ ERROR("load layer: %s failed, remove it", id); ++ goto free_out; ++ } ++ ++ if (do_validate_image_layer(tmpdir, l) != 0) { ++ ERROR("%s is invalid image layer", id); ++ goto free_out; ++ } ++ ++ if (do_validate_rootfs_layer(l) != 0) { ++ ERROR("%s is invalid rootfs layer", id); ++ goto free_out; ++ } ++ ++ layer_valid = true; ++ ++free_out: ++ free(rpath); ++ free(mount_point_path); ++ if (!layer_valid) { ++ free_layer_t(l); ++ l = NULL; ++ } ++ // always return true; ++ // if load layer failed, just remove it ++ return l; ++} ++ ++int remote_load_one_layer(const char *id) ++{ ++ int ret = 0; ++ layer_t *tl = NULL; ++ int i = 0; ++ ++ if (!layer_store_lock(true)) { ++ return -1; ++ } ++ ++ if (map_search(g_metadata.by_id, (void *)id) != NULL) { ++ DEBUG("remote layer already exist, not added: %s", id); ++ goto unlock_out; ++ } ++ ++ tl = load_one_layer_from_json(id); ++ if (tl == NULL) { ++ ret = -1; ++ goto unlock_out; ++ } ++ ++ if (!map_insert(g_metadata.by_id, (void *)tl->slayer->id, (void *)tl)) { ++ ERROR("Insert id: %s for layer failed", tl->slayer->id); ++ ret = -1; ++ goto unlock_out; ++ } ++ ++ for (; i < tl->slayer->names_len; i++) { ++ // this should be done by master isulad ++ if (!map_insert(g_metadata.by_name, (void *)tl->slayer->names[i], (void *)tl)) { ++ ret = -1; ++ ERROR("Insert name: %s for layer failed", tl->slayer->names[i]); ++ goto unlock_out; ++ } ++ } ++ ret = insert_digest_into_map(g_metadata.by_compress_digest, tl->slayer->compressed_diff_digest, tl->slayer->id); ++ if (ret != 0) { ++ ERROR("update layer: %s compress failed", tl->slayer->id); ++ goto unlock_out; ++ } ++ ++ ret = insert_digest_into_map(g_metadata.by_uncompress_digest, tl->slayer->diff_digest, tl->slayer->id); ++ if (ret != 0) { ++ ERROR("update layer: %s uncompress failed", tl->slayer->id); ++ goto unlock_out; ++ } ++ ++ ret = 0; ++unlock_out: ++ layer_store_unlock(); ++ return ret; ++} + #endif +diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_store.h b/src/daemon/modules/image/oci/storage/layer_store/layer_store.h +index 44bd297e..4677e5ee 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/layer_store.h ++++ b/src/daemon/modules/image/oci/storage/layer_store/layer_store.h +@@ -24,9 +24,6 @@ + #include "storage.h" + #include "io_wrapper.h" + #include "map.h" +-#ifdef ENABLE_REMOTE_LAYER_STORE +-#include "remote_support.h" +-#endif + + struct io_read_wrapper; + struct layer_list; +@@ -84,10 +81,8 @@ int layer_store_check(const char *id); + container_inspect_graph_driver *layer_store_get_metadata_by_layer_id(const char *id); + + #ifdef ENABLE_REMOTE_LAYER_STORE +-remote_support *layer_store_impl_remote_support(); +-bool layer_remote_layer_valid(const char *layer_id); +-int load_one_layer(const char *id); +-int remove_memory_stores_with_lock(const char *id); ++int remote_load_one_layer(const char *id); ++int remote_layer_remove_memory_stores_with_lock(const char *id); + #endif + + #ifdef __cplusplus +diff --git a/src/daemon/modules/image/oci/storage/image_store/image_remote_impl.c b/src/daemon/modules/image/oci/storage/remote_layer_support/image_remote_impl.c +similarity index 71% +rename from src/daemon/modules/image/oci/storage/image_store/image_remote_impl.c +rename to src/daemon/modules/image/oci/storage/remote_layer_support/image_remote_impl.c +index 20da8116..a822ea81 100644 +--- a/src/daemon/modules/image/oci/storage/image_store/image_remote_impl.c ++++ b/src/daemon/modules/image/oci/storage/remote_layer_support/image_remote_impl.c +@@ -13,27 +13,23 @@ + * Description: provide remote image store functions + ******************************************************************************/ + #define _GNU_SOURCE +-#include "image_store.h" ++#include "remote_support.h" + + #include + #include + +-#include "remote_support.h" + #include "ro_symlink_maintain.h" + #include "map.h" + #include "utils_file.h" + #include "utils.h" + #include "layer_store.h" ++#include "image_store.h" + #include "utils_array.h" + +-struct remote_image_data { +- const char *image_home; +-}; +- + static map_t *image_byid_old = NULL; + static map_t *image_byid_new = NULL; + +-static void *remote_support_create(const char *remote_home, const char *remote_ro) ++struct remote_image_data *remote_image_create(const char *remote_home, const char *remote_ro) + { + struct remote_image_data *data = util_common_calloc_s(sizeof(struct remote_image_data)); + if (data == NULL) { +@@ -46,7 +42,7 @@ static void *remote_support_create(const char *remote_home, const char *remote_r + return data; + } + +-static void remote_support_destroy(void *data) ++void remote_image_destroy(struct remote_image_data *data) + { + if (data == NULL) { + return; +@@ -59,7 +55,7 @@ static void remote_support_destroy(void *data) + return; + } + +-static int remote_support_scan(void *data) ++static int remote_dir_scan(void *data) + { + int ret = 0; + int nret; +@@ -79,7 +75,7 @@ static int remote_support_scan(void *data) + image_dirs_num = util_array_len((const char **)image_dirs); + + for (i = 0; i < image_dirs_num; i++) { +- bool valid_v1_image = false; ++ bool is_v1_image = false; + + if (util_reg_match(id_patten, image_dirs[i]) != 0) { + DEBUG("Image's json is placed inside image's data directory, so skip any other file or directory: %s", +@@ -93,12 +89,14 @@ static int remote_support_scan(void *data) + continue; + } + +- if (validate_manifest_schema_version_1(image_path, &valid_v1_image) != 0) { ++ if (image_store_validate_manifest_schema_version_1(image_path, &is_v1_image) != 0) { + ERROR("Failed to validate manifest schema version 1 format"); + continue; + } + +- if (!valid_v1_image) { ++ // for refresh, we don't care v1 image, cause image should be handled by master isulad ++ // when master isulad pull images ++ if (!is_v1_image) { + map_insert(image_byid_new, util_strdup_s(image_dirs[i]), (void *)&exist); + } + } +@@ -108,12 +106,13 @@ out: + return ret; + } + +-static int remote_support_add(void *data) ++static int remote_image_add(void *data) + { + char **array_added = NULL; + char **array_deleted = NULL; + char *top_layer = NULL; + map_t *tmp_map = NULL; ++ bool exist = true; + int i = 0; + int ret = 0; + +@@ -121,26 +120,28 @@ static int remote_support_add(void *data) + return -1; + } + +- array_added = added_layers(image_byid_old, image_byid_new); +- array_deleted = deleted_layers(image_byid_old, image_byid_new); ++ array_added = remote_added_layers(image_byid_old, image_byid_new); ++ array_deleted = remote_deleted_layers(image_byid_old, image_byid_new); + + for (i = 0; i < util_array_len((const char **)array_added); i++) { +- top_layer = get_top_layer_from_json(array_added[i]); +- if (top_layer != NULL && !layer_remote_layer_valid(top_layer)) { +- ERROR("ERROR not find valid under layer, remoet image:%s not added", array_added[i]); ++ top_layer = remote_image_get_top_layer_from_json(array_added[i]); ++ if (top_layer != NULL && !remote_layer_layer_valid(top_layer)) { ++ WARN("Current not find valid under layer, remoet image:%s not added", array_added[i]); + map_remove(image_byid_new, (void *)array_added[i]); + continue; + } + +- if (append_image_by_directory_with_lock(array_added[i]) != 0) { ++ if (remote_append_image_by_directory_with_lock(array_added[i]) != 0) { + ERROR("Failed to load image into memrory: %s", array_added[i]); ++ map_remove(image_byid_new, (void *)array_added[i]); + ret = -1; + } + } + + for (i = 0; i < util_array_len((const char **)array_deleted); i++) { +- if (remove_image_from_memory_with_lock(array_deleted[i]) != 0) { ++ if (remote_remove_image_from_memory_with_lock(array_deleted[i]) != 0) { + ERROR("Failed to remove remote memory store"); ++ map_insert(image_byid_new, array_deleted[i], (void *)&exist); + ret = -1; + } + } +@@ -148,7 +149,7 @@ static int remote_support_add(void *data) + tmp_map = image_byid_old; + image_byid_old = image_byid_new; + image_byid_new = tmp_map; +- empty_map(image_byid_new); ++ map_clear(image_byid_new); + + util_free_array(array_added); + util_free_array(array_deleted); +@@ -157,17 +158,13 @@ static int remote_support_add(void *data) + return ret; + } + +-remote_support *image_store_impl_remote_support(void) +-{ +- remote_support *rs = util_common_calloc_s(sizeof(remote_support)); +- if (rs == NULL) { +- return NULL; ++void remote_image_refresh(struct remote_image_data *data) { ++ if (remote_dir_scan(data) != 0) { ++ ERROR("remote overlay failed to scan dir, skip refresh"); ++ return; + } + +- rs->create = remote_support_create; +- rs->destroy = remote_support_destroy; +- rs->scan_remote_dir = remote_support_scan; +- rs->load_item = remote_support_add; +- +- return rs; ++ if (remote_image_add(data) != 0) { ++ ERROR("refresh overlay failed"); ++ } + } +diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c b/src/daemon/modules/image/oci/storage/remote_layer_support/layer_remote_impl.c +similarity index 76% +rename from src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c +rename to src/daemon/modules/image/oci/storage/remote_layer_support/layer_remote_impl.c +index d676458c..3e3afff6 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c ++++ b/src/daemon/modules/image/oci/storage/remote_layer_support/layer_remote_impl.c +@@ -13,7 +13,7 @@ + * Description: remote layer store implementation + ******************************************************************************/ + #define _GNU_SOURCE +-#include "layer_store.h" ++#include "remote_support.h" + + #include + #include +@@ -21,20 +21,15 @@ + + #include "map.h" + #include "utils.h" +-#include "remote_support.h" + #include "ro_symlink_maintain.h" ++#include "layer_store.h" + #include "path.h" + #include "driver_overlay2.h" + +-struct remote_layer_data { +- const char *layer_home; +- const char *layer_ro; +-}; +- + static map_t *layer_byid_old = NULL; + static map_t *layer_byid_new = NULL; + +-static void *remote_support_create(const char *layer_home, const char *layer_ro) ++struct remote_layer_data *remote_layer_create(const char *layer_home, const char *layer_ro) + { + struct remote_layer_data *data = util_common_calloc_s(sizeof(struct remote_layer_data)); + if (data == NULL) { +@@ -49,7 +44,7 @@ static void *remote_support_create(const char *layer_home, const char *layer_ro) + return data; + }; + +-static void remote_support_destroy(void *data) ++void remote_layer_destroy(struct remote_layer_data *data) + { + if (data == NULL) { + return; +@@ -72,10 +67,9 @@ static bool layer_walk_dir_cb(const char *path_name, const struct dirent *sub_di + return true; + } + +-static int remote_support_scan(void *data) ++static int remote_dir_scan(struct remote_layer_data *data) + { +- struct remote_layer_data *remote_data = data; +- return util_scan_subdirs(remote_data->layer_ro, layer_walk_dir_cb, data); ++ return util_scan_subdirs(data->layer_ro, layer_walk_dir_cb, data); + } + + static int remove_one_remote_layer(struct remote_layer_data *data, char *layer_id) +@@ -85,6 +79,11 @@ static int remove_one_remote_layer(struct remote_layer_data *data, char *layer_i + int nret = 0; + int ret = 0; + ++ if (layer_id == NULL) { ++ ERROR("can't delete NULL remote layer"); ++ return -1; ++ } ++ + nret = asprintf(&ro_symlink, "%s/%s", data->layer_home, layer_id); + if (nret < 0 || nret > PATH_MAX) { + SYSERROR("Create layer symbol link path failed"); +@@ -98,11 +97,14 @@ static int remove_one_remote_layer(struct remote_layer_data *data, char *layer_i + goto out; + } + ++ // return 0 if path already removed + if (util_path_remove(clean_path) != 0) { + SYSERROR("Failed to remove link path %s", clean_path); ++ ret = -1; ++ goto out; + } + +- if (remove_memory_stores_with_lock(layer_id) != 0) { ++ if (remote_layer_remove_memory_stores_with_lock(layer_id) != 0) { + ERROR("Failed to remove remote layer store memory"); + ret = -1; + } +@@ -110,7 +112,6 @@ static int remove_one_remote_layer(struct remote_layer_data *data, char *layer_i + out: + free(ro_symlink); + return ret; +- + } + + static int add_one_remote_layer(struct remote_layer_data *data, char *layer_id) +@@ -119,6 +120,11 @@ static int add_one_remote_layer(struct remote_layer_data *data, char *layer_id) + char *layer_dir = NULL; + int ret = 0; + ++ if (layer_id == NULL) { ++ ERROR("can't add NULL remote layer"); ++ return -1; ++ } ++ + ro_symlink = util_path_join(data->layer_home, layer_id); + layer_dir = util_path_join(data->layer_ro, layer_id); + +@@ -140,7 +146,7 @@ static int add_one_remote_layer(struct remote_layer_data *data, char *layer_id) + goto free_out; + } + // insert layer into memory +- if (load_one_layer(layer_id) != 0) { ++ if (remote_load_one_layer(layer_id) != 0) { + ERROR("Failed to load new layer: %s into memory", layer_id); + ret = -1; + } +@@ -152,30 +158,32 @@ free_out: + return ret; + } + +-static int remote_support_add(void *data) ++static int remote_layer_add(struct remote_layer_data *data) + { + int ret = 0; + char **array_added = NULL; + char **array_deleted = NULL; + map_t *tmp_map = NULL; ++ bool exist = true; + int i = 0; + + if (data == NULL) { + return -1; + } + +- array_added = added_layers(layer_byid_old, layer_byid_new); +- array_deleted = deleted_layers(layer_byid_old, layer_byid_new); ++ array_added = remote_added_layers(layer_byid_old, layer_byid_new); ++ array_deleted = remote_deleted_layers(layer_byid_old, layer_byid_new); + + for (i = 0; i < util_array_len((const char **)array_added); i++) { +- if (!overlay_remote_layer_valid(array_added[i]) != 0) { ++ if (!remote_overlay_layer_valid(array_added[i]) != 0) { ++ WARN("remote overlay layer current not valid: %s", array_added[i]); + map_remove(layer_byid_new, (void *)array_added[i]); +- ERROR("remote overlay layer current not valid: %s", array_added[i]); + continue; + } + + if (add_one_remote_layer(data, array_added[i]) != 0) { + ERROR("Failed to add remote layer: %s", array_added[i]); ++ map_remove(layer_byid_new, (void *)array_added[i]); + ret = -1; + } + } +@@ -183,6 +191,7 @@ static int remote_support_add(void *data) + for (i = 0; i < util_array_len((const char **)array_deleted); i++) { + if (remove_one_remote_layer(data, array_deleted[i]) != 0) { + ERROR("Failed to delete remote overlay layer: %s", array_deleted[i]); ++ map_insert(layer_byid_new, array_deleted[i], (void *)&exist); + ret = -1; + } + } +@@ -190,7 +199,7 @@ static int remote_support_add(void *data) + tmp_map = layer_byid_old; + layer_byid_old = layer_byid_new; + layer_byid_new = tmp_map; +- empty_map(layer_byid_new); ++ map_clear(layer_byid_new); + + util_free_array(array_added); + util_free_array(array_deleted); +@@ -198,22 +207,20 @@ static int remote_support_add(void *data) + return ret; + } + +-remote_support *layer_store_impl_remote_support() ++void remote_layer_refresh(struct remote_layer_data *data) + { +- remote_support *rs = util_common_calloc_s(sizeof(remote_support)); +- if (rs == NULL) { +- return NULL; ++ if (remote_dir_scan(data) != 0) { ++ ERROR("remote layer failed to scan dir, skip refresh"); ++ return; + } + +- rs->create = remote_support_create; +- rs->destroy = remote_support_destroy; +- rs->scan_remote_dir = remote_support_scan; +- rs->load_item = remote_support_add; +- +- return rs; ++ if (remote_layer_add(data) != 0) { ++ ERROR("refresh overlay failed"); ++ } + } + +-bool layer_remote_layer_valid(const char *layer_id) ++ ++bool remote_layer_layer_valid(const char *layer_id) + { + return map_search(layer_byid_old, (void *)layer_id) != NULL; + } +diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/overlay_remote_impl.c b/src/daemon/modules/image/oci/storage/remote_layer_support/overlay_remote_impl.c +similarity index 72% +rename from src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/overlay_remote_impl.c +rename to src/daemon/modules/image/oci/storage/remote_layer_support/overlay_remote_impl.c +index a674a00f..de2e583c 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/overlay_remote_impl.c ++++ b/src/daemon/modules/image/oci/storage/remote_layer_support/overlay_remote_impl.c +@@ -13,13 +13,13 @@ + * Description: provide remote implementation for driver overlay + ******************************************************************************/ + #define _GNU_SOURCE +-#include "driver_overlay2.h" ++#include "remote_support.h" + + #include + + #include "map.h" +-#include "remote_support.h" + #include "ro_symlink_maintain.h" ++#include "driver_overlay2.h" + #include "isula_libutils/log.h" + #include "utils.h" + #include "utils_array.h" +@@ -29,15 +29,13 @@ + #define OVERLAY_LINK_DIR "l" + #define OVERLAY_LAYER_LINK "link" + +-struct remote_overlay_data { +- const char *overlay_home; +- const char *overlay_ro; +-}; +- ++// key: id, value: short id in 'l' dir ++// store short id to delete symbol link in 'l' dir + static map_t *overlay_byid_old = NULL; + static map_t *overlay_byid_new = NULL; ++static map_t *overlay_id_link = NULL; + +-static void *remote_support_create(const char *remote_home, const char *remote_ro) ++struct remote_overlay_data *remote_overlay_create(const char *remote_home, const char *remote_ro) + { + struct remote_overlay_data *data = util_common_calloc_s(sizeof(struct remote_overlay_data)); + if (data == NULL) { +@@ -48,11 +46,12 @@ static void *remote_support_create(const char *remote_home, const char *remote_r + data->overlay_ro = remote_ro; + overlay_byid_old = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); + overlay_byid_new = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); ++ overlay_id_link = map_new(MAP_STR_STR, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); + + return data; + } + +-static void remote_support_destroy(void *data) ++void remote_overlay_destroy(struct remote_overlay_data *data) + { + if (data == NULL) { + return; +@@ -60,6 +59,7 @@ static void remote_support_destroy(void *data) + + map_free(overlay_byid_old); + map_free(overlay_byid_new); ++ map_free(overlay_id_link); + free(data); + } + +@@ -74,10 +74,9 @@ static bool overlay_walk_dir_cb(const char *path_name, const struct dirent *sub_ + return true; + } + +-static int remote_support_scan(void *data) ++static int remote_dir_scan(struct remote_overlay_data *data) + { +- struct remote_overlay_data *remote_data = data; +- return util_scan_subdirs(remote_data->overlay_ro, overlay_walk_dir_cb, data); ++ return util_scan_subdirs(data->overlay_ro, overlay_walk_dir_cb, data); + } + + static int do_diff_symlink(const char *id, char *link_id, const char *driver_home) +@@ -128,10 +127,17 @@ out: + static int remove_one_remote_overlay_layer(struct remote_overlay_data *data, const char *overlay_id) + { + char *ro_symlink = NULL; ++ char *link_path = NULL; ++ char *link_id = NULL; + char clean_path[PATH_MAX] = { 0 }; + int nret = 0; + int ret = 0; + ++ if (overlay_id == NULL) { ++ ERROR("can't remove NULL remote layer"); ++ return -1; ++ } ++ + nret = asprintf(&ro_symlink, "%s/%s", data->overlay_home, overlay_id); + if (nret < 0 || nret > PATH_MAX) { + SYSERROR("Create layer symbol link path failed"); +@@ -149,8 +155,40 @@ static int remove_one_remote_overlay_layer(struct remote_overlay_data *data, con + SYSERROR("Failed to remove link path %s", clean_path); + } + ++ link_id = (char *)map_search(overlay_id_link, (void *)overlay_id); ++ ++ if (link_id == NULL) { ++ ERROR("Failed to find link id for overlay layer: %s", overlay_id); ++ ret = -1; ++ goto out; ++ } ++ ++ nret = asprintf(&link_path, "%s/%s/%s", data->overlay_home, OVERLAY_LINK_DIR, link_id); ++ if (nret < 0 || nret > PATH_MAX) { ++ SYSERROR("Create link path failed"); ++ ret = -1; ++ goto out; ++ } ++ ++ if (util_clean_path(link_path, clean_path, sizeof(clean_path)) == NULL) { ++ ERROR("Failed to clean path: %s", ro_symlink); ++ ret = -1; ++ goto out; ++ } ++ ++ if (util_path_remove(clean_path) != 0) { ++ SYSERROR("Failed to remove link path %s", clean_path); ++ } ++ ++ if (!map_remove(overlay_id_link, (void *)overlay_id)) { ++ ERROR("Failed to remove link id for overlay layers: %s", overlay_id); ++ ret = -1; ++ goto out; ++ } ++ + out: + free(ro_symlink); ++ free(link_path); + return ret; + } + +@@ -162,6 +200,11 @@ static int add_one_remote_overlay_layer(struct remote_overlay_data *data, const + char *diff_symlink = NULL; + int ret = 0; + ++ if (overlay_id == NULL) { ++ ERROR("can't add NULL remote layer"); ++ return -1; ++ } ++ + ro_symlink = util_path_join(data->overlay_home, overlay_id); + if (ro_symlink == NULL) { + ERROR("Failed to join ro symlink path: %s", overlay_id); +@@ -211,6 +254,11 @@ static int add_one_remote_overlay_layer(struct remote_overlay_data *data, const + ret = -1; + } + ++ if (!map_insert(overlay_id_link, util_strdup_s(overlay_id), (void *)diff_symlink)) { ++ ERROR("can't insert remote layer into map"); ++ ret = -1; ++ } ++ + free_out: + free(ro_symlink); + free(layer_dir); +@@ -220,24 +268,26 @@ free_out: + return ret; + } + +-static int remote_support_add(void *data) ++static int remote_image_add(struct remote_overlay_data *data) + { + int ret = 0; + char **array_added = NULL; + char **array_deleted = NULL; + map_t *tmp_map = NULL; ++ bool exist = true; + int i = 0; + + if (data == NULL) { + return -1; + } + +- array_added = added_layers(overlay_byid_old, overlay_byid_new); +- array_deleted = deleted_layers(overlay_byid_old, overlay_byid_new); ++ array_added = remote_added_layers(overlay_byid_old, overlay_byid_new); ++ array_deleted = remote_deleted_layers(overlay_byid_old, overlay_byid_new); + + for (i = 0; i < util_array_len((const char **)array_added); i++) { + if (add_one_remote_overlay_layer(data, array_added[i]) != 0) { + ERROR("Failed to add remote overlay layer: %s", array_added[i]); ++ map_remove(overlay_byid_new, (void *)array_added[i]); + ret = -1; + } + } +@@ -245,6 +295,7 @@ static int remote_support_add(void *data) + for (i = 0; i < util_array_len((const char **)array_deleted); i++) { + if (remove_one_remote_overlay_layer(data, array_deleted[i]) != 0) { + ERROR("Failed to delete remote overlay layer: %s", array_deleted[i]); ++ map_insert(overlay_byid_new, array_deleted[i], (void *)&exist); + ret = -1; + } + } +@@ -252,7 +303,7 @@ static int remote_support_add(void *data) + tmp_map = overlay_byid_old; + overlay_byid_old = overlay_byid_new; + overlay_byid_new = tmp_map; +- empty_map(overlay_byid_new); ++ map_clear(overlay_byid_new); + + util_free_array(array_added); + util_free_array(array_deleted); +@@ -260,23 +311,19 @@ static int remote_support_add(void *data) + return ret; + } + +-remote_support *overlay_driver_impl_remote_support(void) ++void remote_overlay_refresh(struct remote_overlay_data *data) + { +- remote_support *rs = util_common_calloc_s(sizeof(remote_support)); +- if (rs == NULL) { +- ERROR("Failed to calloc overlay supporter"); +- return NULL; ++ if (remote_dir_scan(data) != 0) { ++ ERROR("remote overlay failed to scan dir, skip refresh"); ++ return; + } + +- rs->create = remote_support_create; +- rs->destroy = remote_support_destroy; +- rs->scan_remote_dir = remote_support_scan; +- rs->load_item = remote_support_add; +- +- return rs; ++ if (remote_image_add(data) != 0) { ++ ERROR("refresh overlay failed"); ++ } + } + +-bool overlay_remote_layer_valid(const char *layer_id) ++bool remote_overlay_layer_valid(const char *layer_id) + { + return map_search(overlay_byid_old, (void *)layer_id) != NULL; + } +diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c +index 9dc096f7..3c7d0f54 100644 +--- a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c ++++ b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c +@@ -15,108 +15,107 @@ + + #include "remote_support.h" + +-#include "layer_store.h" +-#include "image_store.h" ++#include ++ + #include "isula_libutils/log.h" +-#include "driver_overlay2.h" + #include "utils.h" + +-remote_supporter *create_layer_supporter(const char *remote_home, const char *remote_ro) +-{ +- remote_support *handlers = layer_store_impl_remote_support(); +- if (handlers == NULL || handlers->create == NULL) { +- return NULL; +- } ++struct supporters { ++ struct remote_image_data *image_data; ++ struct remote_layer_data *layer_data; ++ struct remote_overlay_data *overlay_data; ++}; + +- remote_supporter *supporter = (remote_supporter *)util_common_calloc_s(sizeof(remote_supporter)); +- if (supporter == NULL) { +- goto err_out; +- } ++static struct supporters supporters; ++ ++static void *remote_refresh_ro_symbol_link(void *arg) ++{ ++ struct supporters *refresh_supporters = (struct supporters *)arg; ++ prctl(PR_SET_NAME, "RoLayerRefresh"); + +- supporter->handlers = handlers; +- supporter->data = handlers->create(remote_home, remote_ro); ++ while (true) { ++ util_usleep_nointerupt(5 * 1000 * 1000); ++ DEBUG("remote refresh start\n"); + +- return supporter; ++ remote_overlay_refresh(refresh_supporters->overlay_data); ++ remote_layer_refresh(refresh_supporters->layer_data); ++ remote_image_refresh(refresh_supporters->image_data); + +-err_out: +- free(handlers); +- free(supporter); ++ DEBUG("remote refresh end\n"); ++ } + return NULL; + } + +-remote_supporter *create_image_supporter(const char *remote_home, const char *remote_ro) ++int remote_start_refresh_thread(void) + { +- remote_support *handlers = image_store_impl_remote_support(); +- if (handlers == NULL || handlers->create == NULL) { +- return NULL; +- } ++ int res = 0; ++ pthread_t a_thread; ++ maintain_context ctx = get_maintain_context(); + +- remote_supporter *supporter = (remote_supporter *)util_common_calloc_s(sizeof(remote_supporter)); +- if (supporter == NULL) { +- goto err_out; ++ supporters.image_data = remote_image_create(ctx.image_home, NULL); ++ if (supporters.image_data == NULL) { ++ goto free_out; + } + +- supporter->handlers = handlers; +- supporter->data = handlers->create(remote_home, remote_ro); +- +- return supporter; +- +-err_out: +- free(handlers); +- free(supporter); +- return NULL; +-} ++ supporters.layer_data = remote_layer_create(ctx.layer_home, ctx.layer_ro_dir); ++ if (supporters.layer_data == NULL) { ++ goto free_out; ++ } + +-remote_supporter *create_overlay_supporter(const char *remote_home, const char *remote_ro) +-{ +- remote_support *handlers = overlay_driver_impl_remote_support(); +- if (handlers == NULL || handlers->create == NULL) { +- return NULL; ++ supporters.overlay_data = remote_overlay_create(ctx.overlay_home, ctx.overlay_ro_dir); ++ if (supporters.overlay_data == NULL) { ++ goto free_out; + } + +- remote_supporter *supporter = (remote_supporter *)util_common_calloc_s(sizeof(remote_supporter)); +- if (supporter == NULL) { +- goto err_out; ++ res = pthread_create(&a_thread, NULL, remote_refresh_ro_symbol_link, (void *)&supporters); ++ if (res != 0) { ++ CRIT("Thread creation failed"); ++ return -1; + } + +- supporter->handlers = handlers; +- supporter->data = handlers->create(remote_home, remote_ro); ++ if (pthread_detach(a_thread) != 0) { ++ SYSERROR("Failed to detach 0x%lx", a_thread); ++ return -1; ++ } + +- return supporter; ++ return 0; + +-err_out: +- free(handlers); +- free(supporter); +- return NULL; ++free_out: ++ remote_image_destroy(supporters.image_data); ++ remote_layer_destroy(supporters.layer_data); ++ remote_overlay_destroy(supporters.overlay_data); + ++ return -1; + } + +-void destroy_suppoter(remote_supporter *supporter) ++// this function calculate map_a - map_b => diff_list ++// diff_list contains keys inside map_a but not inside map_b ++static char **map_diff(const map_t *map_a, const map_t *map_b) + { +- if (supporter->handlers->destroy == NULL) { +- ERROR("destroy_supporter operation not supported"); +- return; ++ char **array = NULL; ++ map_itor *itor = map_itor_new(map_a); ++ bool *found = NULL; ++ ++ // iter new_map, every item not in old, append them to new_layers ++ for (; map_itor_valid(itor); map_itor_next(itor)) { ++ char *id = map_itor_key(itor); ++ found = map_search(map_b, id); ++ if (found == NULL) { ++ util_array_append(&array, util_strdup_s(id)); ++ } + } + +- supporter->handlers->destroy(supporter->data); +- free(supporter->handlers); +- free(supporter); ++ map_itor_free(itor); ++ ++ return array; + } + +-int scan_remote_dir(remote_supporter *supporter) ++char **remote_deleted_layers(const map_t *old, const map_t *new) + { +- if (supporter->handlers->scan_remote_dir == NULL) { +- ERROR("scan_remote_dir operation not supported"); +- return -1; +- } +- return supporter->handlers->scan_remote_dir(supporter->data); ++ return map_diff(old, new); + } + +-int load_item(remote_supporter *supporter) ++char **remote_added_layers(const map_t *old, const map_t *new) + { +- if (supporter->handlers->scan_remote_dir == NULL) { +- ERROR("load_item operation not supported"); +- return -1; +- } +- return supporter->handlers->load_item(supporter->data); ++ return map_diff(new, old); + } +diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h +index d1f7af35..892a9155 100644 +--- a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h ++++ b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h +@@ -17,39 +17,59 @@ + #define DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_REMOTE_SUPPORT_H + + #include "linked_list.h" +-#define REMOTE_RO_LAYER_DIR "RO" +-#define OVERLAY_RO_DIR "RO" ++#include "map.h" ++#include "ro_symlink_maintain.h" + + #ifdef __cplusplus + extern "C" { + #endif + +-typedef struct { +- void *(*create)(const char *remote_home, const char *remote_ro); +- void (*destroy)(void *data); +- // populate the list contains all dirs +- int (*scan_remote_dir)(void *data); +- // consume the list contains all dirs +- int (*load_item)(void *data); +-} remote_support; ++struct remote_overlay_data { ++ const char *overlay_home; ++ const char *overlay_ro; ++}; + +-typedef struct { +- void *data; +- remote_support *handlers; +-} remote_supporter; ++struct remote_layer_data { ++ const char *layer_home; ++ const char *layer_ro; ++}; + +-// RemoteSupport *impl_remote_support(); +-remote_supporter *create_image_supporter(const char *remote_home, const char *remote_ro); ++struct remote_image_data { ++ const char *image_home; ++}; + +-remote_supporter *create_layer_supporter(const char *remote_home, const char *remote_ro); ++// image impl ++struct remote_image_data *remote_image_create(const char *image_home, const char *image_ro); + +-remote_supporter *create_overlay_supporter(const char *remote_home, const char *remote_ro); ++void remote_image_destroy(struct remote_image_data *data); + +-void destroy_suppoter(remote_supporter *supporter); ++void remote_image_refresh(struct remote_image_data *data); + +-int scan_remote_dir(remote_supporter *supporter); ++// layer impl ++struct remote_layer_data *remote_layer_create(const char *layer_home, const char *layer_ro); + +-int load_item(remote_supporter *supporter); ++void remote_layer_destroy(struct remote_layer_data *data); ++ ++void remote_layer_refresh(struct remote_layer_data *data); ++ ++bool remote_layer_layer_valid(const char *layer_id); ++ ++// overlay impl ++struct remote_overlay_data *remote_overlay_create(const char *overlay_home, const char *overlay_ro); ++ ++void remote_overlay_destroy(struct remote_overlay_data *data); ++ ++void remote_overlay_refresh(struct remote_overlay_data *data); ++ ++bool remote_overlay_layer_valid(const char *layer_id); ++ ++// start refresh remote ++int remote_start_refresh_thread(void); ++ ++// extra map utils ++char **remote_deleted_layers(const map_t *old, const map_t *new_l); ++ ++char **remote_added_layers(const map_t *old, const map_t *new_l); + + #ifdef __cplusplus + } +diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c b/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c +index 7df7a221..a3aa3aa4 100644 +--- a/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c ++++ b/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c +@@ -17,19 +17,15 @@ + + #include + #include ++#include + #include + #include +-#include "map.h" ++ + #include "path.h" + #include "linked_list.h" +-#include "layer_store.h" +-#include "layer.h" + #include "isula_libutils/log.h" +-#include "image_store.h" +-#include "remote_support.h" + #include "utils.h" + #include "utils_file.h" +-#include "stdlib.h" + + #define REMOTE_RO_LAYER_DIR "RO" + +@@ -43,14 +39,6 @@ static char *layer_home; + static char *overlay_ro_dir; + static char *overlay_home; + +-struct supporters { +- remote_supporter *image_supporter; +- remote_supporter *layer_supporter; +- remote_supporter *overlay_supporter; +-}; +- +-static struct supporters supporters; +- + int remote_image_init(const char *root_dir) + { + if (root_dir == NULL) { +@@ -134,75 +122,11 @@ void remote_maintain_cleanup(void) + overlay_ro_dir = NULL; + } + +-// to maintain the symbol links, add new symbol link and delete invalid symbol link +-// arg is const char *driver_home +-// scanning driver->home/RO/ directory, build symlink in driver->home +-static void *remote_refresh_ro_symbol_link(void *arg) +-{ +- struct supporters *supporters = (struct supporters *)arg; +- prctl(PR_SET_NAME, "RoLayerRefresh"); +- +- while (true) { +- util_usleep_nointerupt(5 * 1000 * 1000); +- DEBUG("remote refresh start\n"); +- scan_remote_dir(supporters->overlay_supporter); +- load_item(supporters->overlay_supporter); +- scan_remote_dir(supporters->layer_supporter); +- load_item(supporters->layer_supporter); +- scan_remote_dir(supporters->image_supporter); +- load_item(supporters->image_supporter); +- DEBUG("remote refresh end\n"); +- } +- return NULL; +-} +- +-int start_refresh_thread(void) +-{ +- int res = 0; +- pthread_t a_thread; +- +- supporters.image_supporter = create_image_supporter(image_home, NULL); +- if (supporters.image_supporter == NULL) { +- goto free_out; +- } +- +- supporters.layer_supporter = create_layer_supporter(layer_home, layer_ro_dir); +- if (supporters.layer_supporter == NULL) { +- goto free_out; +- } +- +- supporters.overlay_supporter = create_overlay_supporter(overlay_home, overlay_ro_dir); +- if (supporters.overlay_supporter == NULL) { +- goto free_out; +- } +- +- res = pthread_create(&a_thread, NULL, remote_refresh_ro_symbol_link, (void *)&supporters); +- if (res != 0) { +- CRIT("Thread creation failed"); +- return -1; +- } +- +- if (pthread_detach(a_thread) != 0) { +- SYSERROR("Failed to detach 0x%lx", a_thread); +- return -1; +- } +- +- return 0; +- +-free_out: +- destroy_suppoter(supporters.image_supporter); +- destroy_suppoter(supporters.layer_supporter); +- destroy_suppoter(supporters.overlay_supporter); +- +- return -1; +-} +- + static int do_build_ro_dir(const char *home, const char *id) + { + char *ro_symlink = NULL; + char *ro_layer_dir = NULL; + int nret = 0; +- // bool ret = true; + int ret = 0; + + nret = asprintf(&ro_symlink, "%s/%s", home, id); +@@ -305,43 +229,15 @@ int remote_overlay_remove_ro_dir(const char *id) + return do_remove_ro_dir(overlay_home, id); + } + +-static char **map_diff(map_t *map_a, map_t *map_b) +-{ +- char **array = NULL; +- map_itor *itor = map_itor_new(map_a); +- bool *found = NULL; +- +- // iter new_map, every item not in old, append them to new_layers +- for (; map_itor_valid(itor); map_itor_next(itor)) { +- char *id = map_itor_key(itor); +- found = map_search(map_b, id); +- if (found == NULL) { +- util_array_append(&array, util_strdup_s(id)); +- } +- } +- +- map_itor_free(itor); +- +- return array; +-} +- +-char **deleted_layers(map_t *old, map_t *new) +-{ +- return map_diff(old, new); +-} +- +-char **added_layers(map_t *old, map_t *new) ++maintain_context get_maintain_context(void) + { +- return map_diff(new, old); +-} ++ maintain_context ctx = {0x0}; + +-int empty_map(map_t *mp) +-{ +- if (mp == NULL) { +- return -1; +- } ++ ctx.image_home = image_home; ++ ctx.layer_ro_dir = layer_ro_dir; ++ ctx.layer_home = layer_home; ++ ctx.overlay_ro_dir = overlay_ro_dir; ++ ctx.overlay_home = overlay_home; + +- map_clear(mp); +- mp->store->root = mp->store->nil; +- return 0; ++ return ctx; + } +diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.h b/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.h +index 25712d40..aa2036ea 100644 +--- a/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.h ++++ b/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.h +@@ -15,12 +15,21 @@ + #ifndef DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_RO_SYMLINK_MAINTAIN_H + #define DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_RO_SYMLINK_MAINTAIN_H + +-#include "map.h" ++#define REMOTE_RO_LAYER_DIR "RO" ++#define OVERLAY_RO_DIR "RO" + + #ifdef __cplusplus + extern "C" { + #endif + ++typedef struct { ++ const char *image_home; ++ const char *layer_ro_dir; ++ const char *layer_home; ++ const char *overlay_ro_dir; ++ const char *overlay_home; ++} maintain_context; ++ + int remote_image_init(const char *root_dir); + + int remote_layer_init(const char *root_dir); +@@ -29,8 +38,6 @@ int remote_overlay_init(const char *driver_home); + + void remote_maintain_cleanup(void); + +-int start_refresh_thread(void); +- + int remote_layer_build_ro_dir(const char *id); + + int remote_overlay_build_ro_dir(const char *id); +@@ -39,11 +46,7 @@ int remote_layer_remove_ro_dir(const char *id); + + int remote_overlay_remove_ro_dir(const char *id); + +-char **deleted_layers(map_t *old, map_t *new); +- +-char **added_layers(map_t *old, map_t *new); +- +-int empty_map(map_t *mp); ++maintain_context get_maintain_context(void); + + #ifdef __cplusplus + } +diff --git a/src/daemon/modules/image/oci/storage/storage.c b/src/daemon/modules/image/oci/storage/storage.c +index 31812a22..f9830ac3 100644 +--- a/src/daemon/modules/image/oci/storage/storage.c ++++ b/src/daemon/modules/image/oci/storage/storage.c +@@ -43,7 +43,7 @@ + #include "utils_verify.h" + #include "sha256.h" + #ifdef ENABLE_REMOTE_LAYER_STORE +-#include "ro_symlink_maintain.h" ++#include "remote_support.h" + #endif + + static pthread_rwlock_t g_storage_rwlock; +@@ -1874,7 +1874,7 @@ int storage_module_init(struct storage_module_init_options *opts) + } + + #ifdef ENABLE_REMOTE_LAYER_STORE +- if (opts->enable_remote_layer && start_refresh_thread() != 0) { ++ if (opts->enable_remote_layer && remote_start_refresh_thread() != 0) { + ERROR("Failed to start remote refresh thread"); + } + #endif +diff --git a/src/utils/cutils/map/rb_tree.c b/src/utils/cutils/map/rb_tree.c +index e933003a..b85371e3 100644 +--- a/src/utils/cutils/map/rb_tree.c ++++ b/src/utils/cutils/map/rb_tree.c +@@ -134,6 +134,7 @@ void rbtree_clear(rb_tree_t *tree) + return; + } + rbtree_destroy_all(tree, tree->root); ++ tree->root = tree->nil; + } + + void rbtree_free(rb_tree_t *tree) +diff --git a/test/image/oci/registry/CMakeLists.txt b/test/image/oci/registry/CMakeLists.txt +index 13ed95b2..5475b7b5 100644 +--- a/test/image/oci/registry/CMakeLists.txt ++++ b/test/image/oci/registry/CMakeLists.txt +@@ -26,6 +26,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/registry_type.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/sysinfo.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/storage/image_store/image_store.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/registry/registry.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/registry/registry_apiv2.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/registry/registry_apiv1.c +@@ -57,6 +58,7 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/storage + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/storage/image_store ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/storage/remote_layer_support + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/registry + ${CMAKE_CURRENT_SOURCE_DIR}/../../../mocks + ) +diff --git a/test/image/oci/storage/CMakeLists.txt b/test/image/oci/storage/CMakeLists.txt +index feb81b14..7a9d77fe 100644 +--- a/test/image/oci/storage/CMakeLists.txt ++++ b/test/image/oci/storage/CMakeLists.txt +@@ -3,3 +3,6 @@ project(iSulad_UT) + add_subdirectory(images) + add_subdirectory(rootfs) + add_subdirectory(layers) ++IF (ENABLE_REMOTE_LAYER_STORE) ++add_subdirectory(remote_layer_support) ++ENDIF() +diff --git a/test/image/oci/storage/images/CMakeLists.txt b/test/image/oci/storage/images/CMakeLists.txt +index 3e6b69a4..8446ebba 100644 +--- a/test/image/oci/storage/images/CMakeLists.txt ++++ b/test/image/oci/storage/images/CMakeLists.txt +@@ -21,6 +21,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/image_store/image_type.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/registry_type.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/image_store/image_store.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../mocks/storage_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../mocks/isulad_config_mock.cc + storage_images_ut.cc) +@@ -39,6 +40,7 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/image_store ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/registry + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../mocks + ) +diff --git a/test/image/oci/storage/layers/CMakeLists.txt b/test/image/oci/storage/layers/CMakeLists.txt +index 952e9483..ae0ac9c3 100644 +--- a/test/image/oci/storage/layers/CMakeLists.txt ++++ b/test/image/oci/storage/layers/CMakeLists.txt +@@ -30,6 +30,7 @@ add_executable(${DRIVER_EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/metadata_store.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/wrapper_devmapper.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/quota/project_quota.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../mocks/driver_quota_mock.cc + storage_driver_ut.cc) +@@ -52,6 +53,7 @@ target_include_directories(${DRIVER_EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2 ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/quota + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../mocks + ) +@@ -105,6 +107,7 @@ add_executable(${LAYER_EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/wrapper_devmapper.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/quota/project_quota.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../mocks/driver_quota_mock.cc + storage_layers_ut.cc) + +@@ -129,6 +132,7 @@ target_include_directories(${LAYER_EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2 + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/quota ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../mocks + ) + +diff --git a/test/image/oci/storage/remote_layer_support/CMakeLists.txt b/test/image/oci/storage/remote_layer_support/CMakeLists.txt +new file mode 100644 +index 00000000..c36d9049 +--- /dev/null ++++ b/test/image/oci/storage/remote_layer_support/CMakeLists.txt +@@ -0,0 +1,44 @@ ++project(iSulad_UT) ++ ++SET(EXE remote_layer_support_ut) ++ ++add_executable(${EXE} ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../mocks/remote_store_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils/map/map.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support/overlay_remote_impl.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support/image_remote_impl.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support/layer_remote_impl.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils/utils_file.c ++ remote_layer_ut.cc ++ ) ++ ++target_include_directories(${EXE} PUBLIC ++ ${GTEST_INCLUDE_DIR} ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../include ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/config ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/common ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/api ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/image_store ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2 ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../mocks ++ ) ++ ++target_link_libraries(${EXE} ++ ${GTEST_BOTH_LIBRARIES} ++ ${GMOCK_LIBRARY} ++ ${GMOCK_MAIN_LIBRARY} ++ ${CMAKE_THREAD_LIBS_INIT} ++ ${ISULA_LIBUTILS_LIBRARY} ++ -lgtest -lgtest_main libutils_ut -lcrypto -lyajl -lz) ++ ++add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/image/oci/storage/remote_layer_support/remote_layer_ut.cc b/test/image/oci/storage/remote_layer_support/remote_layer_ut.cc +new file mode 100644 +index 00000000..5f5e92fb +--- /dev/null ++++ b/test/image/oci/storage/remote_layer_support/remote_layer_ut.cc +@@ -0,0 +1,93 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: wangrunze ++ * Create: 2023-03-16 ++ * Description: provide remote layer support ut ++ ******************************************************************************/ ++#include ++ ++#include "remote_store_mock.h" ++#include "ro_symlink_maintain.h" ++#include "remote_support.h" ++#include "map.h" ++ ++using ::testing::Invoke; ++ ++bool invokeOverlayRemoteLayerValid(const char *id) ++{ ++ return true; /* currently always valid overlay layer */ ++} ++ ++bool invokeLayerRemoteLayerValid(const char *id) ++{ ++ return true; ++} ++ ++int invokeLayerLoadOneLayer(const char *id) ++{ ++ return 0; ++} ++ ++int invokeLayerRemoveOneLayer(const char *id) ++{ ++ return 0; ++} ++ ++int invokeImageAppendOneImage(const char *id) ++{ ++ return 0; ++} ++ ++int invokeImageRemoveOneImage(const char *id) ++{ ++ return 0; ++} ++ ++char *invokeImageGetTopLayer(const char *id) ++{ ++ return NULL; ++} ++ ++int invokeImageValidSchemaV1(const char *path, bool *valid) ++{ ++ return 0; ++} ++ ++void mockCommonAll(MockRemoteStore *mock) ++{ ++ EXPECT_CALL(*mock, LayerLoadOneLayer(::testing::_)).WillRepeatedly(Invoke(invokeLayerLoadOneLayer)); ++ EXPECT_CALL(*mock, LayerRemoveOneLayer(::testing::_)).WillRepeatedly(Invoke(invokeLayerRemoveOneLayer)); ++ ++ EXPECT_CALL(*mock, ImageAppendOneImage(::testing::_)).WillRepeatedly(Invoke(invokeImageAppendOneImage)); ++ EXPECT_CALL(*mock, ImageRemoveOneImage(::testing::_)).WillRepeatedly(Invoke(invokeImageRemoveOneImage)); ++ EXPECT_CALL(*mock, ImageGetTopLayer(::testing::_)).WillRepeatedly(Invoke(invokeImageGetTopLayer)); ++ EXPECT_CALL(*mock, ImageValidSchemaV1(::testing::_, ::testing::_)).WillRepeatedly(Invoke(invokeImageValidSchemaV1)); ++} ++ ++TEST(remote_Layer_ut, test_map_diff) ++{ ++ // old: a b x ++ // new: x b c ++ map_t *old_one = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); ++ map_t *new_one = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); ++ bool exist = true; ++ ++ map_insert(old_one, (void *)"a", (void *)&exist); ++ map_insert(old_one, (void *)"b", (void *)&exist); ++ map_insert(new_one, (void *)"b", (void *)&exist); ++ map_insert(new_one, (void *)"c", (void *)&exist); ++ ++ char **added = remote_added_layers(old_one, new_one); ++ char **deleted = remote_deleted_layers(old_one, new_one); ++ ++ ASSERT_EQ(added[0][0], 'c'); ++ ASSERT_EQ(deleted[0][0], 'a'); ++} +diff --git a/test/image/oci/storage/rootfs/CMakeLists.txt b/test/image/oci/storage/rootfs/CMakeLists.txt +index 4d7d3533..5383fbac 100644 +--- a/test/image/oci/storage/rootfs/CMakeLists.txt ++++ b/test/image/oci/storage/rootfs/CMakeLists.txt +@@ -22,6 +22,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/utils_images.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/rootfs_store/rootfs.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/rootfs_store/rootfs_store.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../mocks/storage_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../mocks/isulad_config_mock.cc + storage_rootfs_ut.cc) +@@ -41,6 +42,7 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/rootfs_store ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support + ) + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lcrypto -lyajl -lz) +diff --git a/test/mocks/remote_store_mock.cc b/test/mocks/remote_store_mock.cc +new file mode 100644 +index 00000000..c6428623 +--- /dev/null ++++ b/test/mocks/remote_store_mock.cc +@@ -0,0 +1,68 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: wangrunze ++ * Create: 2023-03-09 ++ * Description: provide mock for image store, layer store and driver overlay ++ ******************************************************************************/ ++ ++#include "remote_store_mock.h" ++ ++namespace { ++MockRemoteStore *g_remote_store_mock = nullptr; ++} ++ ++int remote_load_one_layer(const char *id) ++{ ++ if (g_remote_store_mock != nullptr) { ++ return g_remote_store_mock->LayerLoadOneLayer(id); ++ } ++ return -1; ++} ++ ++int remote_layer_remove_memory_stores_with_lock(const char *id) ++{ ++ if (g_remote_store_mock != nullptr) { ++ return g_remote_store_mock->LayerRemoveOneLayer(id); ++ } ++ return -1; ++} ++ ++int image_store_validate_manifest_schema_version_1(const char *path, bool *valid) ++{ ++ if (g_remote_store_mock != nullptr) { ++ return g_remote_store_mock->ImageValidSchemaV1(path, valid); ++ } ++ return -1; ++} ++ ++int remote_append_image_by_directory_with_lock(const char *image_dir) ++{ ++ if (g_remote_store_mock != nullptr) { ++ return g_remote_store_mock->ImageAppendOneImage(image_dir); ++ } ++ return -1; ++} ++ ++int remote_remove_image_from_memory_with_lock(const char *id) ++{ ++ if (g_remote_store_mock != nullptr) { ++ return g_remote_store_mock->ImageRemoveOneImage(id); ++ } ++ return -1; ++} ++ ++char *remote_image_get_top_layer_from_json(const char *img_id) ++{ ++ if (g_remote_store_mock != nullptr) { ++ return g_remote_store_mock->ImageGetTopLayer(img_id); ++ } ++ return nullptr; ++} +diff --git a/test/mocks/remote_store_mock.h b/test/mocks/remote_store_mock.h +new file mode 100644 +index 00000000..e8d73ef1 +--- /dev/null ++++ b/test/mocks/remote_store_mock.h +@@ -0,0 +1,40 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: wangrunze ++ * Create: 2023-03-09 ++ * Description: provide mock for image store, layer store and driver overlay ++ ******************************************************************************/ ++ ++#ifndef _ISULAD_TEST_MOCKS_REMOTE_STORE_MOCK_H ++#define _ISULAD_TEST_MOCKS_REMOTE_STORE_MOCK_H ++ ++#include ++ ++#include "image_store.h" ++#include "layer_store.h" ++#include "driver_overlay2.h" ++ ++class MockRemoteStore { ++public: ++ virtual ~MockRemoteStore() = default; ++ // MOCK_METHOD1(OverlayRemoteLayerValid, bool(const char *)); ++ ++ // MOCK_METHOD1(LayerRemoteLayerValid, bool(const char *)); ++ MOCK_METHOD1(LayerLoadOneLayer, int(const char *)); ++ MOCK_METHOD1(LayerRemoveOneLayer, int(const char *)); ++ ++ MOCK_METHOD1(ImageAppendOneImage, int(const char *)); ++ MOCK_METHOD1(ImageRemoveOneImage, int(const char *)); ++ MOCK_METHOD1(ImageGetTopLayer, char *(const char *)); ++ MOCK_METHOD2(ImageValidSchemaV1, int(const char *, bool *)); ++}; ++ ++#endif // _ISULAD_TEST_MOCKS_IMAGE_MOCK_H +-- +2.25.1 + diff --git a/0052-add-ci-for-runc.patch b/0052-add-ci-for-runc.patch new file mode 100644 index 0000000..24c4194 --- /dev/null +++ b/0052-add-ci-for-runc.patch @@ -0,0 +1,1666 @@ +From b7b26805535a73eb647111d247d0e7c5f2c81856 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Wed, 22 Mar 2023 09:50:56 +0800 +Subject: [PATCH 52/53] add ci for runc + +Signed-off-by: zhongtao +--- + CI/test_cases/container_cases/annotaion.sh | 31 ++-- + .../container_cases/bind_special_dir.sh | 10 +- + .../check_fd_leakage_of_isulad.sh | 31 +++- + .../container_cases/cleanup_leftover.sh | 49 ++++--- + .../container_cases/cni_bridge_test.sh | 14 +- + CI/test_cases/container_cases/cpu_rt.sh | 88 ++++++----- + CI/test_cases/container_cases/create.sh | 41 ++++-- + .../container_cases/cri_default_namespace.sh | 28 ++-- + CI/test_cases/container_cases/cri_pod_ip.sh | 18 ++- + CI/test_cases/container_cases/cri_stream.sh | 58 ++++++-- + CI/test_cases/container_cases/kill.sh | 19 ++- + CI/test_cases/container_cases/log_test.sh | 138 ++++++++++-------- + CI/test_cases/container_cases/pause.sh | 21 ++- + CI/test_cases/container_cases/resume.sh | 25 ++-- + CI/test_cases/container_cases/rm.sh | 29 ++-- + CI/test_cases/container_cases/start.sh | 39 +++-- + CI/test_cases/container_cases/stats.sh | 16 +- + CI/test_cases/container_cases/ulimit.sh | 24 +-- + CI/test_cases/container_cases/update.sh | 46 ++++-- + CI/test_cases/helpers.sh | 5 +- + 20 files changed, 472 insertions(+), 258 deletions(-) + +diff --git a/CI/test_cases/container_cases/annotaion.sh b/CI/test_cases/container_cases/annotaion.sh +index 65c474e8..93b432e5 100755 +--- a/CI/test_cases/container_cases/annotaion.sh ++++ b/CI/test_cases/container_cases/annotaion.sh +@@ -21,13 +21,12 @@ + + declare -r curr_path=$(dirname $(readlink -f "$0")) + source ../helpers.sh +-test="annotation test => test_annotation" + + function test_label() + { + local ret=0 + +- isula run -tid --name annotation --label "test_long_label=111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111" busybox sh ++ isula run -tid --name annotation --runtime $1 --label "test_long_label=111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111" busybox sh + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container for long label" && ((ret++)) + + isula inspect annotation +@@ -52,7 +51,7 @@ function test_annotation() + { + local ret=0 + +- isula run -tid --name annotation --annotation "test_long_label=111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111" busybox sh ++ isula run -tid --name annotation --runtime $1 --annotation "test_long_label=111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111" busybox sh + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container for long label" && ((ret++)) + + isula inspect annotation +@@ -73,14 +72,28 @@ function test_annotation() + return ${ret} + } + +-declare -i ans=0 ++function do_test_t() ++{ ++ local ret=0 ++ local runtime=$1 ++ local test="annotation test => $runtime" ++ msg_info "${test} starting..." + +-msg_info "${test} starting..." +-[[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - start isulad failed" && ((ret++)) ++ test_label $runtime || ((ret++)) ++ test_annotation $runtime || ((ret++)) + +-test_label || ((ans++)) +-test_annotation || ((ans++)) ++ msg_info "${test} finished with return ${ret}..." ++ return $ret ++} ++ ++declare -i ans=0 + +-msg_info "${test} finished with return ${ans}..." ++for element in ${RUNTIME_LIST[@]}; ++do ++ do_test_t $element ++ if [ $? -ne 0 ];then ++ let "ans=$ans + 1" ++ fi ++done + + show_result ${ans} "${curr_path}/${0}" +diff --git a/CI/test_cases/container_cases/bind_special_dir.sh b/CI/test_cases/container_cases/bind_special_dir.sh +index e855404b..e2653685 100755 +--- a/CI/test_cases/container_cases/bind_special_dir.sh ++++ b/CI/test_cases/container_cases/bind_special_dir.sh +@@ -25,8 +25,9 @@ source ../helpers.sh + function test_bind_special_dir() + { + local ret=0 ++ local runtime=$1 + local image="busybox" +- local test="container bind special directory test => (${FUNCNAME[@]})" ++ local test="container bind special directory test with ($runtime) => (${FUNCNAME[@]})" + + msg_info "${test} starting..." + +@@ -36,7 +37,7 @@ function test_bind_special_dir() + isula images | grep busybox + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${image}" && ((ret++)) + +- c_id=`isula run -itd -v -itd -v /sys/fs:/sys/fs:rw,rshared -v /proc:/proc -v /dev:/dev:ro -v /dev/pts:/dev/pts:rw busybox sh` ++ c_id=`isula run --runtime=$runtime -itd -v -itd -v /sys/fs:/sys/fs:rw,rshared -v /proc:/proc -v /dev:/dev:ro -v /dev/pts:/dev/pts:rw busybox sh` + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++)) + + isula exec -it $c_id sh -c "ls -al /sys/fs" | grep "cgroup" +@@ -51,6 +52,9 @@ function test_bind_special_dir() + + declare -i ans=0 + +-test_bind_special_dir || ((ans++)) ++for element in ${RUNTIME_LIST[@]}; ++do ++ test_bind_special_dir $element || ((ans++)) ++done + + show_result ${ans} "${curr_path}/${0}" +diff --git a/CI/test_cases/container_cases/check_fd_leakage_of_isulad.sh b/CI/test_cases/container_cases/check_fd_leakage_of_isulad.sh +index 39c9f49f..1978b587 100755 +--- a/CI/test_cases/container_cases/check_fd_leakage_of_isulad.sh ++++ b/CI/test_cases/container_cases/check_fd_leakage_of_isulad.sh +@@ -26,6 +26,10 @@ connect="grpc" + + function do_test_t_grpc() + { ++ local runtime=$1 ++ local test="do_test_t_grpc test => $runtime" ++ msg_info "${test} starting..." ++ + if [ $connect != "grpc" ];then + echo "this test is designed for grpc version" + return 0 +@@ -34,7 +38,7 @@ function do_test_t_grpc() + containername=test_fds + isulad_pid=`cat /var/run/isulad.pid` + precount=`ls /proc/$isulad_pid/fd | wc -l` +- isula create -t --name $containername busybox ++ isula create -t --name $containername --runtime $runtime busybox + fn_check_eq "$?" "0" "create failed" + testcontainer $containername inited + +@@ -61,11 +65,15 @@ function do_test_t_grpc() + curcount=`ls /proc/$isulad_pid/fd | wc -l` + fn_check_eq "$precount" "$curcount" "test failed" + ++ msg_info "${test} finished with return ${TC_RET_T}..." + return $TC_RET_T + } + + function do_test_t_rest() + { ++ local runtime=$1 ++ local test="do_test_t_rest test => $runtime" ++ msg_info "${test} starting..." + if [ $connect != "rest" ];then + echo "this test is designed for rest version" + return 0 +@@ -76,7 +84,7 @@ function do_test_t_rest() + isulad_pid=`cat /var/run/isulad.pid` + precount=`ls /proc/$isulad_pid/fd | wc -l` + +- isula create -t --name $containername busybox ++ isula create -t --name $containername --runtime $runtime busybox + fn_check_eq "$?" "0" "create failed" + testcontainer $containername inited + +@@ -118,15 +126,24 @@ function do_test_t_rest() + TC_RET_T=$(($TC_RET_T+1)) + fi + ++ msg_info "${test} finished with return ${TC_RET_T}..." + return $TC_RET_T + } + + ret=0 + +-do_test_t_grpc +-do_test_t_rest +-if [ $? -ne 0 ];then +- let "ret=$ret + 1" +-fi ++for element in ${RUNTIME_LIST[@]}; ++do ++ do_test_t_grpc $element ++ if [ $? -ne 0 ];then ++ let "ret=$ret + 1" ++ fi ++ do_test_t_rest $element ++ if [ $? -ne 0 ];then ++ let "ret=$ret + 1" ++ fi ++done ++ ++ + + show_result $ret "basic check fd leak" +diff --git a/CI/test_cases/container_cases/cleanup_leftover.sh b/CI/test_cases/container_cases/cleanup_leftover.sh +index 054f7d48..6f29180c 100755 +--- a/CI/test_cases/container_cases/cleanup_leftover.sh ++++ b/CI/test_cases/container_cases/cleanup_leftover.sh +@@ -5,26 +5,41 @@ + declare -r curr_path=$(dirname $(readlink -f "$0")) + source ../helpers.sh + +-containerid=$(isula run -tid busybox ls) ++function test_cleanup() ++{ ++ local runtime=$1 ++ local test="cleanup_test => (${runtime})" ++ msg_info "${test} starting..." + +-check_valgrind_log +-rm -rf "$LCR_ROOT_PATH/$containerid" ++ containerid=$(isula run -tid --runtime $runtime busybox ls) + +-start_isulad_with_valgrind +-wait_isulad_running ++ check_valgrind_log ++ rm -rf "$RUNTIME_ROOT_PATH/$runtime/$containerid" ++ ++ start_isulad_with_valgrind ++ wait_isulad_running + +-ret=0 +-ls "/var/lib/isulad/storage/overlay-containers/$containerid" +-if [ $? != 0 ]; then +- echo "ls can't access which is expected" + ret=0 +-else +- ret=1; +-fi +-check_valgrind_log +-[[ $? -ne 0 ]] && msg_err "cleanup leftover - memory leak, please check...." && ((ret++)) +- +-start_isulad_with_valgrind +-wait_isulad_running ++ ls "/var/lib/isulad/storage/overlay-containers/$containerid" ++ if [ $? != 0 ]; then ++ echo "ls can't access which is expected" ++ ret=0 ++ else ++ ret=1; ++ fi ++ check_valgrind_log ++ [[ $? -ne 0 ]] && msg_err "cleanup leftover - memory leak, please check...." && ((ret++)) ++ ++ start_isulad_with_valgrind ++ wait_isulad_running ++ msg_info "${test} finished with return ${ret}..." ++} ++ ++declare -i ret=0 ++ ++for element in ${RUNTIME_LIST[@]}; ++do ++ test_cleanup $element || ((ret++)) ++done + + show_result $ret "${curr_path}/${0}" +\ No newline at end of file +diff --git a/CI/test_cases/container_cases/cni_bridge_test.sh b/CI/test_cases/container_cases/cni_bridge_test.sh +index 369b284d..55c6e37c 100755 +--- a/CI/test_cases/container_cases/cni_bridge_test.sh ++++ b/CI/test_cases/container_cases/cni_bridge_test.sh +@@ -72,8 +72,9 @@ function do_post() + function do_test_help() + { + local ret=0 ++ local runtime=$4 + +- msg_info "this is $0 do_test" ++ msg_info "this is $0 do_test with runtime $runtime" + + crictl pull busybox + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Failed to pull busybox image" && ((ret++)) +@@ -81,14 +82,14 @@ function do_test_help() + crictl images | grep "mirrorgooglecontainers/pause-amd64" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Failed to find mirrorgooglecontainers/pause-amd64 image" && ((ret++)) + +- sid1=`crictl runp ${data_path}/$1` ++ sid1=`crictl runp --runtime $runtime ${data_path}/$1` + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Failed to run sandbox1" && ((ret++)) + + spid1=`isula inspect -f '{{json .State.Pid}}' $sid1` + nsenter -t $spid1 -n ifconfig eth0 + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Sandbox1 network config failed" && ((ret++)) + +- sid2=`crictl runp ${data_path}/$2` ++ sid2=`crictl runp --runtime $runtime ${data_path}/$2` + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Failed to run sandbox2" && ((ret++)) + + spid2=`isula inspect -f '{{json .State.Pid}}' $sid2` +@@ -123,7 +124,7 @@ function do_test_help() + crictl rmp $sid1 $sid2 + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - rm sandbox failed" && ((ret++)) + +- msg_info "$0 do_test finished with return ${ret}..." ++ msg_info "$0 do_test with runtime $runtime finished with return ${ret}..." + return ${ret} + } + +@@ -131,7 +132,10 @@ declare -i ans=0 + + do_pre || ((ans++)) + +-do_test_help "sandbox-config.json" "sandbox-config2.json" "10\.2\." || ((ans++)) ++for element in ${RUNTIME_LIST[@]}; ++do ++ do_test_help "sandbox-config.json" "sandbox-config2.json" "10\.2\." $element || ((ans++)) ++done + + do_post + +diff --git a/CI/test_cases/container_cases/cpu_rt.sh b/CI/test_cases/container_cases/cpu_rt.sh +index 39c0b427..bdc43a5e 100755 +--- a/CI/test_cases/container_cases/cpu_rt.sh ++++ b/CI/test_cases/container_cases/cpu_rt.sh +@@ -47,10 +47,9 @@ function test_isula_update_normal() + msg_info "${test} starting..." + + #start isulad with cpu_rt +- isulad --cpu-rt-period 1000000 --cpu-rt-runtime 950000 -l DEBUG > /dev/null 2>&1 & +- wait_isulad_running ++ start_isulad_without_valgrind --cpu-rt-period 1000000 --cpu-rt-runtime 950000 + +- c_id=`isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 1000 ${image} sh` ++ c_id=`isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 1000 --runtime $1 ${image} sh` + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++)) + + isula update --cpu-rt-period 900000 --cpu-rt-runtime 2000 $c_id +@@ -67,8 +66,7 @@ function test_isula_update_normal() + + stop_isulad_without_valgrind + #set cpu-rt to the initial state +- isulad --cpu-rt-period 1000000 --cpu-rt-runtime 0 -l DEBUG > /dev/null 2>&1 & +- wait_isulad_running ++ start_isulad_without_valgrind --cpu-rt-period 1000000 --cpu-rt-runtime 0 + + msg_info "${test} finished with return ${ret}..." + return ${ret} +@@ -84,23 +82,18 @@ function test_isula_update_abnormal() + isulad --cpu-rt-period 1000000 --cpu-rt-runtime 950000 -l DEBUG > /dev/null 2>&1 & + wait_isulad_running + +- c_id=`isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 1000 ${image} sh` ++ c_id=`isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 1000 --runtime $1 ${image} sh` + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++)) + + isula update --cpu-rt-period 800000 --cpu-rt-runtime 900000 $c_id 2>&1 | grep "Invalid --cpu-rt-runtime: rt runtime cannot be higher than rt period" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to update container cpu-rt-runtime" && ((ret++)) + +- isula update --cpu-rt-runtime 1000000 $c_id 2>&1 | grep "updating cgroup cpu.rt_runtime_us to 1000000: Invalid argument" ++ isula update --cpu-rt-runtime 1000000 $c_id 2>&1 | grep -i "invalid argument" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to update container cpu-rt-runtime" && ((ret++)) + + isula rm -f $c_id + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to rm container ${c_id}" && ((ret++)) + +- stop_isulad_without_valgrind +- #set cpu-rt to the initial state +- isulad --cpu-rt-period 1000000 --cpu-rt-runtime 0 -l DEBUG > /dev/null 2>&1 & +- wait_isulad_running +- + msg_info "${test} finished with return ${ret}..." + return ${ret} + } +@@ -113,8 +106,7 @@ function test_kernel_without_cpurt() + + msg_info "${test} starting..." + +- isulad --cpu-rt-period 1000000 --cpu-rt-runtime 950000 -l DEBUG > /dev/null 2>&1 & +- wait_isulad_running ++ start_isulad_without_valgrind --cpu-rt-period 1000000 --cpu-rt-runtime 950000 + + isula pull ${image} + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to pull image: ${image}" && return ${FAILURE} +@@ -122,7 +114,7 @@ function test_kernel_without_cpurt() + isula images | grep busybox + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${image}" && ((ret++)) + +- isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 900000 $image /bin/sh 2>&1 | grep "Your kernel does not support cgroup rt" ++ isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 900000 --runtime $1 $image /bin/sh 2>&1 | grep "Your kernel does not support cgroup rt" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - kernel does not support cpu-rt" && ((ret++)) + + msg_info "${test} finished with return ${ret}..." +@@ -137,9 +129,8 @@ function test_isula_run_abnormal() + + msg_info "${test} starting..." + +- #start isulad without cpu_rt +- isulad --cpu-rt-period 1000000 --cpu-rt-runtime 950000 -l DEBUG > /dev/null 2>&1 & +- wait_isulad_running ++ #start isulad with cpu_rt ++ start_isulad_without_valgrind --cpu-rt-period 1000000 --cpu-rt-runtime 950000 + + isula pull ${image} + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to pull image: ${image}" && return ${FAILURE} +@@ -147,25 +138,25 @@ function test_isula_run_abnormal() + isula images | grep busybox + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${image}" && ((ret++)) + +- isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime -1 $image /bin/sh 2>&1 | grep "failed to write -1" | grep "cpu.rt_runtime_us: Invalid argument" ++ isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime -1 --runtime $1 $image /bin/sh 2>&1 | grep "failed to write" | grep -i "cpu.rt_runtime_us: Invalid argument" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Invalid argument for cpu-rt-runtime" && ((ret++)) + +- isula run -itd --cpu-rt-period xx --cpu-rt-runtime 10000 $image /bin/sh 2>&1 | grep 'Invalid value "xx" for flag --cpu-rt-period: Invalid argument' ++ isula run -itd --cpu-rt-period xx --cpu-rt-runtime 10000 --runtime $1 $image /bin/sh 2>&1 | grep 'Invalid value "xx" for flag --cpu-rt-period: Invalid argument' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Invalid argument for cpu-rt-period" && ((ret++)) + +- isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime xx $image /bin/sh 2>&1 | grep 'Invalid value "xx" for flag --cpu-rt-runtime: Invalid argument' ++ isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime xx --runtime $1 $image /bin/sh 2>&1 | grep 'Invalid value "xx" for flag --cpu-rt-runtime: Invalid argument' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Invalid argument for cpu-rt-runtime" && ((ret++)) + +- isula run -itd --cpu-rt-period xx --cpu-rt-runtime xx $image /bin/sh 2>&1 | grep 'Invalid value "xx" for flag --cpu-rt-period: Invalid argument' ++ isula run -itd --cpu-rt-period xx --cpu-rt-runtime xx --runtime $1 $image /bin/sh 2>&1 | grep 'Invalid value "xx" for flag --cpu-rt-period: Invalid argument' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Invalid argument for cpu-rt-period" && ((ret++)) + +- isula run -itd --cpu-rt-period -1 --cpu-rt-runtime 10000 $image /bin/sh 2>&1 | grep "Invalid --cpu-rt-runtime: rt runtime cannot be higher than rt period" ++ isula run -itd --cpu-rt-period -1 --cpu-rt-runtime 10000 --runtime $1 $image /bin/sh 2>&1 | grep "Invalid --cpu-rt-runtime: rt runtime cannot be higher than rt period" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cpu-rt-runtime cannot be higher than cpu-rt-period" && ((ret++)) + +- isula run -itd --cpu-rt-period 100 --cpu-rt-runtime 10000 $image /bin/sh 2>&1 | grep "Invalid --cpu-rt-runtime: rt runtime cannot be higher than rt period" ++ isula run -itd --cpu-rt-period 100 --cpu-rt-runtime 10000 --runtime $1 $image /bin/sh 2>&1 | grep "Invalid --cpu-rt-runtime: rt runtime cannot be higher than rt period" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cpu-rt-runtime cannot be higher than cpu-rt-period" && ((ret++)) + +- isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 960000 $image /bin/sh 2>&1 | grep "failed to write 960000" | grep "cpu.rt_runtime_us: Invalid argument" ++ isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 960000 --runtime $1 $image /bin/sh 2>&1 | grep "failed to write" | grep -i "cpu.rt_runtime_us: Invalid argument" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Invalid argument for cpu-rt-runtime" && ((ret++)) + + msg_info "${test} finished with return ${ret}..." +@@ -177,7 +168,7 @@ function test_isula_run_normal() + local ret=0 + local image="busybox" + +- isula run -itd -n box --cpu-rt-period 1000000 --cpu-rt-runtime 1000 $image /bin/sh 2>&1 ++ isula run -itd -n box --cpu-rt-period 1000000 --cpu-rt-runtime 1000 --runtime $1 $image /bin/sh 2>&1 + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container" && ((ret++)) + + isula rm -f box +@@ -187,18 +178,41 @@ function test_isula_run_normal() + return ${ret} + } + ++function do_test() ++{ ++ local ret=0 ++ local runtime=$1 ++ local test="cpu_rt_test => (${runtime})" ++ msg_info "${test} starting..." ++ ++ if [ -f "/sys/fs/cgroup/cpu/cpu.rt_runtime_us" ];then ++ test_isula_run_abnormal $runtime|| ((ret++)) ++ test_isula_run_normal $runtime || ((ret++)) ++ test_cpurt_isulad_abnormal $runtime || ((ret++)) ++ test_isula_update_normal $runtime || ((ret++)) ++ test_isula_update_abnormal $runtime || ((ret++)) ++ else ++ test_kernel_without_cpurt $runtime || ((ans++)) ++ fi ++ ++ msg_info "${test} finished with return ${ret}..." ++ ++ return ${ret} ++} ++ + declare -i ans=0 + +-if [ -f "/sys/fs/cgroup/cpu/cpu.rt_runtime_us" ];then +- test_isula_run_abnormal || ((ans++)) +- test_isula_run_normal || ((ans++)) +- test_cpurt_isulad_abnormal || ((ans++)) +- test_isula_update_normal || ((ans++)) +- test_isula_update_abnormal || ((ans++)) +-else +- test_kernel_without_cpurt || ((ans++)) +-fi +- +-isula rm -f $(isula ps -aq) ++for element in ${RUNTIME_LIST[@]}; ++do ++ check_valgrind_log ++ ++ do_test $element || ((ans++)) ++ ++ stop_isulad_without_valgrind ++ # set cpu-rt to the initial state ++ start_isulad_with_valgrind --cpu-rt-period 1000000 --cpu-rt-runtime 0 ++ ++ isula rm -f $(isula ps -aq) ++done + + show_result ${ans} "${curr_path}/${0}" +diff --git a/CI/test_cases/container_cases/create.sh b/CI/test_cases/container_cases/create.sh +index 2c28a4fa..868a9b22 100755 +--- a/CI/test_cases/container_cases/create.sh ++++ b/CI/test_cases/container_cases/create.sh +@@ -23,8 +23,12 @@ source ../helpers.sh + + function do_test_t() + { ++ local runtime=$1 ++ local test="create_test => (${runtime})" ++ msg_info "${test} starting..." ++ + containername=test_create +- isula run -itd --name $containername busybox ++ isula run -itd --name $containername --runtime $runtime busybox + fn_check_eq "$?" "0" "create failed" + testcontainer $containername running + +@@ -34,44 +38,44 @@ function do_test_t() + isula inspect $containername + fn_check_ne "$?" "0" "inspect should fail" + +- containerid=`isula run -itd --name $containername --cpu-shares 1024 busybox` ++ containerid=`isula run -itd --name $containername --cpu-shares 1024 --runtime $runtime busybox` + fn_check_eq "$?" "0" "create failed" + +- cat "$LCR_ROOT_PATH/$containerid/config" | grep "cpu.shares = 1024" ++ cat "$RUNTIME_ROOT_PATH/$runtime/$containerid/config.json" | grep "\"shares\": 1024" + fn_check_eq "$?" "0" "create failed" + + isula rm -f $containername + fn_check_eq "$?" "0" "rm failed" + +- containerid=`isula run -itd --name $containername --cpu-quota 50000 busybox` ++ containerid=`isula run -itd --name $containername --cpu-quota 50000 --runtime $runtime busybox` + fn_check_eq "$?" "0" "create failed" + +- cat "$LCR_ROOT_PATH/$containerid/config" | grep "cpu.cfs_quota_us = 50000" ++ cat "$RUNTIME_ROOT_PATH/$runtime/$containerid/config.json" | grep "\"quota\": 50000" + fn_check_eq "$?" "0" "create failed" + + isula rm -f $containername + fn_check_eq "$?" "0" "rm failed" + +- containerid=`isula run -itd --name $containername --cpuset-cpus 0-1 busybox` ++ containerid=`isula run -itd --name $containername --cpuset-cpus 0-1 --runtime $runtime busybox` + fn_check_eq "$?" "0" "create failed" + +- cat "$LCR_ROOT_PATH/$containerid/config" | grep "cpuset.cpus = 0-1" ++ cat "$RUNTIME_ROOT_PATH/$runtime/$containerid/config.json" | grep "\"cpus\": \"0-1\"" + fn_check_eq "$?" "0" "create failed" + + isula rm -f $containername + fn_check_eq "$?" "0" "rm failed" + +- containerid=`isula run -itd --name $containername --memory 1000000000 busybox` ++ containerid=`isula run -itd --name $containername --memory 1000000000 --runtime $runtime busybox` + fn_check_eq "$?" "0" "create failed" + +- cat "$LCR_ROOT_PATH/$containerid/config" | grep "memory.limit_in_bytes = 1000000000" ++ cat "$RUNTIME_ROOT_PATH/$runtime/$containerid/config.json" | grep "\"limit\": 1000000000" + fn_check_eq "$?" "0" "create failed" + + isula rm -f $containername + fn_check_eq "$?" "0" "rm failed" + + # validate --label +- containerid=`isula run -itd --name $containername --label "iSulad=lcrd" busybox` ++ containerid=`isula run -itd --name $containername --label "iSulad=lcrd" --runtime $runtime busybox` + fn_check_eq "$?" "0" "create failed" + + isula inspect -f "{{.Config.Labels}}" ${containerid} | grep iSulad | grep lcrd +@@ -82,7 +86,7 @@ function do_test_t() + + # validate --label-file + echo "iSulad=lcrd\n abc=kkk" > ./label_file +- containerid=`isula run -itd --name $containername --label-file ./label_file busybox` ++ containerid=`isula run -itd --name $containername --label-file ./label_file --runtime $runtime busybox` + fn_check_eq "$?" "0" "create failed" + + isula inspect -f "{{.Config.Labels}}" ${containerid} | grep iSulad | grep lcrd +@@ -97,7 +101,7 @@ function do_test_t() + fn_check_eq "$?" "0" "rm failed" + + # validate --dns --dns-search --dns-opt +- containerid=`isula run -itd --name $containername --dns 8.8.8.8 --dns-opt debug --dns-search example.com busybox` ++ containerid=`isula run -itd --name $containername --dns 8.8.8.8 --dns-opt debug --dns-search example.com --runtime $runtime busybox` + fn_check_eq "$?" "0" "create failed" + + isula exec -it ${containerid} cat /etc/resolv.conf | grep "nameserver 8.8.8.8" +@@ -112,14 +116,19 @@ function do_test_t() + isula rm -f $containername + fn_check_eq "$?" "0" "rm failed" + ++ msg_info "${test} finished with return ${TC_RET_T}..." ++ + return $TC_RET_T + } + + ret=0 + +-do_test_t +-if [ $? -ne 0 ];then +- let "ret=$ret + 1" +-fi ++for element in ${RUNTIME_LIST[@]}; ++do ++ do_test_t $element ++ if [ $? -ne 0 ];then ++ let "ret=$ret + 1" ++ fi ++done + + show_result $ret "basic create" +diff --git a/CI/test_cases/container_cases/cri_default_namespace.sh b/CI/test_cases/container_cases/cri_default_namespace.sh +index 6a17289b..d013c6fa 100755 +--- a/CI/test_cases/container_cases/cri_default_namespace.sh ++++ b/CI/test_cases/container_cases/cri_default_namespace.sh +@@ -43,10 +43,11 @@ function set_up() + function test_cri_default_namespace_in_pod_fun() + { + local ret=0 +- local test="test_cri_default_namespace_in_pod_fun => (${FUNCNAME[@]})" ++ local runtime=$1 ++ local test="test_cri_default_namespace_in_pod_fun => (${runtime})" + msg_info "${test} starting..." + +- sid=$(crictl runp ${data_path}/sandbox-config.json) ++ sid=$(crictl runp --runtime $runtime ${data_path}/sandbox-config.json) + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run sandbox" && ((ret++)) + + cid=$(crictl create $sid ${data_path}/container-config-default-namespace.json ${data_path}/sandbox-config.json) +@@ -73,14 +74,6 @@ function test_cri_default_namespace_in_pod_fun() + [[ x"$sandboxns" == x"$conatainerns" ]] && msg_err "${FUNCNAME[0]}:${LINENO} - $element namespace should be not shared in pod" && ((ret++)) + done + +- msg_info "${test} finished with return ${ret}..." +- return ${ret} +-} +- +-function tear_down() +-{ +- local ret=0 +- + crictl stop $cid + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to stop container" && ((ret++)) + +@@ -93,19 +86,26 @@ function tear_down() + crictl rmp $sid + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to rm sandbox" && ((ret++)) + ++ msg_info "${test} finished with return ${ret}..." ++ return ${ret} ++} ++ ++function tear_down() ++{ + cp -f /etc/isulad/daemon.bak /etc/isulad/daemon.json + check_valgrind_log + start_isulad_with_valgrind +- +- return ${ret} + } + + declare -i ans=0 + + set_up || ((ans++)) + +-test_cri_default_namespace_in_pod_fun || ((ans++)) ++for element in ${RUNTIME_LIST[@]}; ++do ++ test_cri_default_namespace_in_pod_fun $element || ((ans++)) ++done + +-tear_down || ((ans++)) ++tear_down + + show_result ${ans} "${curr_path}/${0}" +diff --git a/CI/test_cases/container_cases/cri_pod_ip.sh b/CI/test_cases/container_cases/cri_pod_ip.sh +index 4048ed74..7bf7833b 100755 +--- a/CI/test_cases/container_cases/cri_pod_ip.sh ++++ b/CI/test_cases/container_cases/cri_pod_ip.sh +@@ -7,7 +7,6 @@ + curr_path=$(dirname $(readlink -f "$0")) + data_path=$(realpath $curr_path/criconfigs) + pause_img_path=$(realpath $curr_path/test_data) +-work_path="/var/lib/isulad/engines/lcr" + pod_config="sandbox-config.json" + source ../helpers.sh + +@@ -41,7 +40,7 @@ function do_post() + + function do_test() + { +- msg_info "this is $0 do_test" ++ msg_info "this is $0 do_test -> ($1)" + + crictl pull busybox + if [ $? -ne 0 ]; then +@@ -55,7 +54,7 @@ function do_test() + TC_RET_T=$(($TC_RET_T+1)) + fi + +- pod_id=`crictl runp ${data_path}/$pod_config` ++ pod_id=`crictl runp --runtime $1 ${data_path}/$pod_config` + if [ $? -ne 0 ]; then + msg_err "Failed to run sandbox" + TC_RET_T=$(($TC_RET_T+1)) +@@ -83,7 +82,7 @@ function do_test() + TC_RET_T=$(($TC_RET_T+1)) + fi + +- cat ${work_path}/${pod_id}/network_settings.json | grep "$ip" ++ cat ${RUNTIME_ROOT_PATH}/${1}/${pod_id}/network_settings.json | grep "$ip" + if [ $? -ne 0 ];then + msg_err "expect ip: $ip, network_settings.json cannot get it" + TC_RET_T=$(($TC_RET_T+1)) +@@ -112,10 +111,13 @@ if [ $? -ne 0 ];then + let "ret=$ret + 1" + fi + +-do_test +-if [ $? -ne 0 ];then +- let "ret=$ret + 1" +-fi ++for element in ${RUNTIME_LIST[@]}; ++do ++ do_test $element ++ if [ $? -ne 0 ];then ++ let "ret=$ret + 1" ++ fi ++done + + do_post + +diff --git a/CI/test_cases/container_cases/cri_stream.sh b/CI/test_cases/container_cases/cri_stream.sh +index 6c89ab38..6124c783 100755 +--- a/CI/test_cases/container_cases/cri_stream.sh ++++ b/CI/test_cases/container_cases/cri_stream.sh +@@ -8,8 +8,9 @@ curr_path=$(dirname $(readlink -f "$0")) + data_path=$(realpath $curr_path/criconfigs) + pause_img_path=$(realpath $curr_path/test_data) + source ../helpers.sh ++export CONTAINER_RUNTIME_ENDPOINT=unix:///var/run/isulad.sock + +-function set_up() ++function do_pre() + { + local ret=0 + local image="busybox" +@@ -21,7 +22,8 @@ function set_up() + cp /etc/isulad/daemon.json /etc/isulad/daemon.bak + sed -i "s#\"pod-sandbox-image\": \"\"#\"pod-sandbox-image\": \"mirrorgooglecontainers/pause-amd64:3.0\"#g" /etc/isulad/daemon.json + +- check_valgrind_log ++ # check_valgrind_log ++ stop_isulad_without_valgrind + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to stop isulad" && return ${FAILURE} + + start_isulad_without_valgrind +@@ -36,7 +38,13 @@ function set_up() + crictl images | grep ${podimage} + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${podimage}" && ((ret++)) + +- sid=$(crictl runp ${data_path}/sandbox-config.json) ++ return ${ret} ++} ++ ++function set_up() ++{ ++ local ret=0 ++ sid=$(crictl runp --runtime $1 ${data_path}/sandbox-config.json) + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run sandbox" && ((ret++)) + + cid=$(crictl create $sid ${data_path}/container-config.json ${data_path}/sandbox-config.json) +@@ -135,22 +143,50 @@ function tear_down() + crictl rmp $sid + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to rm sandbox" && ((ret++)) + ++ return ${ret} ++} ++ ++function do_post() ++{ + cp -f /etc/isulad/daemon.bak /etc/isulad/daemon.json ++ # check_valgrind_log ++ + stop_isulad_without_valgrind +- start_isulad_with_valgrind ++ start_isulad_without_valgrind ++} + +- return ${ret} ++function do_test_t() ++{ ++ local ret=0 ++ local runtime=$1 ++ local test="cri_stream_test => (${runtime})" ++ msg_info "${test} starting..." ++ ++ set_up $runtime || ((ret++)) ++ ++ test_cri_exec_fun || ((ret++)) ++ test_cri_exec_abn || ((ret++)) ++ ++ # runc attach not support ++ if [ $runtime == "lcr" ]; then ++ test_cri_attach || ((ret++)) ++ fi ++ tear_down || ((ret++)) ++ ++ msg_info "${test} finished with return ${ret}..." ++ ++ return $ret + } + + declare -i ans=0 + +-set_up || ((ans++)) +- +-test_cri_exec_fun || ((ans++)) +-test_cri_exec_abn || ((ans++)) ++do_pre || ((ans++)) + +-test_cri_attach || ((ans++)) ++for element in ${RUNTIME_LIST[@]}; ++do ++ do_test_t $element || ((ans++)) ++done + +-tear_down || ((ans++)) ++do_post + + show_result ${ans} "${curr_path}/${0}" +diff --git a/CI/test_cases/container_cases/kill.sh b/CI/test_cases/container_cases/kill.sh +index b31714a7..7b50b094 100755 +--- a/CI/test_cases/container_cases/kill.sh ++++ b/CI/test_cases/container_cases/kill.sh +@@ -25,8 +25,12 @@ source ../helpers.sh + + function do_test_t() + { ++ local runtime=$1 ++ local test="kill_test => (${runtime})" ++ msg_info "${test} starting..." ++ + containername=test_kill +- isula create -t --name $containername busybox ++ isula create -t --name $containername --runtime $runtime busybox + fn_check_eq "$?" "0" "create failed" + testcontainer $containername inited + +@@ -42,14 +46,19 @@ function do_test_t() + isula rm $containername + fn_check_eq "$?" "0" "rm failed" + ++ msg_info "${test} finished with return ${TC_RET_T}..." ++ + return $TC_RET_T + } + + ret=0 + +-do_test_t +-if [ $? -ne 0 ];then +- let "ret=$ret + 1" +-fi ++for element in ${RUNTIME_LIST[@]}; ++do ++ do_test_t $element ++ if [ $? -ne 0 ];then ++ let "ret=$ret + 1" ++ fi ++done + + show_result $ret "basic kill" +diff --git a/CI/test_cases/container_cases/log_test.sh b/CI/test_cases/container_cases/log_test.sh +index 119a005a..d13dd702 100755 +--- a/CI/test_cases/container_cases/log_test.sh ++++ b/CI/test_cases/container_cases/log_test.sh +@@ -24,10 +24,10 @@ function do_post() + + function do_check_item() + { +- cat ${ISULAD_ROOT_PATH}/engines/lcr/$1/config | grep console | grep "$2" ++ cat ${RUNTIME_ROOT_PATH}/$1/$2/config.json | grep console | grep "$3" + if [ $? -ne 0 ]; then +- cat ${ISULAD_ROOT_PATH}/engines/lcr/$1/config | grep console +- msg_err "expect $2" ++ cat ${RUNTIME_ROOT_PATH}/$1/$2/config.json | grep console ++ msg_err "expect $3" + TC_RET_T=$(($TC_RET_T+1)) + fi + } +@@ -42,16 +42,16 @@ function do_test_syslog_helper() + TC_RET_T=$(($TC_RET_T+1)) + fi + +- cid=`isula run -tid busybox sh` ++ cid=`isula run -tid --runtime $2 busybox sh` + if [ $? -ne 0 ]; then + msg_err "Failed to run container" + TC_RET_T=$(($TC_RET_T+1)) + fi + +- do_check_item ${cid} "logdriver = syslog" ++ do_check_item $2 ${cid} "driver\": \"syslog" + + if [ "x$1" != "x" ]; then +- do_check_item ${cid} "syslog_tag = $1" ++ do_check_item $2 ${cid} "tag\": \"$1" + fi + + isula rm -f ${cid} +@@ -74,91 +74,91 @@ function do_test_syslog_tag() + TC_RET_T=$(($TC_RET_T+1)) + fi + +- isula run -ti --log-opt="syslog-tag={{.xxx}}" busybox date ++ isula run -ti --log-opt="syslog-tag={{.xxx}}" --runtime $1 busybox date + if [ $? -eq 0 ]; then + msg_err "run container success with invalid syslog-tag" + TC_RET_T=$(($TC_RET_T+1)) + fi + +- isula run -ti --log-opt="syslog-tag={{" busybox date ++ isula run -ti --log-opt="syslog-tag={{" --runtime $1 busybox date + if [ $? -eq 0 ]; then + msg_err "run container success with invalid syslog-tag" + TC_RET_T=$(($TC_RET_T+1)) + fi + +- isula run -ti --log-opt="syslog-tag=aab{{cd" busybox date ++ isula run -ti --log-opt="syslog-tag=aab{{cd" --runtime $1 busybox date + if [ $? -eq 0 ]; then + msg_err "run container success with invalid syslog-tag" + TC_RET_T=$(($TC_RET_T+1)) + fi + +- cid=$(isula run -tid --log-opt="syslog-tag={{.DaemonName}}" busybox sh) ++ cid=$(isula run -tid --log-opt="syslog-tag={{.DaemonName}}" --runtime $1 busybox sh) + if [ $? -ne 0 ]; then + msg_err "Failed to run container" + TC_RET_T=$(($TC_RET_T+1)) + fi +- do_check_item ${cid} "logdriver = syslog" +- do_check_item ${cid} "syslog_tag = iSulad" ++ do_check_item $1 ${cid} "driver\": \"syslog" ++ do_check_item $1 ${cid} "tag\": \"iSulad" + +- cid=`isula run -tid --log-opt="syslog-tag={{.ID}}" busybox sh` ++ cid=`isula run -tid --log-opt="syslog-tag={{.ID}}" --runtime $1 busybox sh` + if [ $? -ne 0 ]; then + msg_err "Failed to run container" + TC_RET_T=$(($TC_RET_T+1)) + fi +- do_check_item ${cid} "logdriver = syslog" +- do_check_item ${cid} "syslog_tag = ${cid: 0: 12}" ++ do_check_item $1 ${cid} "driver\": \"syslog" ++ do_check_item $1 ${cid} "tag\": \"${cid: 0: 12}" + +- cid=`isula run -tid --name=haozi --log-opt="syslog-tag={{.ID}}xx{{.Name}}" busybox sh` ++ cid=`isula run -tid --name=haozi --log-opt="syslog-tag={{.ID}}xx{{.Name}}" --runtime $1 busybox sh` + if [ $? -ne 0 ]; then + msg_err "Failed to run container" + TC_RET_T=$(($TC_RET_T+1)) + fi +- do_check_item ${cid} "logdriver = syslog" +- do_check_item ${cid} "syslog_tag = ${cid: 0: 12}xxhaozi" ++ do_check_item $1 ${cid} "driver\": \"syslog" ++ do_check_item $1 ${cid} "tag\": \"${cid: 0: 12}xxhaozi" + isula rm -f haozi + +- cid=`isula run -tid --log-opt="syslog-tag={{.FullID}}" busybox sh` ++ cid=`isula run -tid --log-opt="syslog-tag={{.FullID}}" --runtime $1 busybox sh` + if [ $? -ne 0 ]; then + msg_err "Failed to run container" + TC_RET_T=$(($TC_RET_T+1)) + fi +- do_check_item ${cid} "logdriver = syslog" +- do_check_item ${cid} "syslog_tag = ${cid}" ++ do_check_item $1 ${cid} "driver\": \"syslog" ++ do_check_item $1 ${cid} "tag\": \"${cid}" + +- cid=`isula run -tid --name haozi --log-opt="syslog-tag={{.Name}}" busybox sh` ++ cid=`isula run -tid --name haozi --log-opt="syslog-tag={{.Name}}" --runtime $1 busybox sh` + if [ $? -ne 0 ]; then + msg_err "Failed to run container" + TC_RET_T=$(($TC_RET_T+1)) + fi +- do_check_item ${cid} "logdriver = syslog" +- do_check_item ${cid} "syslog_tag = haozi" ++ do_check_item $1 ${cid} "driver\": \"syslog" ++ do_check_item $1 ${cid} "tag\": \"haozi" + isula rm -f haozi + +- cid=`isula run -tid --name haozi --log-opt="syslog-tag=xx{{.Name}}yy" busybox sh` ++ cid=`isula run -tid --name haozi --log-opt="syslog-tag=xx{{.Name}}yy" --runtime $1 busybox sh` + if [ $? -ne 0 ]; then + msg_err "Failed to run container" + TC_RET_T=$(($TC_RET_T+1)) + fi +- do_check_item ${cid} "logdriver = syslog" +- do_check_item ${cid} "syslog_tag = xxhaoziyy" ++ do_check_item $1 ${cid} "driver\": \"syslog" ++ do_check_item $1 ${cid} "tag\": \"xxhaoziyy" + isula rm -f haozi + +- cid=`isula run -tid --log-opt="syslog-tag={{.ImageName}}" busybox sh` ++ cid=`isula run -tid --log-opt="syslog-tag={{.ImageName}}" --runtime $1 busybox sh` + if [ $? -ne 0 ]; then + msg_err "Failed to run container" + TC_RET_T=$(($TC_RET_T+1)) + fi +- do_check_item ${cid} "logdriver = syslog" +- do_check_item ${cid} "syslog_tag = busybox" ++ do_check_item $1 ${cid} "driver\": \"syslog" ++ do_check_item $1 ${cid} "tag\": \"busybox" + +- cid=`isula run -tid --log-opt="syslog-tag={{.ImageID}}" busybox sh` ++ cid=`isula run -tid --log-opt="syslog-tag={{.ImageID}}" --runtime $1 busybox sh` + if [ $? -ne 0 ]; then + msg_err "Failed to run container" + TC_RET_T=$(($TC_RET_T+1)) + fi + img_id=`isula inspect -f '{{.image.id}}' busybox` +- do_check_item ${cid} "logdriver = syslog" +- do_check_item ${cid} "syslog_tag = sha256:${img_id:0:5}" ++ do_check_item $1 ${cid} "driver\": \"syslog" ++ do_check_item $1 ${cid} "tag\": \"sha256:${img_id:0:5}" + + isula rm -f `isula ps -aq` + if [ $? -ne 0 ]; then +@@ -182,15 +182,15 @@ function do_test_json_file_helper() + file_size=$2 + fi + +- cid=`isula run -tid busybox sh` ++ cid=`isula run -tid --runtime $3 busybox sh` + if [ $? -ne 0 ]; then + msg_err "Failed to run container" + TC_RET_T=$(($TC_RET_T+1)) + fi + +- do_check_item ${cid} "logdriver = json-file" +- do_check_item ${cid} "rotate = $file_cnt" +- do_check_item ${cid} "size = $file_size" ++ do_check_item $3 ${cid} "driver\": \"json-file" ++ do_check_item $3 ${cid} "rotate\": \"$file_cnt" ++ do_check_item $3 ${cid} "size\": \"$file_size" + + isula rm -f ${cid} + if [ $? -ne 0 ]; then +@@ -207,40 +207,40 @@ function do_test_container_log() + cat /etc/isulad/daemon.json + ps aux | grep -i isulad + +- cid=`isula run -tid --log-driver=json-file busybox sh` ++ cid=`isula run -tid --log-driver=json-file --runtime $1 busybox sh` + if [ $? -ne 0 ]; then + msg_err "Failed to run container" + TC_RET_T=$(($TC_RET_T+1)) + fi +- do_check_item ${cid} "logdriver = json-file" +- do_check_item ${cid} "rotate = 7" +- do_check_item ${cid} "size = 1MB" ++ do_check_item $1 ${cid} "driver\": \"json-file" ++ do_check_item $1 ${cid} "rotate\": \"7" ++ do_check_item $1 ${cid} "size\": \"1MB" + +- cid=`isula run -tid --log-driver=json-file --log-opt="max-file=8" busybox sh` ++ cid=`isula run -tid --log-driver=json-file --log-opt="max-file=8" --runtime $1 busybox sh` + if [ $? -ne 0 ]; then + msg_err "Failed to run container" + TC_RET_T=$(($TC_RET_T+1)) + fi +- do_check_item ${cid} "logdriver = json-file" +- do_check_item ${cid} "rotate = 8" +- do_check_item ${cid} "size = 1MB" ++ do_check_item $1 ${cid} "driver\": \"json-file" ++ do_check_item $1 ${cid} "rotate\": \"8" ++ do_check_item $1 ${cid} "size\": \"1MB" + +- cid=`isula run -tid --log-driver=json-file --log-opt="max-size=128KB" busybox sh` ++ cid=`isula run -tid --log-driver=json-file --log-opt="max-size=128KB" --runtime $1 busybox sh` + if [ $? -ne 0 ]; then + msg_err "Failed to run container" + TC_RET_T=$(($TC_RET_T+1)) + fi +- do_check_item ${cid} "logdriver = json-file" +- do_check_item ${cid} "rotate = 7" +- do_check_item ${cid} "size = 128KB" ++ do_check_item $1 ${cid} "driver\": \"json-file" ++ do_check_item $1 ${cid} "rotate\": \"7" ++ do_check_item $1 ${cid} "size\": \"128KB" + +- cid=`isula run -tid --log-driver=json-file --log-opt="disable-log=true" busybox sh` ++ cid=`isula run -tid --log-driver=json-file --log-opt="disable-log=true" --runtime $1 busybox sh` + if [ $? -ne 0 ]; then + msg_err "Failed to run container" + TC_RET_T=$(($TC_RET_T+1)) + fi +- cat ${ISULAD_ROOT_PATH}/engines/lcr/${cid}/config | grep console | grep "logfile =" +- if [ $? -eq 0 ]; then ++ cat ${RUNTIME_ROOT_PATH}/$1/$cid/config.json | grep console | grep "\"log.console.file\": \"none\"" ++ if [ $? -ne 0 ]; then + msg_err "Failed to disable log" + TC_RET_T=$(($TC_RET_T+1)) + fi +@@ -250,32 +250,48 @@ function do_test_container_log() + } + + function do_test_container_syslog() { +- do_test_syslog_helper "xxxx" ++ do_test_syslog_helper "xxxx" $1 + +- do_test_syslog_tag ++ do_test_syslog_tag $1 + } + + function do_test() { ++ local runtime=$1 ++ local test="log_test => (${runtime})" ++ msg_info "${test} starting..." ++ + check_valgrind_log + start_isulad_with_valgrind --container-log-opts="syslog-tag=xxxx" + +- do_test_container_syslog ++ do_test_container_syslog $runtime + + check_valgrind_log + start_isulad_with_valgrind --container-log-driver=json-file --container-log-opts="max-size=10MB" --container-log-opts="max-file=3" +- do_test_json_file_helper "3" "10MB" ++ ++ do_test_json_file_helper "3" "10MB" $runtime + + check_valgrind_log + start_isulad_with_valgrind +- do_test_container_log +-} + +-ret=0 ++ do_test_container_log $runtime ++ ++ msg_info "${test} finished with return ${TC_RET_T}..." ++ ++ return $TC_RET_T ++} + + do_pre + +-do_test ++ret=0 ++ ++for element in ${RUNTIME_LIST[@]}; ++do ++ do_test $element ++ if [ $? -ne 0 ];then ++ let "ret=$ret + 1" ++ fi ++done + + do_post + +-show_result $TC_RET_T "container log test" ++show_result $ret "container log test" +diff --git a/CI/test_cases/container_cases/pause.sh b/CI/test_cases/container_cases/pause.sh +index fe749cd1..5778642b 100755 +--- a/CI/test_cases/container_cases/pause.sh ++++ b/CI/test_cases/container_cases/pause.sh +@@ -25,10 +25,12 @@ source ../helpers.sh + + function do_test_t() + { +- echo "Do not support pause function now" +- return 0 ++ local runtime=$1 ++ local test="rm_test => (${runtime})" ++ msg_info "${test} starting..." ++ + containername=test_pause +- isula create -t --name $containername busybox ++ isula create -t --name $containername --runtime $runtime busybox + fn_check_eq "$?" "0" "create failed" + testcontainer $containername inited + +@@ -48,14 +50,19 @@ function do_test_t() + isula rm -f $containername + fn_check_eq "$?" "0" "rm failed" + ++ msg_info "${test} finished with return ${TC_RET_T}..." ++ + return $TC_RET_T + } + + ret=0 + +-do_test_t +-if [ $? -ne 0 ];then +- let "ret=$ret + 1" +-fi ++for element in ${RUNTIME_LIST[@]}; ++do ++ do_test_t $element ++ if [ $? -ne 0 ];then ++ let "ret=$ret + 1" ++ fi ++done + + show_result $ret "basic pause" +diff --git a/CI/test_cases/container_cases/resume.sh b/CI/test_cases/container_cases/resume.sh +index d0b6ba98..e7de41bb 100755 +--- a/CI/test_cases/container_cases/resume.sh ++++ b/CI/test_cases/container_cases/resume.sh +@@ -25,14 +25,16 @@ source ../helpers.sh + + function do_test_t() + { +- echo "Do not support resume function now" +- return 0 ++ local runtime=$1 ++ local test="kill_test => (${runtime})" ++ msg_info "${test} starting..." ++ + containername=test_resume +- isula create -t --name $containername busybox ++ isula create -t --name $containername --runtime $runtime busybox + fn_check_eq "$?" "0" "create failed" + testcontainer $containername inited + +- isula resume $containername ++ isula unpause $containername + fn_check_ne "$?" "0" "resume should fail" + testcontainer $containername inited + +@@ -45,21 +47,26 @@ function do_test_t() + + testcontainer $containername paused + +- isula resume $containername ++ isula unpause $containername + fn_check_eq "$?" "0" "resume failed" + testcontainer $containername running + + isula rm -f $containername + fn_check_eq "$?" "0" "rm failed" + ++ msg_info "${test} finished with return ${TC_RET_T}..." ++ + return $TC_RET_T + } + + ret=0 + +-do_test_t +-if [ $? -ne 0 ];then +- let "ret=$ret + 1" +-fi ++for element in ${RUNTIME_LIST[@]}; ++do ++ do_test_t $element ++ if [ $? -ne 0 ];then ++ let "ret=$ret + 1" ++ fi ++done + + show_result $ret "basic resume" +diff --git a/CI/test_cases/container_cases/rm.sh b/CI/test_cases/container_cases/rm.sh +index 2903edab..3deae453 100755 +--- a/CI/test_cases/container_cases/rm.sh ++++ b/CI/test_cases/container_cases/rm.sh +@@ -26,7 +26,7 @@ source ../helpers.sh + function rm_stopped_container() + { + containername=test_rm_stopped +- isula create -t --name $containername busybox ++ isula create -t --name $containername --runtime $1 busybox + fn_check_eq "$?" "0" "create failed" + testcontainer $containername inited + +@@ -40,7 +40,7 @@ function rm_stopped_container() + function rm_running_container() + { + containername=test_rm_running +- isula run -td --name $containername busybox ++ isula run -td --name $containername --runtime $1 busybox + fn_check_eq "$?" "0" "run failed" + testcontainer $containername running + +@@ -61,7 +61,7 @@ function rm_running_container() + function rm_running_container_force() + { + containername=test_rm_running_force +- conID=$(isula run -td --name $containername busybox) ++ conID=$(isula run -td --name $containername --runtime $1 busybox) + fn_check_eq "$?" "0" "run failed" + testcontainer $containername running + +@@ -83,18 +83,27 @@ function rm_running_container_force() + + function do_test_t() + { +- rm_stopped_container +- rm_running_container +- rm_running_container_force ++ local runtime=$1 ++ local test="rm_test => (${runtime})" ++ msg_info "${test} starting..." ++ ++ rm_stopped_container $runtime ++ rm_running_container $runtime ++ rm_running_container_force $runtime ++ ++ msg_info "${test} finished with return ${TC_RET_T}..." + + return $TC_RET_T + } + + ret=0 + +-do_test_t +-if [ $? -ne 0 ];then +- let "ret=$ret + 1" +-fi ++for element in ${RUNTIME_LIST[@]}; ++do ++ do_test_t $element ++ if [ $? -ne 0 ];then ++ let "ret=$ret + 1" ++ fi ++done + + show_result $ret "basic rm" +diff --git a/CI/test_cases/container_cases/start.sh b/CI/test_cases/container_cases/start.sh +index c3dc9c27..c2d0d02f 100755 +--- a/CI/test_cases/container_cases/start.sh ++++ b/CI/test_cases/container_cases/start.sh +@@ -26,7 +26,7 @@ source ../helpers.sh + function do_test_t() + { + containername=test_start +- isula create -t --name $containername busybox ++ isula create -t --name $containername --runtime $1 busybox + fn_check_eq "$?" "0" "create failed" + testcontainer $containername inited + +@@ -51,7 +51,7 @@ function do_attach_local_test_t() + local test="container start with --attach local test => (${FUNCNAME[@]})" + + containername=start_attach +- isula create -ti --name $containername busybox /bin/sh -c 'echo "hello"' ++ isula create -ti --name $containername --runtime $1 busybox /bin/sh -c 'echo "hello"' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to create ${containername}" && ((ret++)) + testcontainer $containername inited + +@@ -62,7 +62,7 @@ function do_attach_local_test_t() + isula rm -f $containername + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to rm container" && ((ret++)) + +- id=`isula create -ti busybox /bin/sh -c 'ech "hello"'` ++ id=`isula create -ti --runtime $1 busybox /bin/sh -c 'ech "hello"'` + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to create container" && ((ret++)) + + isula start -a $id +@@ -88,19 +88,19 @@ function do_attach_remote_test_t() + start_isulad_with_valgrind -H "$config" + + containername=start_attach +- isula create -ti -H "$config" --name $containername busybox /bin/sh -c 'echo "hello"' ++ isula create -ti -H "$config" --name $containername --runtime $1 busybox /bin/sh -c 'echo "hello"' + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to create ${containername} remote" && ((ret++)) +- testcontainer $containername inited ++ testcontainer $containername inited "-H $config" + + result=`isula start -a -H "$config" $containername` + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to start -a ${containername} remote" && ((ret++)) +- testcontainer $containername exited ++ testcontainer $containername exited "-H $config" + + isula rm -f -H "$config" $containername + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to rm container remote" && ((ret++)) + + containername=start_exit +- isula run -it -H "$config" --name $containername busybox /bin/sh -c 'exit 5' ++ isula run -it -H "$config" --name $containername --runtime $1 busybox /bin/sh -c 'exit 5' + [[ $? -ne 5 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - invalid exit code with remote start" && ((ret++)) + + isula start -a -H "$config" $containername +@@ -119,12 +119,29 @@ function do_attach_remote_test_t() + return ${ret} + } + +-declare -i ans=0 ++function test_start() ++{ ++ local ret=0 ++ local runtime=$1 ++ local test="start_test => (${runtime})" ++ msg_info "${test} starting..." ++ ++ do_test_t $runtime || ((ret++)) + +-do_test_t || ((ans++)) ++ do_attach_local_test_t $runtime || ((ret++)) + +-do_attach_local_test_t || ((ans++)) ++ do_attach_remote_test_t $runtime || ((ret++)) ++ ++ msg_info "${test} finished with return ${ret}..." ++ ++ return $ret ++} ++ ++declare -i ans=0 + +-do_attach_remote_test_t || ((ans++)) ++for element in ${RUNTIME_LIST[@]}; ++do ++ test_start $element || ((ans++)) ++done + + show_result ${ans} "${curr_path}/${0}" +diff --git a/CI/test_cases/container_cases/stats.sh b/CI/test_cases/container_cases/stats.sh +index 705f7a7c..df241b90 100755 +--- a/CI/test_cases/container_cases/stats.sh ++++ b/CI/test_cases/container_cases/stats.sh +@@ -26,7 +26,8 @@ function test_stats_spec() + { + local ret=0 + local image="busybox" +- local test="container stats test => (${FUNCNAME[@]})" ++ local runtime=$1 ++ local test="container stats test with (${runtime}) => (${FUNCNAME[@]})" + statslog=/tmp/stats.log + + msg_info "${test} starting..." +@@ -44,11 +45,11 @@ function test_stats_spec() + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++)) + + container_name_init=stats_inited +- id_init=`isula create -t -n $container_name_init $image /bin/sh` ++ id_init=`isula create -t -n $container_name_init --runtime $runtime $image /bin/sh` + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++)) + + container_name_running=stats_running +- id_running=`isula run -td -n $container_name_running $image /bin/sh` ++ id_running=`isula run -td -n $container_name_running --runtime $runtime $image /bin/sh` + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++)) + + isula stats --no-stream > $statslog +@@ -73,14 +74,14 @@ function test_stats_spec() + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++)) + + container_name_pause=stats_paused +- id_pause=`isula run -td -n $container_name_pause $image /bin/sh` ++ id_pause=`isula run -td -n $container_name_pause --runtime $runtime $image /bin/sh` + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++)) + + isula pause $id_pause + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to pause running container" && ((ret++)) + + container_name_stop=stats_stopped +- id_stop=`isula run -td -n $container_name_stop $image /bin/sh` ++ id_stop=`isula run -td -n $container_name_stop --runtime $runtime $image /bin/sh` + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++)) + + isula stop -t 0 $id_stop +@@ -129,6 +130,9 @@ function test_stats_spec() + + declare -i ans=0 + +-test_stats_spec || ((ans++)) ++for element in ${RUNTIME_LIST[@]}; ++do ++ test_stats_spec $element || ((ans++)) ++done + + show_result ${ans} "${curr_path}/${0}" +diff --git a/CI/test_cases/container_cases/ulimit.sh b/CI/test_cases/container_cases/ulimit.sh +index c134a71a..29868041 100755 +--- a/CI/test_cases/container_cases/ulimit.sh ++++ b/CI/test_cases/container_cases/ulimit.sh +@@ -26,10 +26,11 @@ source ../helpers.sh + function test_ulimit() + { + local ret=0 ++ local runtime=$1 + local image="busybox" + ulimitlog=/tmp/ulimit.log + +- local test="ulimit test => (${FUNCNAME[@]})" ++ local test="ulimit test with (${runtime})=> (${FUNCNAME[@]})" + msg_info "${test} starting..." + + check_valgrind_log +@@ -38,37 +39,37 @@ function test_ulimit() + start_isulad_with_valgrind --default-ulimit nproc=2048:4096 --default-ulimit nproc=2048:8192 --default-ulimit nofile=1024:4096 + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - start isulad failed" && ((ret++)) + +- isula run --ulimit nproc= $image /bin/sh > $ulimitlog 2>&1 ++ isula run --ulimit nproc= $image --runtime $runtime /bin/sh > $ulimitlog 2>&1 + cat $ulimitlog | grep "delimiter '=' can't be the first or the last character" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check failed" && ((ret++)) + +- isula run --ulimit nproc=1024: $image /bin/sh > $ulimitlog 2>&1 ++ isula run --ulimit nproc=1024: --runtime $runtime $image /bin/sh > $ulimitlog 2>&1 + cat $ulimitlog | grep "delimiter ':' can't be the first or the last character" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check failed" && ((ret++)) + +- isula run --ulimit npro=1024:2048 $image /bin/sh > $ulimitlog 2>&1 ++ isula run --ulimit npro=1024:2048 --runtime $runtime $image /bin/sh > $ulimitlog 2>&1 + cat $ulimitlog | grep "Invalid ulimit type" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check failed" && ((ret++)) + +- isula run --ulimit nproc=4096:2048 $image /bin/sh > $ulimitlog 2>&1 ++ isula run --ulimit nproc=4096:2048 --runtime $runtime $image /bin/sh > $ulimitlog 2>&1 + cat $ulimitlog | grep "Ulimit soft limit must be less than or equal to hard limit" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check failed" && ((ret++)) + +- isula run --ulimit nproc=2048:4096.5 $image /bin/sh > $ulimitlog 2>&1 ++ isula run --ulimit nproc=2048:4096.5 --runtime $runtime $image /bin/sh > $ulimitlog 2>&1 + cat $ulimitlog | grep "Invalid ulimit hard value" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check failed" && ((ret++)) + +- isula run --ulimit nproc==2048:4096 $image /bin/sh > $ulimitlog 2>&1 ++ isula run --ulimit nproc==2048:4096 --runtime $runtime $image /bin/sh > $ulimitlog 2>&1 + cat $ulimitlog | grep "Invalid ulimit argument" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check failed" && ((ret++)) + +- isula run --ulimit nproc=2048::4096 $image /bin/sh > $ulimitlog 2>&1 ++ isula run --ulimit nproc=2048::4096 --runtime $runtime $image /bin/sh > $ulimitlog 2>&1 + cat $ulimitlog | grep "Too many limit value arguments" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check failed" && ((ret++)) + + container_name="ulimit_test" + +- isula run -td -n $container_name --ulimit nofile=20480:40960 --ulimit core=1024:2048 $image /bin/sh ++ isula run -td -n $container_name --ulimit nofile=20480:40960 --ulimit core=1024:2048 --runtime $runtime $image /bin/sh + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check failed" && ((ret++)) + + isula exec $container_name /bin/sh -c "cat /proc/self/limits" | grep "Max open files" |awk '{ print $(NF-1) }' |grep 40960 +@@ -106,6 +107,9 @@ function test_ulimit() + + declare -i ans=0 + +-test_ulimit || ((ans++)) ++for element in ${RUNTIME_LIST[@]}; ++do ++ test_ulimit $element || ((ans++)) ++done + + show_result ${ans} "${curr_path}/${0}" +diff --git a/CI/test_cases/container_cases/update.sh b/CI/test_cases/container_cases/update.sh +index 1e1e9cdc..d379acc2 100755 +--- a/CI/test_cases/container_cases/update.sh ++++ b/CI/test_cases/container_cases/update.sh +@@ -26,7 +26,7 @@ source ../helpers.sh + function do_test_t() + { + containername=test_update +- containerid=`isula create -t --name $containername busybox` ++ containerid=`isula create -t --runtime $1 --name $containername busybox` + fn_check_eq "$?" "0" "create failed" + testcontainer $containername inited + +@@ -101,7 +101,24 @@ function do_test_t() + + main=$(uname -r | awk -F . '{print $1}') + minor=$(uname -r | awk -F . '{print $2}') +- if [[ ${main} -lt 5 ]] || [[ ${main} -eq 5 ]] && [[ ${minor} -lt 11 ]]; then ++ enable=1 ++ if [ $1 == "runc" ]; then ++ version=$(runc --version | grep runc) ++ # Runc does not support '--kernel-memory' options from v1.0.0-rc94 version ++ limit=(1 0 0 93) ++ array=`echo $version |egrep -o "[0-9]*"` ++ index=0 ++ for i in $(echo $array| awk '{print $1,$2}') ++ do ++ echo $i ++ if [[ $i -gt ${limit[index]} ]]; then ++ enable=0 ++ break ++ fi ++ let "index+=1" ++ done ++ fi ++ if [[ ${main} -lt 5 ]] || [[ ${main} -eq 5 ]] && [[ ${minor} -lt 11 ]] && [[ ${enable} -eq 1 ]]; then + isula update --kernel-memory 2000000000 $containername + fn_check_eq "$?" "0" "update failed" + +@@ -124,7 +141,7 @@ function do_test_t() + function do_test_t1() + { + containername=test_update1 +- containerid=`isula run -itd --memory 500M --name $containername busybox` ++ containerid=`isula run -itd --runtime $1 --memory 500M --name $containername busybox` + fn_check_eq "$?" "0" "run failed" + + isula inspect $containerid | grep "MemorySwap" | grep "1048576000" +@@ -146,16 +163,23 @@ function do_test_t1() + + ret=0 + +-do_test_t +-if [ $? -ne 0 ];then +- let "ret=$ret + 1" +-fi ++for element in ${RUNTIME_LIST[@]}; ++do ++ test="update test => (${element})" ++ msg_info "${test} starting..." + +-if [ -f "/sys/fs/cgroup/memory/memory.memsw.usage_in_bytes" ];then +- do_test_t1 ++ do_test_t $element + if [ $? -ne 0 ];then +- let "ret=$ret + 1" ++ let "ret=$ret + 1" ++ fi ++ ++ if [ -f "/sys/fs/cgroup/memory/memory.memsw.usage_in_bytes" ];then ++ do_test_t1 $element ++ if [ $? -ne 0 ];then ++ let "ret=$ret + 1" ++ fi + fi +-fi ++ msg_info "${test} finished with return ${ret}..." ++done + + show_result $ret "basic update" +diff --git a/CI/test_cases/helpers.sh b/CI/test_cases/helpers.sh +index 5ea4ff94..6b4eea24 100755 +--- a/CI/test_cases/helpers.sh ++++ b/CI/test_cases/helpers.sh +@@ -21,11 +21,14 @@ declare -a lines + + # Root directory of integration tests. + LCR_ROOT_PATH="/var/lib/isulad/engines/lcr" ++RUNTIME_ROOT_PATH="/var/lib/isulad/engines" + valgrind_log="/tmp/valgrind.log" + ISUALD_LOG="/var/lib/isulad/isulad.log" + ISULAD_ROOT_PATH="/var/lib/isulad" + ISULAD_RUN_ROOT_PATH="/var/run/isulad" + ++RUNTIME_LIST=(lcr runc) ++ + enable_native_network=0 + + declare -r -i FAILURE=1 +@@ -60,7 +63,7 @@ function fn_check_ne() { + } + + function testcontainer() { +- st=`isula inspect -f '{{json .State.Status}}' "$1"` ++ st=`isula inspect -f '{{json .State.Status}}' $3 "$1"` + if ! [[ "${st}" =~ "$2" ]];then + echo "expect status $2, but get ${st}" + TC_RET_T=$(($TC_RET_T+1)) +-- +2.25.1 + diff --git a/0053-bugfix-when-refresh-can-t-load-or-pull-images.patch b/0053-bugfix-when-refresh-can-t-load-or-pull-images.patch new file mode 100644 index 0000000..da4bffe --- /dev/null +++ b/0053-bugfix-when-refresh-can-t-load-or-pull-images.patch @@ -0,0 +1,319 @@ +From 9d6df0b3065867d5ca1a597bedb10eab5a1c9235 Mon Sep 17 00:00:00 2001 +From: "Neil.wrz" +Date: Mon, 20 Mar 2023 23:47:25 -0700 +Subject: [PATCH 53/53] bugfix when refresh can't load or pull images + +Signed-off-by: Neil.wrz +--- + src/daemon/modules/image/oci/oci_image.c | 105 +++++++++++++++++- + .../remote_layer_support/remote_support.c | 34 +++++- + .../remote_layer_support/remote_support.h | 4 +- + .../modules/image/oci/storage/storage.c | 2 +- + .../modules/image/oci/storage/storage.h | 2 + + 5 files changed, 143 insertions(+), 4 deletions(-) + +diff --git a/src/daemon/modules/image/oci/oci_image.c b/src/daemon/modules/image/oci/oci_image.c +index fa92a861..40e9a88f 100644 +--- a/src/daemon/modules/image/oci/oci_image.c ++++ b/src/daemon/modules/image/oci/oci_image.c +@@ -44,6 +44,39 @@ + + struct oci_image_module_data g_oci_image_module_data = { 0 }; + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++// intend to make remote refresh and oci ops exlusive ++static bool g_enable_remote; ++static pthread_rwlock_t g_remote_lock = PTHREAD_RWLOCK_INITIALIZER; ++ ++static inline bool oci_remote_lock(pthread_rwlock_t *remote_lock, bool writable) ++{ ++ int nret = 0; ++ ++ if (writable) { ++ nret = pthread_rwlock_wrlock(remote_lock); ++ } else { ++ nret = pthread_rwlock_rdlock(remote_lock); ++ } ++ if (nret != 0) { ++ ERROR("Lock memory store failed: %s", strerror(nret)); ++ return false; ++ } ++ ++ return true; ++} ++ ++static inline void oci_remote_unlock(pthread_rwlock_t *remote_lock) ++{ ++ int nret = 0; ++ ++ nret = pthread_rwlock_unlock(remote_lock); ++ if (nret != 0) { ++ FATAL("Unlock memory store failed: %s", strerror(nret)); ++ } ++} ++#endif ++ + static void free_oci_image_data(void) + { + free(g_oci_image_module_data.root_dir); +@@ -220,6 +253,7 @@ static int storage_module_init_helper(const isulad_daemon_configs *args) + + #ifdef ENABLE_REMOTE_LAYER_STORE + storage_opts->enable_remote_layer = args->storage_enable_remote_layer; ++ storage_opts->remote_lock = &g_remote_lock; + #endif + + if (util_dup_array_of_strings((const char **)args->storage_opts, args->storage_opts_len, &storage_opts->driver_opts, +@@ -303,6 +337,10 @@ int oci_init(const isulad_daemon_configs *args) + goto out; + } + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ g_enable_remote = args->storage_enable_remote_layer; ++#endif ++ + if (storage_module_init_helper(args) != 0) { + ret = -1; + goto out; +@@ -321,6 +359,7 @@ void oci_exit() + + int oci_pull_rf(const im_pull_request *request, im_pull_response *response) + { ++ int ret = 0; + if (request == NULL || request->image == NULL || response == NULL) { + ERROR("Invalid NULL param"); + return -1; +@@ -331,8 +370,24 @@ int oci_pull_rf(const im_pull_request *request, im_pull_response *response) + isulad_try_set_error_message("Invalid image name: %s", request->image); + return -1; + } ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ // read lock here because pull have exclusive access against remote refresh ++ // pull can work concurrently with other oci operations. ++ if (g_enable_remote && !oci_remote_lock(&g_remote_lock, false)) { ++ ERROR("Failed to lock oci remote lock when load image"); ++ return -1; ++ } ++#endif ++ ++ ret = oci_do_pull_image(request, response); ++ ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ if (g_enable_remote) { ++ oci_remote_unlock(&g_remote_lock); ++ } ++#endif + +- return oci_do_pull_image(request, response); ++ return ret; + } + + int oci_prepare_rf(const im_prepare_request *request, char **real_rootfs) +@@ -441,6 +496,15 @@ int oci_rmi(const im_rmi_request *request) + return -1; + } + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ // read lock here because load have exclusive access against remote refresh ++ // load can work concurrently with other oci operations. ++ if (g_enable_remote && !oci_remote_lock(&g_remote_lock, false)) { ++ ERROR("Failed to lock oci remote lock when load image"); ++ return -1; ++ } ++#endif ++ + if (!util_valid_image_name(request->image.image)) { + ERROR("Invalid image name: %s", request->image.image); + isulad_try_set_error_message("Invalid image name: %s", request->image.image); +@@ -502,6 +566,11 @@ int oci_rmi(const im_rmi_request *request) + } + + out: ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ if (g_enable_remote) { ++ oci_remote_unlock(&g_remote_lock); ++ } ++#endif + free(real_image_name); + free(image_ID); + util_free_array_by_len(image_names, image_names_len); +@@ -527,7 +596,24 @@ int oci_import(const im_import_request *request, char **id) + goto err_out; + } + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ // read lock here because import have exclusive access against remote refresh ++ // import can work concurrently with other oci operations. ++ if (g_enable_remote && !oci_remote_lock(&g_remote_lock, false)) { ++ ERROR("Failed to lock oci remote lock when load image"); ++ ret = -1; ++ goto err_out; ++ } ++#endif ++ + ret = oci_do_import(request->file, dest_name, id); ++ ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ if (g_enable_remote) { ++ oci_remote_unlock(&g_remote_lock); ++ } ++#endif ++ + if (ret != 0) { + goto err_out; + } +@@ -677,7 +763,24 @@ int oci_load_image(const im_load_request *request) + goto out; + } + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ // read lock here because load have exclusive access against remote refresh ++ // load can work concurrently with other oci operations. ++ if (g_enable_remote && !oci_remote_lock(&g_remote_lock, false)) { ++ ERROR("Failed to lock oci remote lock when load image"); ++ ret = -1; ++ goto out; ++ } ++#endif ++ + ret = oci_do_load(request); ++ ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ if (g_enable_remote) { ++ oci_remote_unlock(&g_remote_lock); ++ } ++#endif ++ + if (ret != 0) { + ERROR("Failed to load image"); + goto out; +diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c +index 3c7d0f54..7d457755 100644 +--- a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c ++++ b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c +@@ -24,10 +24,38 @@ struct supporters { + struct remote_image_data *image_data; + struct remote_layer_data *layer_data; + struct remote_overlay_data *overlay_data; ++ pthread_rwlock_t *remote_lock; + }; + + static struct supporters supporters; + ++static inline bool remote_refresh_lock(pthread_rwlock_t *remote_lock, bool writable) ++{ ++ int nret = 0; ++ ++ if (writable) { ++ nret = pthread_rwlock_wrlock(remote_lock); ++ } else { ++ nret = pthread_rwlock_rdlock(remote_lock); ++ } ++ if (nret != 0) { ++ ERROR("Lock memory store failed: %s", strerror(nret)); ++ return false; ++ } ++ ++ return true; ++} ++ ++static inline void remote_refresh_unlock(pthread_rwlock_t *remote_lock) ++{ ++ int nret = 0; ++ ++ nret = pthread_rwlock_unlock(remote_lock); ++ if (nret != 0) { ++ FATAL("Unlock memory store failed: %s", strerror(nret)); ++ } ++} ++ + static void *remote_refresh_ro_symbol_link(void *arg) + { + struct supporters *refresh_supporters = (struct supporters *)arg; +@@ -37,16 +65,18 @@ static void *remote_refresh_ro_symbol_link(void *arg) + util_usleep_nointerupt(5 * 1000 * 1000); + DEBUG("remote refresh start\n"); + ++ remote_refresh_lock(supporters.remote_lock, true); + remote_overlay_refresh(refresh_supporters->overlay_data); + remote_layer_refresh(refresh_supporters->layer_data); + remote_image_refresh(refresh_supporters->image_data); ++ remote_refresh_unlock(supporters.remote_lock); + + DEBUG("remote refresh end\n"); + } + return NULL; + } + +-int remote_start_refresh_thread(void) ++int remote_start_refresh_thread(pthread_rwlock_t *remote_lock) + { + int res = 0; + pthread_t a_thread; +@@ -67,6 +97,8 @@ int remote_start_refresh_thread(void) + goto free_out; + } + ++ supporters.remote_lock = remote_lock; ++ + res = pthread_create(&a_thread, NULL, remote_refresh_ro_symbol_link, (void *)&supporters); + if (res != 0) { + CRIT("Thread creation failed"); +diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h +index 892a9155..30e3ebb0 100644 +--- a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h ++++ b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h +@@ -16,6 +16,8 @@ + #ifndef DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_REMOTE_SUPPORT_H + #define DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_REMOTE_SUPPORT_H + ++#include ++ + #include "linked_list.h" + #include "map.h" + #include "ro_symlink_maintain.h" +@@ -64,7 +66,7 @@ void remote_overlay_refresh(struct remote_overlay_data *data); + bool remote_overlay_layer_valid(const char *layer_id); + + // start refresh remote +-int remote_start_refresh_thread(void); ++int remote_start_refresh_thread(pthread_rwlock_t *remote_lock); + + // extra map utils + char **remote_deleted_layers(const map_t *old, const map_t *new_l); +diff --git a/src/daemon/modules/image/oci/storage/storage.c b/src/daemon/modules/image/oci/storage/storage.c +index f9830ac3..836ccf4d 100644 +--- a/src/daemon/modules/image/oci/storage/storage.c ++++ b/src/daemon/modules/image/oci/storage/storage.c +@@ -1874,7 +1874,7 @@ int storage_module_init(struct storage_module_init_options *opts) + } + + #ifdef ENABLE_REMOTE_LAYER_STORE +- if (opts->enable_remote_layer && remote_start_refresh_thread() != 0) { ++ if (opts->enable_remote_layer && remote_start_refresh_thread(opts->remote_lock) != 0) { + ERROR("Failed to start remote refresh thread"); + } + #endif +diff --git a/src/daemon/modules/image/oci/storage/storage.h b/src/daemon/modules/image/oci/storage/storage.h +index 7404ee54..df9fd761 100644 +--- a/src/daemon/modules/image/oci/storage/storage.h ++++ b/src/daemon/modules/image/oci/storage/storage.h +@@ -18,6 +18,7 @@ + #include + #include + #include ++#include + #include + #include + +@@ -72,6 +73,7 @@ struct storage_module_init_options { + bool integration_check; + #ifdef ENABLE_REMOTE_LAYER_STORE + bool enable_remote_layer; ++ pthread_rwlock_t *remote_lock; + #endif + }; + +-- +2.25.1 + diff --git a/iSulad.spec b/iSulad.spec new file mode 100644 index 0000000..8779b85 --- /dev/null +++ b/iSulad.spec @@ -0,0 +1,877 @@ +%global _version 2.1.1 +%global _release 6 +%global is_systemd 1 +%global enable_shimv2 1 +%global is_embedded 1 + +Name: iSulad +Version: %{_version} +Release: %{_release} +Summary: Lightweight Container Runtime Daemon +License: Mulan PSL v2 +URL: https://gitee.com/openeuler/iSulad +Source: https://gitee.com/openeuler/iSulad/repository/archive/v%{version}.tar.gz +BuildRoot: {_tmppath}/iSulad-%{version} + +Patch0001: 0001-modify-dependence-from-lcr-to-libisula.patch +Patch0002: 0002-Add-unified-memory_swap_limit_in_bytes-fields-into-C.patch +Patch0003: 0003-Add-macro-for-protoc-cmake.patch +Patch0004: 0004-fix-design-typo.patch +Patch0005: 0005-fix-cpu-rt-review-comments.patch +Patch0006: 0006-fix-inspect.sh-failed.patch +Patch0007: 0007-add-CRI-ContainerStats-Service.patch +Patch0008: 0008-fix-isula-cpu-rt-CI.patch +Patch0009: 0009-fix-cpu-rt-CI.patch +Patch0010: 0010-fix-cpu-rt-CI.patch +Patch0011: 0011-Bugfix-in-config-and-executor.patch +Patch0012: 0012-fix-cpu-rt-disable-after-reboot-machine.patch +Patch0013: 0013-fix-selinux_label_ut-timeout-and-add-timeout-for-all.patch +Patch0014: 0014-add-retry-for-read-write.patch +Patch0015: 0015-support-pull-image-with-digest.patch +Patch0016: 0016-isulad-shim-support-execSync-with-timeout.patch +Patch0017: 0017-Refine-the-commit-info.patch +Patch0018: 0018-Refine-typo-of-word-container.patch +Patch0019: 0019-cleancode-for-read-write.patch +Patch0020: 0020-add-crictl-timeout-and-sync-for-CI.patch +Patch0021: 0021-unlock-m_podsLock-if-new-failed.patch +Patch0022: 0022-Update-CRI.patch +Patch0023: 0023-add-cgroup-cpu-ut.patch +Patch0024: 0024-remove-temp-variables.patch +Patch0025: 0025-fix-read-member-error-from-struct.patch +Patch0026: 0026-Fix-PR-runc.patch +Patch0027: 0027-allow-the-paused-container-to-be-stopped.patch +Patch0028: 0028-Refine.patch +Patch0029: 0029-support-isula-update-when-runtime-is-runc.patch +Patch0030: 0030-Refine-as-others-feedback.patch +Patch0031: 0031-fix-CRI-SetupPod-and-TearDownPod-deadlock.patch +Patch0032: 0032-remote-layer-store-demo.patch +Patch0033: 0033-add-ci-for-remote-ro.patch +Patch0034: 0034-change-sleep-to-usleep-to-avoid-lossing-of-accuracy.patch +Patch0035: 0035-fix-compile-error-when-not-enable-remote-ro.patch +Patch0036: 0036-adapt-to-repo-of-openeuler-url-changed.patch +Patch0037: 0037-change-goto-branch.patch +Patch0038: 0038-CI-not-enable-remote-ro-for-ut.patch +Patch0039: 0039-use-auto-free-to-proc_t.patch +Patch0040: 0040-modifying-cpurt-file-permissions.patch +Patch0041: 0041-use-CURLOPT_XFERINFOFUNCTION-instead-of-deprecated-C.patch +Patch0042: 0042-bugfix-remote-ro-try-add-or-remove-image-layer-twice.patch +Patch0043: 0043-bugfix-can-t-delete-layers-under-dir-overlay-layers.patch +Patch0044: 0044-doc-add-document-about-support-remote-ro-directory.patch +Patch0045: 0045-Refine-a-minor-log-message.patch +Patch0046: 0046-modify-the-return-value-of-the-util_waitpid_with_tim.patch +Patch0047: 0047-fix-util_getgrent_r-overflow.patch +Patch0048: 0048-add-ut-for-runc.patch +Patch0049: 0049-add-runc-doc.patch +Patch0050: 0050-fix-isula_rt_ops_ut-bugs.patch +Patch0051: 0051-refactor-remote-ro-code.patch +Patch0052: 0052-add-ci-for-runc.patch +Patch0053: 0053-bugfix-when-refresh-can-t-load-or-pull-images.patch + +%ifarch x86_64 aarch64 +Provides: libhttpclient.so()(64bit) +Provides: libisula_client.so()(64bit) +Provides: libisulad_img.so()(64bit) +Provides: libisulad_tools.so()(64bit) +%endif + +%if 0%{?is_systemd} +# Systemd 230 and up no longer have libsystemd-journal +BuildRequires: pkgconfig(systemd) +Requires: systemd-units +%else +Requires(post): chkconfig +Requires(preun): chkconfig +# This is for /sbin/service +Requires(preun): initscripts +%endif + +%if 0%{?is_embedded} +BuildRequires: sqlite-devel +Requires: sqlite +%endif + +%if %{defined openeuler} +BuildRequires: gtest-devel gmock-devel +%endif + +%define lcrver_lower 2.1.1-0 +%define lcrver_upper 2.1.2-0 + +BuildRequires: lcr-devel > %{lcrver_lower} lcr-devel < %{lcrver_upper} +BuildRequires: libisula-devel > %{lcrver_lower} libisula-devel < %{lcrver_upper} +BuildRequires: cmake gcc-c++ yajl-devel lxc lxc-devel +BuildRequires: grpc grpc-plugins grpc-devel protobuf-devel +BuildRequires: libcurl libcurl-devel libarchive-devel device-mapper-devel +BuildRequires: http-parser-devel +BuildRequires: libseccomp-devel libcap-devel libselinux-devel libwebsockets libwebsockets-devel +BuildRequires: systemd-devel git +%if 0%{?enable_shimv2} +BuildRequires: lib-shim-v2 lib-shim-v2-devel +%endif + + +Requires: libisula > %{lcrver_lower} libisula < %{lcrver_upper} +Requires: lcr > %{lcrver_lower} lcr < %{lcrver_upper} +Requires: grpc protobuf lxc +Requires: libcurl +Requires: http-parser libseccomp +Requires: libcap libselinux libwebsockets libarchive device-mapper +Requires: systemd +%if 0%{?enable_shimv2} +Requires: lib-shim-v2 +%endif + +%description +This is a umbrella project for gRPC-services based Lightweight Container +Runtime Daemon, written by C. + +%prep +%autosetup -n iSulad-v%{_version} -Sgit -p1 + +%build +mkdir -p build +cd build +%if 0%{?enable_shimv2} +%if %{defined openeuler} +%cmake -DDEBUG=ON -DCMAKE_SKIP_RPATH=TRUE -DLIB_INSTALL_DIR=%{_libdir} -DCMAKE_INSTALL_PREFIX=/usr -DENABLE_SHIM_V2=ON -DENABLE_UT=ON ../ +%else +%cmake -DDEBUG=ON -DCMAKE_SKIP_RPATH=TRUE -DLIB_INSTALL_DIR=%{_libdir} -DCMAKE_INSTALL_PREFIX=/usr -DENABLE_SHIM_V2=ON ../ +%endif +%else +%if %{defined openeuler} +%cmake -DDEBUG=ON -DCMAKE_SKIP_RPATH=TRUE -DLIB_INSTALL_DIR=%{_libdir} -DCMAKE_INSTALL_PREFIX=/usr -DENABLE_UT=ON ../ +%else +%cmake -DDEBUG=ON -DCMAKE_SKIP_RPATH=TRUE -DLIB_INSTALL_DIR=%{_libdir} -DCMAKE_INSTALL_PREFIX=/usr ../ +%endif +%endif +%make_build + +%check +%if %{defined openeuler} +cd build +# registry_images_ut and volume_ut must run with root user +ctest -E "registry_images_ut|volume_ut" +%endif + +%install +rm -rf %{buildroot} +cd build +install -d $RPM_BUILD_ROOT/%{_libdir} +install -m 0755 ./src/libisula_client.so %{buildroot}/%{_libdir}/libisula_client.so +install -m 0755 ./src/utils/http/libhttpclient.so %{buildroot}/%{_libdir}/libhttpclient.so +install -m 0755 ./src/libisulad_tools.so %{buildroot}/%{_libdir}/libisulad_tools.so +install -m 0755 ./src/daemon/modules/image/libisulad_img.so %{buildroot}/%{_libdir}/libisulad_img.so + +install -d $RPM_BUILD_ROOT/%{_libdir}/pkgconfig +install -m 0640 ./conf/isulad.pc %{buildroot}/%{_libdir}/pkgconfig/isulad.pc + +install -d $RPM_BUILD_ROOT/%{_bindir} + +install -m 0755 ./src/isula %{buildroot}/%{_bindir}/isula +install -m 0755 ./src/isulad-shim %{buildroot}/%{_bindir}/isulad-shim + +install -m 0755 ./src/isulad %{buildroot}/%{_bindir}/isulad + +install -d $RPM_BUILD_ROOT/%{_includedir}/isulad +install -m 0644 ../src/daemon/modules/api/image_api.h %{buildroot}/%{_includedir}/isulad/image_api.h + +install -d $RPM_BUILD_ROOT/%{_sysconfdir}/isulad +install -m 0640 ../src/contrib/config/daemon.json %{buildroot}/%{_sysconfdir}/isulad/daemon.json +install -m 0440 ../src/contrib/config/daemon_constants.json %{buildroot}/%{_sysconfdir}/isulad/daemon_constants.json +install -m 0640 ../src/contrib/config/seccomp_default.json %{buildroot}/%{_sysconfdir}/isulad/seccomp_default.json + +install -d $RPM_BUILD_ROOT/%{_sysconfdir}/default/isulad +install -m 0640 ../src/contrib/config/config.json %{buildroot}/%{_sysconfdir}/default/isulad/config.json +install -m 0640 ../src/contrib/config/systemcontainer_config.json %{buildroot}/%{_sysconfdir}/default/isulad/systemcontainer_config.json +install -m 0550 ../src/contrib/sysmonitor/isulad-check.sh %{buildroot}/%{_sysconfdir}/default/isulad/isulad-check.sh + +mkdir -p $RPM_BUILD_ROOT/%{_sysconfdir}/sysmonitor/process +cp ../src/contrib/sysmonitor/isulad-monit $RPM_BUILD_ROOT/etc/sysmonitor/process + +install -d $RPM_BUILD_ROOT/%{_sysconfdir}/default/isulad/hooks +install -m 0640 ../src/contrib/config/hooks/default.json %{buildroot}/%{_sysconfdir}/default/isulad/hooks/default.json + +install -d $RPM_BUILD_ROOT/%{_sysconfdir}/sysconfig +install -p -m 0640 ../src/contrib/config/iSulad.sysconfig $RPM_BUILD_ROOT/%{_sysconfdir}/sysconfig/iSulad + +%if 0%{?is_systemd} +install -d $RPM_BUILD_ROOT/%{_unitdir} +install -p -m 0640 ../src/contrib/init/isulad.service $RPM_BUILD_ROOT/%{_unitdir}/isulad.service +%else +install -d $RPM_BUILD_ROOT/%{_initddir} +install -p -m 0640 ../src/contrib/init/isulad.init $RPM_BUILD_ROOT/%{_initddir}/isulad.init +%endif + +%clean +rm -rf %{buildroot} + +%pre +# support update from lcrd to isulad, will remove in next version +if [ "$1" = "2" ]; then +%if 0%{?is_systemd} +systemctl stop lcrd &>/dev/null +systemctl disable lcrd &>/dev/null +if [ -e %{_sysconfdir}/isulad/daemon.json ];then + sed -i 's#/etc/default/lcrd/hooks#/etc/default/isulad/hooks#g' %{_sysconfdir}/isulad/daemon.json +fi +%else +/sbin/chkconfig --del lcrd &>/dev/null +%endif +fi + +%post +if ! getent group isula > /dev/null; then + groupadd --system isula +fi + +if [ "$1" = "1" ]; then +%if 0%{?is_systemd} +systemctl enable isulad +systemctl start isulad +%else +/sbin/chkconfig --add isulad +%endif +elif [ "$1" = "2" ]; then +%if 0%{?is_systemd} +# support update from lcrd to isulad, will remove in next version +if [ -e %{_unitdir}/lcrd.service.rpmsave ]; then + mv %{_unitdir}/lcrd.service.rpmsave %{_unitdir}/isulad.service + sed -i 's/lcrd/isulad/g' %{_unitdir}/isulad.service +fi +systemctl status isulad | grep 'Active:' | grep 'running' +if [ $? -eq 0 ]; then + systemctl restart isulad +else + systemctl start isulad +fi +%else +/sbin/service isulad status | grep 'Active:' | grep 'running' +if [ $? -eq 0 ]; then + /sbin/service isulad restart +fi +%endif +fi + +if ! getent group isula > /dev/null; then + groupadd --system isula +fi + +%preun +%if 0%{?is_systemd} +%systemd_preun isulad +%else +if [ $1 -eq 0 ] ; then + /sbin/service isulad stop >/dev/null 2>&1 + /sbin/chkconfig --del isulad +fi +%endif + +%postun +%if 0%{?is_systemd} +%systemd_postun_with_restart isulad +%else +if [ "$1" -ge "1" ] ; then + /sbin/service isulad condrestart >/dev/null 2>&1 || : +fi +%endif + +%files +%attr(0600,root,root) %{_sysconfdir}/sysmonitor/process/isulad-monit +%attr(0550,root,root) %{_sysconfdir}/default/isulad/isulad-check.sh +%defattr(0640,root,root,0750) +%{_sysconfdir}/isulad +%{_sysconfdir}/isulad/* +%{_sysconfdir}/default/* +%defattr(-,root,root,-) +%if 0%{?is_systemd} +%{_unitdir}/isulad.service +%attr(0640,root,root) %{_unitdir}/isulad.service +%else +%{_initddir}/isulad.init +%attr(0640,root,root) %{_initddir}/isulad.init +%endif +%{_includedir}/isulad/* +%attr(0755,root,root) %{_libdir}/pkgconfig +%attr(0640,root,root) %{_libdir}/pkgconfig/isulad.pc +%defattr(0755,root,root,0755) +%{_bindir}/* +%{_libdir}/* +%attr(0640,root,root) %{_sysconfdir}/sysconfig/iSulad +%attr(0640,root,root) %{_sysconfdir}/isulad/daemon.json + +%config(noreplace,missingok) %{_sysconfdir}/sysconfig/iSulad +%config(noreplace,missingok) %{_sysconfdir}/isulad/daemon.json +%if 0%{?is_systemd} +%config(noreplace,missingok) %{_unitdir}/isulad.service +%else +%config(noreplace,missingok) %{_initddir}/isulad.init +%endif + +%changelog +* Fri Mar 24 2023 wangrunze - 2.1.1-6 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: update from upstream to include remote feature + +* Thu Mar 16 2023 zhangxiaoyu - 2.1.1-5 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: use CURLOPT_XFERINFOFUNCTION instead of deprecated CURLOPT_PROGRESSFUNCTION since curl 7.32.0 + +* Wed Feb 22 2023 zhangxiaoyu - 2.1.1-4 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: upgrade from upstream + +* Thu Feb 16 2023 zhangxiaoyu - 2.1.1-3 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: add check + +* Mon Feb 06 2023 zhangxiaoyu - 2.1.1-2 +- Type: update +- ID: NA +- SUG: NA +- DESC: modify dependence from lcr to libisula + +* Mon Feb 06 2023 zhangxiaoyu - 2.1.1-1 +- Type: update +- ID: NA +- SUG: NA +- DESC: update to v2.1.1 + +* Tue Jan 03 2023 zhangxiaoyu - 2.0.18-1 +- Type: update +- ID: NA +- SUG: NA +- DESC: update to v2.0.18 + +* Thu Dec 22 2022 zhangxiaoyu - 2.0.17-14 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: remove clean_module_fill_ctx for libisulad_img.so + +* Mon Dec 19 2022 zhangxiaoyu - 2.0.17-13 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: update lcr dependence version + +* Fri Dec 16 2022 zhangxiaoyu - 2.0.17-12 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: update from upstream + +* Tue Dec 06 2022 zhangxiaoyu - 2.0.17-11 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: update from upstream + +* Mon Nov 28 2022 yangjiaqi - 2.0.17-10 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: remove chmod 751 permission for dirs by engine when user-remap enabled + +* Fri Nov 25 2022 zhangxiaoyu - 2.0.17-9 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: add primary group to additional groups + +* Mon Nov 21 2022 zhangxiaoyu - 2.0.17-8 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fix cannot install isulad and unknown option + +* Wed Nov 16 2022 zhangxiaoyu - 2.0.17-7 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: update tar package from source + +* Wed Nov 02 2022 wangrunze - 2.0.17-6 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fix cleanup module memory leak + +* Tue Nov 01 2022 zhangxiaoyu - 2.0.17-5 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: add ut and bugfix for device mapper and websocket + +* Mon Oct 31 2022 wujing - 2.0.17-4 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sync from openEuler + +* Wed Oct 19 2022 zhangxiaoyu - 2.0.17-3 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: add required package lcr clibcni lower and upper version + +* Mon Oct 10 2022 zhangxiaoyu - 2.0.17-2 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: use epoll instead of select for wait_exit_fifo + +* Sun Oct 09 2022 zhangxiaoyu - 2.0.17-1 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: update iSulad version to 2.0.17-1 + +* Thu Sep 29 2022 haozi007 - 2.0.16-8 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sync from openEuler + +* Tue Sep 20 2022 zhangxiaoyu - 2.0.16-7 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: change libisulad_tools.so mode + +* Thu Sep 15 2022 zhangxiaoyu - 2.0.16-6 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: shield upgrade error if lcrd not exist + +* Tue Sep 13 2022 zhangxiaoyu - 2.0.16-5 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: calculate timezone by tm_gmtoff + +* Thu Sep 08 2022 zhangxiaoyu - 2.0.16-4 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: invalid free default-runtime and cri-runtime after free json-confs + +* Wed Sep 07 2022 zhangxiaoyu - 2.0.16-3 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sycn patches from openeuler/iSulad + +* Tue Aug 30 2022 leizhongkai - 2.0.16-2 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sync from upstream openeuler/iSulad + +* Tue Aug 23 2022 wangfengtu - 2.0.16-1 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: update iSulad version to 2.0.16-1 + +* Mon Aug 22 2022 zhongtao - 2.0.15-6 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: remove rpath by cmake + +* Wed Aug 17 2022 haozi007 - 2.0.15-5 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sycn patches from openeuler + +* Mon Aug 15 2022 wangfengtu - 2.0.15-4 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: change default umask to 0022 + +* Tue Aug 9 2022 haozi007 - 2.0.15-3 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sycn patches from openeuler + +* Mon Aug 1 2022 chengzeruizhi - 2.0.15-2 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sycn patches from openeuler branch + +* Fri Jul 8 2022 haozi007 - 2.0.15-1 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: update version to v2.0.15 + +* Fri Jul 8 2022 haozi007 - 2.0.14-11 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: add limit to lcr version + +* Wed Jun 22 2022 yangjiaqi - 2.0.14-10 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sync from upstream openeuler/iSulad + +* Tue Jun 21 2022 zhangxiaoyu - 2.0.14-9 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sync from upstream openeuler/iSulad + +* Wed Jun 15 2022 chengzeruizhi - 2.0.14-8 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sync from upstream openeuler/iSulad + +* Tue May 31 2022 zhangxiaoyu - 2.0.14-7 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: fix type convert, add null pointer check, remove unuse macro + +* Tue May 31 2022 zhangxiaoyu - 2.0.14-6 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: fix different type convert and add check to arguments + +* Mon May 30 2022 chengzrz - 2.0.14-5 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: seccomp optimization + +* Fri May 27 2022 haozi007 - 2.0.14-4 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: improve fuzz test for pw and gr parser + +* Tue May 24 2022 wangfengtu - 2.0.14-3 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: fix install error when android + +* Tue May 24 2022 wangfengtu - 2.0.14-2 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: do not mkdir of isulad if no controller found + +* Mon May 23 2022 haozi007 - 2.0.14-1 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: update version to v2.0.14 + +* Mon May 16 2022 haozi007 - 2.0.13-5 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sync from upstream openEuler/iSulad + +* Tue May 10 2022 hejunjie - 2.0.13-4 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: bionic adaptation, increase lcov coverage + +* Thu May 5 2022 hejunjie - 2.0.13-3 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: bionic adaptation for pwgr obj parser + +* Mon Apr 25 2022 zhangxiaoyu - 2.0.13-2 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: sync from upstream + +* Mon Apr 18 2022 wangfengtu - 2.0.13-1 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: update version to v2.0.13 + +* Fri Mar 25 2022 wangfengtu - 2.0.12-1 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: update version to v2.0.12 + +* Thu Mar 17 2022 haozi007 - 2.0.11-6 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: remove unnecessary error message + +* Thu Mar 17 2022 wangfengtu - 2.0.11-5 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fix uid/gid error when load image + +* Wed Mar 09 2022 wangfengtu - 2.0.11-4 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: add provides of libisulad_tools.so + +* Thu Mar 03 2022 wangfengtu - 2.0.11-3 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: Add the function of isolating the user namespaces + +* Thu Mar 03 2022 wangfengtu - 2.0.11-2 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: add libisulad_tools.so + +* Thu Feb 24 2022 wangfengtu - 2.0.11-1 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: update version to v2.0.11 + +* Wed Jan 12 2022 wangfengtu - 2.0.10-15 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fix compile error of isula-transform + +* Wed Jan 12 2022 zhangxiaoyu - 2.0.10-14 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fix compile error with grpc 1.41.x + +* Tue Jan 4 2022 wangfengtu - 2.0.10-13 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fix compile error when building embedded image + +* Mon Dec 27 2021 wangfengtu - 2.0.10-12 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: sync patches from upstream + +* Thu Dec 09 2021 chengzeruizhi - 2.0.10-11 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fixed a bug that occurs when starting a container in host mode + +* Thu Dec 09 2021 wangfengtu - 2.0.10-10 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: remove dependance of sqlite + +* Mon Dec 06 2021 gaohuatao - 2.0.10-9 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: specify version + +* Fri Dec 03 2021 wangfengtu - 2.0.10-8 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fix undefined reference to `service_arguments_free' in libisulad_img.so + +* Thu Dec 02 2021 wangfengtu - 2.0.10-7 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: sync patches from upstream + +* Tue Nov 23 2021 chengzeruizhi - 2.0.10-6 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: modify the procedure of running a pod + +* Fri Nov 19 2021 gaohuatao - 2.0.10-5 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: sync from upstream + +* Fri Nov 19 2021 wangfengtu - 2.0.10-4 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fix memleak when use multiple --volumes-from + +* Tue Nov 16 2021 wujing - 2.0.10-3 +- Type: enhancement +- ID: NA +- SUG: NA +- DESC: add shimv2 build switch + +* Tue Nov 16 2021 wujing - 2.0.10-2 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: remove build platform restrictions + +* Tue Nov 09 2021 gaohuatao - 2.0.10-1 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: update from openeuler + +* Tue Oct 19 2021 wangfengtu - 2.0.9-20211019.121837.gitf067b3ce +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: strip sha256 prefix when decrease hold references + +* Fri Jun 25 2021 wujing - 2.0.9-20210625.165022.git5a088d9c +- Type: update to v2.0.9 +- ID: NA +- SUG: NA +- DESC: update from master + +* Tue May 18 2021 wangfengtu - 2.0.8-20210518.144540.git5288ed92 +- Type: sync from upstream +- ID: NA +- SUG: NA +- DESC: update from master + +* Fri Mar 26 2021 zhangxiaoyu - 2.0.8-20210326.094027.gitac974aa6 +- Type: sync from upstream +- ID: NA +- SUG: NA +- DESC: update from master + +* Tue Mar 23 2021 haozi007 - 20210323.094917.git7e6aa593 +- Type: sync from upstream +- ID: NA +- SUG: NA +- DESC: update from master + +* Tue Feb 2 2021 lifeng - 2.0.8-20210202.153251.gite082dcf3 +- Type: sync from upstream +- ID: NA +- SUG: NA +- DESC: update from master + +* Mon Jan 18 2021 lifeng - 2.0.8-20210118.195254.git077e10f2 +- Type: sync from upstream +- ID: NA +- SUG: NA +- DESC: update from master + +* Wed Dec 30 2020 lifeng - 2.0.8-20201230.155843.git6557a6eb +- Type: update to v2.0.8 +- ID: NA +- SUG: NA +- DESC: update from master + +* Mon Dec 7 2020 zhangxiaoyu - 2.0.7-20201207.151847.gita1fce123 +- Type: update +- ID: NA +- SUG: NA +- DESC: update from master + +* Sat Dec 5 2020 lifeng - 2.0.7-20201205.145752.gita461cc51 +- Type: bugfix +- ID:NA +- SUG:NA +- DESC: ignore list containers errors + +* Thu Dec 3 2020 haozi007 - 2.0.7-20201203.190902.git48f598fd +- Type:update from master +- ID:NA +- SUG:NA +- DESC: update from master + +* Sat Nov 28 2020 lifeng - 2.0.7-20201128.095506.git1e1623a5 +- Type: bugfix +- ID:NA +- SUG:NA +- DESC: Mounts: only qsort the configed mounts and make possible to bind mount /proc and /sys/fs. +- related lxc PR fixed: +- 1.add check whether have /proc mounts entry, if has, skip the auto +- 2.mount cgroup before do mount entrys +- 3.pass if the mount on top of /proc and the source of the mount is a proc filesystem + +* Wed Nov 25 2020 wangfengtu - 2.0.7-20201125.165149.git7d150c3c +- Type: bugfix +- ID:NA +- SUG:NA +- DESC: update from openeuler + +* Wed Nov 25 2020 wangfengtu - 2.0.6-20201125.160534.git9fb5e75d +- Type: bugfix +- ID:NA +- SUG:NA +- DESC: fix rpath not work + +* Thu Nov 12 2020 gaohuatao - 2.0.6-20201112.193005.git8a6b73c8 +- Type: update from openeuler +- ID:NA +- SUG:NA +- DESC: update from openeuler + +* Wed Oct 14 2020 lifeng68 - 2.0.6-20201014.152749.gitc8a43925 +- Type: upgrade to v2.0.6 +- ID:NA +- SUG:NA +- DESC: upgrade to v2.0.6 + +* Fri Sep 18 2020 - 2.0.5-20200918.112827.git9aea9b75 +- Type:bugfix +- ID:NA +- SUG:NA +- DESC: modify log level to warn + +* Mon Sep 14 2020 - 2.0.5-20200914.172527.gitae86920a +- Type:bugfix +- ID:NA +- SUG:NA +- DESC: remove unused config + +* Thu Sep 10 2020 - 2.0.5-20200910.144345.git71b1055b +- Type:enhancement +- ID:NA +- SUG:NA +- DESC: add chrpath + +* Fri Sep 04 2020 zhangxiaoyu - 2.0.5-20200904.114315.gitff1761c3 +- Type:enhancement +- ID:NA +- SUG:NA +- DESC: upgrade from v2.0.3 to v2.0.5 + +* Wed Sep 02 2020 YoungJQ - 2.0.3-20200902.114727.git6d945f26 +- Type:enhancement +- ID:NA +- SUG:NA +- DESC: modify source0 address diff --git a/sources b/sources new file mode 100644 index 0000000..62da0f6 --- /dev/null +++ b/sources @@ -0,0 +1 @@ +667c9f751daaac3b991659c27722724d v2.1.1.tar.gz -- cgit v1.2.3